From df48acb41d55734e5f0e37afa0fecf7bf741b68f Mon Sep 17 00:00:00 2001 From: George Kraft Date: Fri, 27 Jan 2017 09:32:51 -0600 Subject: [PATCH 1/4] slightly clean up kubedns reactive step --- .../reactive/kubernetes_master.py | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 6e80e490499..0958468c1a9 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -297,21 +297,6 @@ def remove_dashboard_addons(): @when_not('kube-dns.available') def start_kube_dns(): ''' State guard to starting DNS ''' - - # Interrogate the cluster to find out if we have at least one worker - # that is capable of running the workload. - - cmd = ['kubectl', 'get', 'nodes'] - try: - out = check_output(cmd) - if b'NAME' not in out: - hookenv.log('Unable to determine node count, waiting ' - 'until nodes are ready') - return - except CalledProcessError: - hookenv.log('kube-apiserver not ready, not requesting dns deployment') - return - message = 'Rendering the Kubernetes DNS files.' hookenv.log(message) hookenv.status_set('maintenance', message) @@ -325,8 +310,14 @@ def start_kube_dns(): 'dns_domain': hookenv.config('dns_domain') } } - create_addon('kubedns-controller.yaml', context) - create_addon('kubedns-svc.yaml', context) + + try: + create_addon('kubedns-controller.yaml', context) + create_addon('kubedns-svc.yaml', context) + except CalledProcessError: + hookenv.log('kubedns deploy failed, will try again soon') + return + set_state('kube-dns.available') From 5c5fa4370047c8da3a885cde44ec8f3ff31a796a Mon Sep 17 00:00:00 2001 From: George Kraft Date: Fri, 27 Jan 2017 09:33:04 -0600 Subject: [PATCH 2/4] Remove "Kubernetes master services ready." status message This message was occurring prematurely - the master is running, but the charm still has work to do. e.g. deploy KubeDNS. --- .../juju/layers/kubernetes-master/reactive/kubernetes_master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 0958468c1a9..0fd56553d50 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -208,7 +208,7 @@ def start_master(etcd, tls): hookenv.log('Starting {0} service.'.format(service)) host.service_start(service) hookenv.open_port(6443) - hookenv.status_set('active', 'Kubernetes master services ready.') + hookenv.log('Kubernetes master services ready.') set_state('kubernetes-master.components.started') From a371b1bc27e18ab85ceb371b38953c7344ebafcf Mon Sep 17 00:00:00 2001 From: George Kraft Date: Fri, 27 Jan 2017 09:33:10 -0600 Subject: [PATCH 3/4] Add "Waiting for kube-system pods to start" status message --- .../reactive/kubernetes_master.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 0fd56553d50..b23c1466966 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -19,7 +19,7 @@ import os import random import socket import string - +import json from shlex import split from subprocess import call @@ -182,7 +182,9 @@ def set_app_version(): @when('kube-dns.available', 'kubernetes-master.components.installed') def idle_status(): ''' Signal at the end of the run that we are running. ''' - if hookenv.config('service-cidr') != service_cidr(): + if not all_kube_system_pods_running(): + hookenv.status_set('waiting', 'Waiting for kube-system pods to start') + elif hookenv.config('service-cidr') != service_cidr(): hookenv.status_set('active', 'WARN: cannot change service-cidr, still using ' + service_cidr()) else: hookenv.status_set('active', 'Kubernetes master running.') @@ -657,3 +659,24 @@ def setup_tokens(token, username, user): token = ''.join(random.SystemRandom().choice(alpha) for _ in range(32)) with open(known_tokens, 'w') as stream: stream.write('{0},{1},{2}'.format(token, username, user)) + + +def all_kube_system_pods_running(): + ''' Check pod status in the kube-system namespace. Returns True if all + pods are running, False otherwise. ''' + cmd = ['kubectl', 'get', 'po', '-n', 'kube-system', '-o', 'json'] + + try: + output = check_output(cmd).decode('utf-8') + except CalledProcessError: + hookenv.log('failed to get kube-system pod status') + return False + + result = json.loads(output) + + for pod in result['items']: + status = pod['status']['phase'] + if status != 'Running': + return False + + return True From b0a9c00bbb3763dd3bac4f7cf98706449ff09d35 Mon Sep 17 00:00:00 2001 From: George Kraft Date: Fri, 27 Jan 2017 09:33:14 -0600 Subject: [PATCH 4/4] improve status messages around KubeDNS --- .../kubernetes-master/reactive/kubernetes_master.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index b23c1466966..8316e2d17f0 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -210,7 +210,6 @@ def start_master(etcd, tls): hookenv.log('Starting {0} service.'.format(service)) host.service_start(service) hookenv.open_port(6443) - hookenv.log('Kubernetes master services ready.') set_state('kubernetes-master.components.started') @@ -295,13 +294,11 @@ def remove_dashboard_addons(): remove_state('kubernetes.dashboard.available') -@when('kubernetes-master.components.installed') +@when('kubernetes-master.components.started') @when_not('kube-dns.available') def start_kube_dns(): ''' State guard to starting DNS ''' - message = 'Rendering the Kubernetes DNS files.' - hookenv.log(message) - hookenv.status_set('maintenance', message) + hookenv.status_set('maintenance', 'Deploying KubeDNS') context = { 'arch': arch(), @@ -317,7 +314,7 @@ def start_kube_dns(): create_addon('kubedns-controller.yaml', context) create_addon('kubedns-svc.yaml', context) except CalledProcessError: - hookenv.log('kubedns deploy failed, will try again soon') + hookenv.status_set('waiting', 'Waiting to retry KubeDNS deployment') return set_state('kube-dns.available')