Merge pull request #40691 from Cynerva/gkk/fix-master-ready-status

Automatic merge from submit-queue (batch tested with PRs 40691, 40551, 40683, 40700, 40702)

Juju kubernetes-master charm: improve status messages

**What this PR does / why we need it**:

This update to the kubernetes-master charm does the following:
1. Remove "Kubernetes master services ready" status which was occurring too early
2. Add "Waiting for kube-system pods to start" status
3. Replace "Rendering the Kubernetes DNS files." status with "Deploying KubeDNS"
4. Add "Waiting to retry KubeDNS deployment" status

The purpose of this is to give better feedback to the operator during cluster deployment.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*:

Fixes https://github.com/juju-solutions/bundle-canonical-kubernetes/issues/143, which we are tracking in a separate repository

**Special notes for your reviewer**:

This is a rebase of https://github.com/juju-solutions/kubernetes/pull/103, where prior review was done, though it was targeted against a fork.

**Release note**:

```release-note
Juju kubernetes-master charm: improve status messages
```

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -19,7 +19,7 @@ import os
 import random
 import socket
 import string
-
+import json
 
 from shlex import split
 from subprocess import call
@@ -182,7 +182,9 @@ def set_app_version():
 @when('kube-dns.available', 'kubernetes-master.components.installed')
 def idle_status():
     ''' Signal at the end of the run that we are running. '''
-    if hookenv.config('service-cidr') != service_cidr():
+    if not all_kube_system_pods_running():
+        hookenv.status_set('waiting', 'Waiting for kube-system pods to start')
+    elif hookenv.config('service-cidr') != service_cidr():
         hookenv.status_set('active', 'WARN: cannot change service-cidr, still using ' + service_cidr())
     else:
         hookenv.status_set('active', 'Kubernetes master running.')
@@ -208,7 +210,6 @@ def start_master(etcd, tls):
         hookenv.log('Starting {0} service.'.format(service))
         host.service_start(service)
     hookenv.open_port(6443)
-    hookenv.status_set('active', 'Kubernetes master services ready.')
     set_state('kubernetes-master.components.started')
 
 
@@ -293,28 +294,11 @@ def remove_dashboard_addons():
         remove_state('kubernetes.dashboard.available')
 
 
-@when('kubernetes-master.components.installed')
+@when('kubernetes-master.components.started')
 @when_not('kube-dns.available')
 def start_kube_dns():
     ''' State guard to starting DNS '''
-
-    # Interrogate the cluster to find out if we have at least one worker
-    # that is capable of running the workload.
-
-    cmd = ['kubectl', 'get', 'nodes']
-    try:
-        out = check_output(cmd)
-        if b'NAME' not in out:
-            hookenv.log('Unable to determine node count, waiting '
-                        'until nodes are ready')
-            return
-    except CalledProcessError:
-        hookenv.log('kube-apiserver not ready, not requesting dns deployment')
-        return
-
-    message = 'Rendering the Kubernetes DNS files.'
-    hookenv.log(message)
-    hookenv.status_set('maintenance', message)
+    hookenv.status_set('maintenance', 'Deploying KubeDNS')
 
     context = {
         'arch': arch(),
@@ -325,8 +309,14 @@ def start_kube_dns():
             'dns_domain': hookenv.config('dns_domain')
         }
     }
-    create_addon('kubedns-controller.yaml', context)
-    create_addon('kubedns-svc.yaml', context)
+
+    try:
+        create_addon('kubedns-controller.yaml', context)
+        create_addon('kubedns-svc.yaml', context)
+    except CalledProcessError:
+        hookenv.status_set('waiting', 'Waiting to retry KubeDNS deployment')
+        return
+
     set_state('kube-dns.available')
 
 
@@ -666,3 +656,24 @@ def setup_tokens(token, username, user):
         token = ''.join(random.SystemRandom().choice(alpha) for _ in range(32))
     with open(known_tokens, 'w') as stream:
         stream.write('{0},{1},{2}'.format(token, username, user))
+
+
+def all_kube_system_pods_running():
+    ''' Check pod status in the kube-system namespace. Returns True if all
+    pods are running, False otherwise. '''
+    cmd = ['kubectl', 'get', 'po', '-n', 'kube-system', '-o', 'json']
+
+    try:
+        output = check_output(cmd).decode('utf-8')
+    except CalledProcessError:
+        hookenv.log('failed to get kube-system pod status')
+        return False
+
+    result = json.loads(output)
+
+    for pod in result['items']:
+        status = pod['status']['phase']
+        if status != 'Running':
+            return False
+
+    return True