From 6d32bb001fe2cb5a0543665c0bd3363d45bded37 Mon Sep 17 00:00:00 2001
From: Konstantinos Tsakalozos <kos.tsakalozos@canonical.com>
Date: Wed, 26 Apr 2017 19:04:18 +0300
Subject: [PATCH 1/2] Retry methods and set states so we immediately report
 config changes.

---
 .../lib/charms/kubernetes/common.py           | 36 +++++++++++++++++++
 .../reactive/kubernetes_master.py             | 25 +++++++++++--
 2 files changed, 58 insertions(+), 3 deletions(-)

diff --git a/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py b/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py
index 054399aeef2..0f9caa523f8 100644
--- a/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py
+++ b/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py
@@ -17,6 +17,8 @@
 import re
 import subprocess
 
+from time import sleep
+
 
 def get_version(bin_name):
     """Get the version of an installed Kubernetes binary.
@@ -33,3 +35,37 @@ def get_version(bin_name):
     cmd = '{} --version'.format(bin_name).split()
     version_string = subprocess.check_output(cmd).decode('utf-8')
     return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3])
+
+
+def retry(times, delay_secs):
+    """ Decorator for retrying a method call.
+
+    Args:
+        times: How many times should we retry before giving up
+        delay_secs: Delay in secs
+
+    Returns: A callable that would return the last call outcome
+    """
+
+    def retry_decorator(func):
+        """ Decorator to wrap the function provided.
+
+        Args:
+            func: Provided function should return either True od False
+
+        Returns: A callable that would return the last call outcome
+
+        """
+        def _wrapped(*args, **kwargs):
+            res = func(*args, **kwargs)
+            attempt = 0
+            while not res and attempt < times:
+                sleep(delay_secs)
+                res = func(*args, **kwargs)
+                if res:
+                    break
+                attempt += 1
+            return res
+        return _wrapped
+
+    return retry_decorator
diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py
index a392a9dbff5..da815ef6be2 100644
--- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py
+++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py
@@ -39,6 +39,7 @@ from charms.reactive import is_state
 from charms.reactive import when, when_any, when_not
 from charms.reactive.helpers import data_changed
 from charms.kubernetes.common import get_version
+from charms.kubernetes.common import retry
 from charms.kubernetes.flagmanager import FlagManager
 
 from charmhelpers.core import hookenv
@@ -368,6 +369,7 @@ def push_api_data(kube_api):
 @when('kubernetes-master.components.started')
 def configure_cdk_addons():
     ''' Configure CDK addons '''
+    remove_state('cdk-addons.configured')
     dbEnabled = str(hookenv.config('enable-dashboard-addons')).lower()
     args = [
         'arch=' + arch(),
@@ -376,15 +378,31 @@ def configure_cdk_addons():
         'enable-dashboard=' + dbEnabled
     ]
     check_call(['snap', 'set', 'cdk-addons'] + args)
-    try:
-        check_call(['cdk-addons.apply'])
-    except CalledProcessError:
+    if not addons_ready():
         hookenv.status_set('waiting', 'Waiting to retry addon deployment')
         remove_state('cdk-addons.configured')
         return
+
     set_state('cdk-addons.configured')
 
 
+@retry(times=3, delay_secs=20)
+def addons_ready():
+    """
+    Test if the add ons got installed
+
+    Returns: True is the addons got applied
+
+    """
+    try:
+        check_call(['cdk-addons.apply'])
+        return True
+    except CalledProcessError:
+        hookenv.log("Addons are not ready yet.")
+        return False
+
+
+
 @when('loadbalancer.available', 'certificates.ca.available',
       'certificates.client.cert.available')
 def loadbalancer_kubeconfig(loadbalancer, ca, client):
@@ -792,6 +810,7 @@ def setup_tokens(token, username, user):
         stream.write('{0},{1},{2}'.format(token, username, user))
 
 
+@retry(times=2, delay_secs=20)
 def all_kube_system_pods_running():
     ''' Check pod status in the kube-system namespace. Returns True if all
     pods are running, False otherwise. '''

From 7c6547a145e30a9f07aabefa7d2469fd96840a59 Mon Sep 17 00:00:00 2001
From: Konstantinos Tsakalozos <kos.tsakalozos@canonical.com>
Date: Fri, 28 Apr 2017 18:06:39 +0300
Subject: [PATCH 2/2] Retry of pods availablity 6 times over a minute

---
 .../juju/layers/kubernetes-master/reactive/kubernetes_master.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py
index da815ef6be2..b506386f1c0 100644
--- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py
+++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py
@@ -810,7 +810,7 @@ def setup_tokens(token, username, user):
         stream.write('{0},{1},{2}'.format(token, username, user))
 
 
-@retry(times=2, delay_secs=20)
+@retry(times=3, delay_secs=10)
 def all_kube_system_pods_running():
     ''' Check pod status in the kube-system namespace. Returns True if all
     pods are running, False otherwise. '''