diff --git a/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py b/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py index 054399aeef2..0f9caa523f8 100644 --- a/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py +++ b/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py @@ -17,6 +17,8 @@ import re import subprocess +from time import sleep + def get_version(bin_name): """Get the version of an installed Kubernetes binary. @@ -33,3 +35,37 @@ def get_version(bin_name): cmd = '{} --version'.format(bin_name).split() version_string = subprocess.check_output(cmd).decode('utf-8') return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3]) + + +def retry(times, delay_secs): + """ Decorator for retrying a method call. + + Args: + times: How many times should we retry before giving up + delay_secs: Delay in secs + + Returns: A callable that would return the last call outcome + """ + + def retry_decorator(func): + """ Decorator to wrap the function provided. + + Args: + func: Provided function should return either True od False + + Returns: A callable that would return the last call outcome + + """ + def _wrapped(*args, **kwargs): + res = func(*args, **kwargs) + attempt = 0 + while not res and attempt < times: + sleep(delay_secs) + res = func(*args, **kwargs) + if res: + break + attempt += 1 + return res + return _wrapped + + return retry_decorator diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index e6dfe5fbf8f..1dc970dcc69 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -39,6 +39,7 @@ from charms.reactive import is_state from charms.reactive import when, when_any, when_not from charms.reactive.helpers import data_changed from charms.kubernetes.common import get_version +from charms.kubernetes.common import retry from charms.kubernetes.flagmanager import FlagManager from charmhelpers.core import hookenv @@ -414,6 +415,7 @@ def push_api_data(kube_api): @when('kubernetes-master.components.started') def configure_cdk_addons(): ''' Configure CDK addons ''' + remove_state('cdk-addons.configured') dbEnabled = str(hookenv.config('enable-dashboard-addons')).lower() args = [ 'arch=' + arch(), @@ -422,15 +424,31 @@ def configure_cdk_addons(): 'enable-dashboard=' + dbEnabled ] check_call(['snap', 'set', 'cdk-addons'] + args) - try: - check_call(['cdk-addons.apply']) - except CalledProcessError: + if not addons_ready(): hookenv.status_set('waiting', 'Waiting to retry addon deployment') remove_state('cdk-addons.configured') return + set_state('cdk-addons.configured') +@retry(times=3, delay_secs=20) +def addons_ready(): + """ + Test if the add ons got installed + + Returns: True is the addons got applied + + """ + try: + check_call(['cdk-addons.apply']) + return True + except CalledProcessError: + hookenv.log("Addons are not ready yet.") + return False + + + @when('loadbalancer.available', 'certificates.ca.available', 'certificates.client.cert.available') def loadbalancer_kubeconfig(loadbalancer, ca, client): @@ -838,6 +856,7 @@ def setup_tokens(token, username, user): stream.write('{0},{1},{2}\n'.format(token, username, user)) +@retry(times=3, delay_secs=10) def all_kube_system_pods_running(): ''' Check pod status in the kube-system namespace. Returns True if all pods are running, False otherwise. '''