diff --git a/cluster/juju/layers/kubernetes-master/actions.yaml b/cluster/juju/layers/kubernetes-master/actions.yaml index cfb74c248fc..89cd502b049 100644 --- a/cluster/juju/layers/kubernetes-master/actions.yaml +++ b/cluster/juju/layers/kubernetes-master/actions.yaml @@ -1,7 +1,7 @@ restart: description: Restart the Kubernetes master services on demand. create-rbd-pv: - description: Create RADOS Block Device (RDB) volume in Ceph and creates PersistentVolume. + description: Create RADOS Block Device (RDB) volume in Ceph and creates PersistentVolume. Note this is deprecated on Kubernetes >= 1.10 in favor of CSI, where PersistentVolumes are created dynamically to back PersistentVolumeClaims. params: name: type: string diff --git a/cluster/juju/layers/kubernetes-master/actions/create-rbd-pv b/cluster/juju/layers/kubernetes-master/actions/create-rbd-pv index 697d8d92227..b6b229d63a6 100755 --- a/cluster/juju/layers/kubernetes-master/actions/create-rbd-pv +++ b/cluster/juju/layers/kubernetes-master/actions/create-rbd-pv @@ -38,6 +38,14 @@ def main(): this script thinks the environment is 'sane' enough to provision volumes. ''' + # k8s >= 1.10 uses CSI and doesn't directly create persistent volumes + if get_version('kube-apiserver') >= (1, 10): + print('This action is deprecated in favor of CSI creation of persistent volumes') + print('in Kubernetes >= 1.10. Just create the PVC and a PV will be created') + print('for you.') + action_fail('Deprecated, just create PVC.') + return + # validate relationship pre-reqs before additional steps can be taken if not validate_relation(): print('Failed ceph relationship check') @@ -89,6 +97,23 @@ def main(): check_call(cmd) +def get_version(bin_name): + """Get the version of an installed Kubernetes binary. + + :param str bin_name: Name of binary + :return: 3-tuple version (maj, min, patch) + + Example:: + + >>> `get_version('kubelet') + (1, 6, 0) + + """ + cmd = '{} --version'.format(bin_name).split() + version_string = check_output(cmd).decode('utf-8') + return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3]) + + def action_get_or_default(key): ''' Convenience method to manage defaults since actions dont appear to properly support defaults ''' diff --git a/cluster/juju/layers/kubernetes-master/config.yaml b/cluster/juju/layers/kubernetes-master/config.yaml index b24eaf9e7a3..01171c058fd 100644 --- a/cluster/juju/layers/kubernetes-master/config.yaml +++ b/cluster/juju/layers/kubernetes-master/config.yaml @@ -147,3 +147,9 @@ options: default: true description: | If true the metrics server for Kubernetes will be deployed onto the cluster. + default-storage: + type: string + default: "auto" + description: | + The storage class to make the default storage class. Allowed values are "auto", + "none", "ceph-xfs", "ceph-ext4". Note: Only works in Kubernetes >= 1.10 diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index d0b22ab9daa..8842930d588 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -28,6 +28,7 @@ import ipaddress from charms.leadership import leader_get, leader_set from shutil import move +from tempfile import TemporaryDirectory from pathlib import Path from shlex import split @@ -711,7 +712,8 @@ def kick_api_server(tls): tls_client.reset_certificate_write_flag('server') -@when('kubernetes-master.components.started') +@when_any('kubernetes-master.components.started', 'ceph-storage.configured') +@when('leadership.is_leader') def configure_cdk_addons(): ''' Configure CDK addons ''' remove_state('cdk-addons.configured') @@ -723,6 +725,21 @@ def configure_cdk_addons(): dbEnabled = str(hookenv.config('enable-dashboard-addons')).lower() dnsEnabled = str(hookenv.config('enable-kube-dns')).lower() metricsEnabled = str(hookenv.config('enable-metrics')).lower() + if (is_state('ceph-storage.configured') and + get_version('kube-apiserver') >= (1, 10)): + cephEnabled = "true" + else: + cephEnabled = "false" + ceph_ep = endpoint_from_flag('ceph-storage.available') + ceph = {} + default_storage = '' + if ceph_ep: + b64_ceph_key = base64.b64encode(ceph_ep.key().encode('utf-8')) + ceph['admin_key'] = b64_ceph_key.decode('ascii') + ceph['kubernetes_key'] = b64_ceph_key.decode('ascii') + ceph['mon_hosts'] = ceph_ep.mon_hosts() + default_storage = hookenv.config('default-storage') + args = [ 'arch=' + arch(), 'dns-ip=' + get_deprecated_dns_ip(), @@ -731,7 +748,12 @@ def configure_cdk_addons(): 'enable-dashboard=' + dbEnabled, 'enable-kube-dns=' + dnsEnabled, 'enable-metrics=' + metricsEnabled, - 'enable-gpu=' + str(gpuEnable).lower() + 'enable-gpu=' + str(gpuEnable).lower(), + 'enable-ceph=' + cephEnabled, + 'ceph-admin-key=' + (ceph.get('admin_key', '')), + 'ceph-kubernetes-key=' + (ceph.get('admin_key', '')), + 'ceph-mon-hosts="' + (ceph.get('mon_hosts', '')) + '"', + 'default-storage=' + default_storage, ] check_call(['snap', 'set', 'cdk-addons'] + args) if not addons_ready(): @@ -806,6 +828,15 @@ def ceph_storage(ceph_admin): configuration, and the ceph secret key file used for authentication. This method will install the client package, and render the requisit files in order to consume the ceph-storage relation.''' + + # deprecated in 1.10 in favor of using CSI + if get_version('kube-apiserver') >= (1, 10): + # this is actually false, but by setting this flag we won't keep + # running this function for no reason. Also note that we watch this + # flag to run cdk-addons.apply. + set_state('ceph-storage.configured') + return + ceph_context = { 'mon_hosts': ceph_admin.mon_hosts(), 'fsid': ceph_admin.fsid(), @@ -1662,6 +1693,10 @@ def _daemon_env_path(component): return _snap_common_path(component) / 'environment' +def _cdk_addons_template_path(): + return Path('/snap/cdk-addons/current/templates') + + def _write_gcp_snap_config(component): # gcp requires additional credentials setup gcp = endpoint_from_flag('endpoint.gcp.ready')