Updating ceph to use CSI for k8s >= 1.10

cluster/juju/layers/kubernetes-master/actions/create-rbd-pv

@@ -38,6 +38,14 @@ def main():
     this script thinks the environment is 'sane' enough to provision volumes.
     '''
 
+    # k8s >= 1.10 uses CSI and doesn't directly create persistent volumes
+    if get_version('kube-apiserver') >= (1, 10):
+       print('This action is deprecated in favor of CSI creation of persistent volumes')
+       print('in Kubernetes >= 1.10. Just create the PVC and a PV will be created')
+       print('for you.')
+       action_fail('Deprecated, just create PVC.')
+       return
+
     # validate relationship pre-reqs before additional steps can be taken
     if not validate_relation():
         print('Failed ceph relationship check')
@@ -89,6 +97,23 @@ def main():
         check_call(cmd)
 
 
+def get_version(bin_name):
+    """Get the version of an installed Kubernetes binary.
+
+    :param str bin_name: Name of binary
+    :return: 3-tuple version (maj, min, patch)
+
+    Example::
+
+        >>> `get_version('kubelet')
+        (1, 6, 0)
+
+    """
+    cmd = '{} --version'.format(bin_name).split()
+    version_string = check_output(cmd).decode('utf-8')
+    return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3])
+
+
 def action_get_or_default(key):
     ''' Convenience method to manage defaults since actions dont appear to
     properly support defaults '''

cluster/juju/layers/kubernetes-master/config.yaml

@@ -99,3 +99,9 @@ options:
     default: true
     description: |
       If true the metrics server for Kubernetes will be deployed onto the cluster.
+  default-storage:
+    type: string
+    default: "auto"
+    description: |
+      The storage class to make the default storage class. Allowed values are "auto",
+      "none", "ceph-xfs", "ceph-ext4". Note: Only works in Kubernetes >= 1.10

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -27,6 +27,7 @@ import ipaddress
 from charms.leadership import leader_get, leader_set
 
 from shutil import move
+from tempfile import TemporaryDirectory
 
 from pathlib import Path
 from shlex import split
@@ -666,6 +667,7 @@ def kick_api_server(tls):
 
 
 @when('kubernetes-master.components.started')
+@when('leadership.is_leader')
 def configure_cdk_addons():
     ''' Configure CDK addons '''
     remove_state('cdk-addons.configured')
@@ -758,6 +760,11 @@ def ceph_storage(ceph_admin):
     configuration, and the ceph secret key file used for authentication.
     This method will install the client package, and render the requisit files
     in order to consume the ceph-storage relation.'''
+
+    # deprecated in 1.10 in favor of using CSI
+    if get_version('kube-apiserver') >= (1, 10):
+        return
+
     ceph_context = {
         'mon_hosts': ceph_admin.mon_hosts(),
         'fsid': ceph_admin.fsid(),
@@ -817,6 +824,140 @@ def ceph_storage(ceph_admin):
     set_state('ceph-storage.configured')
 
 
+@when('config.changed.default-storage')
+def render_ceph():
+    remove_state('ceph-storage.configured')
+
+
+@when('ceph-storage.available')
+@when('cdk-addons.configured')
+@when_not('ceph-storage.configured')
+def ceph_csi_storage(ceph_admin):
+    '''Ceph on kubernetes will require a few things put on the cluster.
+    This includes a provisioner and storage class. This method will
+    put those things on the cluster.'''
+
+    # Pre-CSI versions still use the old apt version and manual volume
+    # creation, so skip the CSI stuff
+    if get_version('kube-apiserver') < (1, 10):
+        return
+
+    if ceph_admin.key():
+        encoded_key = base64.b64encode(ceph_admin.key().encode('utf-8'))
+    else:
+        # We didn't have a key, and cannot proceed. Do not set state and
+        # allow this method to re-execute
+        return
+
+    ceph_context = {
+        'mon_hosts': ceph_admin.mon_hosts(),
+        'fsid': ceph_admin.fsid(),
+        'auth_supported': ceph_admin.auth(),
+        'use_syslog': "true",
+        'ceph_public_network': '',
+        'ceph_cluster_network': '',
+        'loglevel': 1,
+        'hostname': socket.gethostname(),
+        'admin_key': encoded_key.decode('ascii'),
+        'kubernetes_key': encoded_key.decode('ascii'),
+    }
+
+    # Install the ceph common utilities.
+    apt_install(['ceph-common'], fatal=True)
+
+    etc_ceph_directory = '/etc/ceph'
+    if not os.path.isdir(etc_ceph_directory):
+        os.makedirs(etc_ceph_directory)
+    charm_ceph_conf = os.path.join(etc_ceph_directory, 'ceph.conf')
+    # Render the ceph configuration from the ceph conf template
+    render('ceph.conf', charm_ceph_conf, ceph_context)
+
+    config = hookenv.config()
+    default_storage = config.get('default-storage')
+
+    # Create the ceph yamls from templates in cdk-addons
+    with TemporaryDirectory() as template_path:
+        addon_path = str(_cdk_addons_template_path())
+        render('rbd-secrets.yaml',
+               '{}/rbd-secrets.yaml'.format(template_path), ceph_context,
+               templates_dir=addon_path)
+        render('rbdplugin.yaml',
+               '{}/rbdplugin.yaml'.format(template_path), ceph_context,
+               templates_dir=addon_path)
+        render('csi-provisioner.yaml',
+               '{}/csi-provisioner.yaml'.format(template_path), ceph_context,
+               templates_dir=addon_path)
+        render('csi-attacher.yaml',
+               '{}/csi-attacher.yaml'.format(template_path), ceph_context,
+               templates_dir=addon_path)
+        ceph_context['pool_name'] = 'ext4-pool'
+        ceph_context['fs_type'] = 'ext4'
+        ceph_context['sc_name'] = 'ceph-ext4'
+        if default_storage == 'ceph-ext4':
+            ceph_context['default'] = True
+        else:
+            ceph_context['default'] = False
+        render('rbd-storage-class.yaml',
+               '{}/rbd-storage-class-ext4.yaml'.format(template_path),
+               ceph_context, templates_dir=addon_path)
+        ceph_context['pool_name'] = 'xfs-pool'
+        ceph_context['fs_type'] = 'ext4'
+        ceph_context['sc_name'] = 'ceph-xfs'
+        if default_storage == 'ceph-xfs' or default_storage == 'auto':
+            ceph_context['default'] = True
+        else:
+            ceph_context['default'] = False
+        render('rbd-storage-class.yaml',
+               '{}/rbd-storage-class-xfs.yaml'.format(template_path),
+               ceph_context, templates_dir=addon_path)
+
+        try:
+            # At first glance this is deceptive. The apply stanza will create
+            # if it doesn't exist, otherwise it will update the entry
+            cmd = ['kubectl', 'apply', '-f', template_path]
+            check_call(cmd)
+        except:  # NOQA
+            # the enlistment in kubernetes failed, return and prepare for
+            # re-exec
+            return
+
+    # when complete, set a state relating to configuration of the storage
+    # backend that will allow other modules to hook into this and verify we
+    # have performed the necessary pre-req steps to interface with a ceph
+    # deployment.
+    set_state('ceph-storage.configured')
+
+
+@when_not('ceph-storage.available')
+@when('ceph-storage.configured')
+def remove_ceph():
+    # Pre-CSI versions still use the old apt version and manual volume
+    # creation, so skip the CSI stuff
+    if get_version('kube-apiserver') < (1, 10):
+        return
+
+    ceph_things = [['secret', 'csi-ceph-secret'],
+                   ['sc', 'ceph-xfs'],
+                   ['sc', 'ceph-ext4'],
+                   ['serviceaccount', 'csi-rbdplugin'],
+                   ['clusterrole', 'csi-rbdplugin'],
+                   ['clusterrolebinding', 'csi-rbdplugin'],
+                   ['daemonset', 'csi-rbdplugin'],
+                   ['serviceaccount', 'csi-attacher'],
+                   ['clusterrole', 'external-attacher-runner'],
+                   ['clusterrolebinding', 'csi-attacher-role'],
+                   ['service', 'csi-attacher'],
+                   ['statefulset', 'csi-attacher'],
+                   ['serviceaccount', 'csi-provisioner'],
+                   ['clusterrole', 'external-provisioner-runner'],
+                   ['clusterrolebinding', 'csi-provisioner-role'],
+                   ['service', 'csi-provisioner'],
+                   ['statefulset', 'csi-provisioner']]
+    for entity in ceph_things:
+        cmd = 'kubectl delete {} {}'.format(entity[0], entity[1])
+        check_call(split(cmd))
+
+
 @when('nrpe-external-master.available')
 @when_not('nrpe-external-master.initial-config')
 def initial_nrpe_config(nagios=None):
@@ -1581,6 +1722,10 @@ def _daemon_env_path(component):
     return _snap_common_path(component) / 'environment'
 
 
+def _cdk_addons_template_path():
+    return Path('/snap/cdk-addons/current/templates')
+
+
 def _write_gcp_snap_config(component):
     # gcp requires additional credentials setup
     gcp = endpoint_from_flag('endpoint.gcp.ready')