From 1bb4263de633c277adcca1ddf43b187eb01af79f Mon Sep 17 00:00:00 2001 From: Mike Wilson Date: Wed, 1 Aug 2018 11:05:32 -0400 Subject: [PATCH] Changing ceph CSI to use cdk-addons for template rendering --- .../layers/kubernetes-master/actions.yaml | 2 +- .../reactive/kubernetes_master.py | 160 +++--------------- 2 files changed, 25 insertions(+), 137 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/actions.yaml b/cluster/juju/layers/kubernetes-master/actions.yaml index cfb74c248fc..89cd502b049 100644 --- a/cluster/juju/layers/kubernetes-master/actions.yaml +++ b/cluster/juju/layers/kubernetes-master/actions.yaml @@ -1,7 +1,7 @@ restart: description: Restart the Kubernetes master services on demand. create-rbd-pv: - description: Create RADOS Block Device (RDB) volume in Ceph and creates PersistentVolume. + description: Create RADOS Block Device (RDB) volume in Ceph and creates PersistentVolume. Note this is deprecated on Kubernetes >= 1.10 in favor of CSI, where PersistentVolumes are created dynamically to back PersistentVolumeClaims. params: name: type: string diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 5d0a68dd379..00654934eff 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -666,7 +666,7 @@ def kick_api_server(tls): tls_client.reset_certificate_write_flag('server') -@when('kubernetes-master.components.started') +@when_any('kubernetes-master.components.started', 'ceph-storage.configured') @when('leadership.is_leader') def configure_cdk_addons(): ''' Configure CDK addons ''' @@ -678,6 +678,19 @@ def configure_cdk_addons(): dbEnabled = str(hookenv.config('enable-dashboard-addons')).lower() dnsEnabled = str(hookenv.config('enable-kube-dns')).lower() metricsEnabled = str(hookenv.config('enable-metrics')).lower() + if (is_state('ceph-storage.configured') and + get_version('kube-apiserver') >= (1, 10)): + cephEnabled = "true" + else: + cephEnabled = "false" + ceph = endpoint_from_flag('ceph-storage.available') + if ceph: + b64_ceph_key = base64.b64encode(ceph.key().encode('utf-8')) + ceph_admin_key = b64_ceph_key.decode('ascii') + ceph_kubernetes_key = b64_ceph_key.decode('ascii') + ceph_mon_hosts = ceph.mon_hosts() + default_storage = hookenv.config('default-storage') + args = [ 'arch=' + arch(), 'dns-ip=' + get_deprecated_dns_ip(), @@ -685,7 +698,12 @@ def configure_cdk_addons(): 'enable-dashboard=' + dbEnabled, 'enable-kube-dns=' + dnsEnabled, 'enable-metrics=' + metricsEnabled, - 'enable-gpu=' + str(gpuEnable).lower() + 'enable-gpu=' + str(gpuEnable).lower(), + 'enable-ceph=' + cephEnabled, + 'ceph-admin-key=' + (ceph_admin_key or ''), + 'ceph-kubernetes-key=' + (ceph_admin_key or ''), + 'ceph-mon-hosts="' + (ceph_mon_hosts or '') + '"', + 'default-storage=' + (default_storage or ''), ] check_call(['snap', 'set', 'cdk-addons'] + args) if not addons_ready(): @@ -763,6 +781,10 @@ def ceph_storage(ceph_admin): # deprecated in 1.10 in favor of using CSI if get_version('kube-apiserver') >= (1, 10): + # this is actually false, but by setting this flag we won't keep + # running this function for no reason. Also note that we watch this + # flag to run cdk-addons.apply. + set_state('ceph-storage.configured') return ceph_context = { @@ -824,140 +846,6 @@ def ceph_storage(ceph_admin): set_state('ceph-storage.configured') -@when('config.changed.default-storage') -def render_ceph(): - remove_state('ceph-storage.configured') - - -@when('ceph-storage.available') -@when('cdk-addons.configured') -@when_not('ceph-storage.configured') -def ceph_csi_storage(ceph_admin): - '''Ceph on kubernetes will require a few things put on the cluster. - This includes a provisioner and storage class. This method will - put those things on the cluster.''' - - # Pre-CSI versions still use the old apt version and manual volume - # creation, so skip the CSI stuff - if get_version('kube-apiserver') < (1, 10): - return - - if ceph_admin.key(): - encoded_key = base64.b64encode(ceph_admin.key().encode('utf-8')) - else: - # We didn't have a key, and cannot proceed. Do not set state and - # allow this method to re-execute - return - - ceph_context = { - 'mon_hosts': ceph_admin.mon_hosts(), - 'fsid': ceph_admin.fsid(), - 'auth_supported': ceph_admin.auth(), - 'use_syslog': "true", - 'ceph_public_network': '', - 'ceph_cluster_network': '', - 'loglevel': 1, - 'hostname': socket.gethostname(), - 'admin_key': encoded_key.decode('ascii'), - 'kubernetes_key': encoded_key.decode('ascii'), - } - - # Install the ceph common utilities. - apt_install(['ceph-common'], fatal=True) - - etc_ceph_directory = '/etc/ceph' - if not os.path.isdir(etc_ceph_directory): - os.makedirs(etc_ceph_directory) - charm_ceph_conf = os.path.join(etc_ceph_directory, 'ceph.conf') - # Render the ceph configuration from the ceph conf template - render('ceph.conf', charm_ceph_conf, ceph_context) - - config = hookenv.config() - default_storage = config.get('default-storage') - - # Create the ceph yamls from templates in cdk-addons - with TemporaryDirectory() as template_path: - addon_path = str(_cdk_addons_template_path()) - render('rbd-secrets.yaml', - '{}/rbd-secrets.yaml'.format(template_path), ceph_context, - templates_dir=addon_path) - render('rbdplugin.yaml', - '{}/rbdplugin.yaml'.format(template_path), ceph_context, - templates_dir=addon_path) - render('csi-provisioner.yaml', - '{}/csi-provisioner.yaml'.format(template_path), ceph_context, - templates_dir=addon_path) - render('csi-attacher.yaml', - '{}/csi-attacher.yaml'.format(template_path), ceph_context, - templates_dir=addon_path) - ceph_context['pool_name'] = 'ext4-pool' - ceph_context['fs_type'] = 'ext4' - ceph_context['sc_name'] = 'ceph-ext4' - if default_storage == 'ceph-ext4': - ceph_context['default'] = True - else: - ceph_context['default'] = False - render('rbd-storage-class.yaml', - '{}/rbd-storage-class-ext4.yaml'.format(template_path), - ceph_context, templates_dir=addon_path) - ceph_context['pool_name'] = 'xfs-pool' - ceph_context['fs_type'] = 'ext4' - ceph_context['sc_name'] = 'ceph-xfs' - if default_storage == 'ceph-xfs' or default_storage == 'auto': - ceph_context['default'] = True - else: - ceph_context['default'] = False - render('rbd-storage-class.yaml', - '{}/rbd-storage-class-xfs.yaml'.format(template_path), - ceph_context, templates_dir=addon_path) - - try: - # At first glance this is deceptive. The apply stanza will create - # if it doesn't exist, otherwise it will update the entry - cmd = ['kubectl', 'apply', '-f', template_path] - check_call(cmd) - except: # NOQA - # the enlistment in kubernetes failed, return and prepare for - # re-exec - return - - # when complete, set a state relating to configuration of the storage - # backend that will allow other modules to hook into this and verify we - # have performed the necessary pre-req steps to interface with a ceph - # deployment. - set_state('ceph-storage.configured') - - -@when_not('ceph-storage.available') -@when('ceph-storage.configured') -def remove_ceph(): - # Pre-CSI versions still use the old apt version and manual volume - # creation, so skip the CSI stuff - if get_version('kube-apiserver') < (1, 10): - return - - ceph_things = [['secret', 'csi-ceph-secret'], - ['sc', 'ceph-xfs'], - ['sc', 'ceph-ext4'], - ['serviceaccount', 'csi-rbdplugin'], - ['clusterrole', 'csi-rbdplugin'], - ['clusterrolebinding', 'csi-rbdplugin'], - ['daemonset', 'csi-rbdplugin'], - ['serviceaccount', 'csi-attacher'], - ['clusterrole', 'external-attacher-runner'], - ['clusterrolebinding', 'csi-attacher-role'], - ['service', 'csi-attacher'], - ['statefulset', 'csi-attacher'], - ['serviceaccount', 'csi-provisioner'], - ['clusterrole', 'external-provisioner-runner'], - ['clusterrolebinding', 'csi-provisioner-role'], - ['service', 'csi-provisioner'], - ['statefulset', 'csi-provisioner']] - for entity in ceph_things: - cmd = 'kubectl delete {} {}'.format(entity[0], entity[1]) - check_call(split(cmd)) - - @when('nrpe-external-master.available') @when_not('nrpe-external-master.initial-config') def initial_nrpe_config(nagios=None):