Updating ceph to use CSI for k8s >= 1.10

This commit is contained in:
Mike Wilson 2018-07-16 14:02:33 -04:00
parent 218b334d26
commit bc3e25146f
3 changed files with 176 additions and 0 deletions

View File

@ -38,6 +38,14 @@ def main():
this script thinks the environment is 'sane' enough to provision volumes.
'''
# k8s >= 1.10 uses CSI and doesn't directly create persistent volumes
if get_version('kube-apiserver') >= (1, 10):
print('This action is deprecated in favor of CSI creation of persistent volumes')
print('in Kubernetes >= 1.10. Just create the PVC and a PV will be created')
print('for you.')
action_fail('Deprecated, just create PVC.')
return
# validate relationship pre-reqs before additional steps can be taken
if not validate_relation():
print('Failed ceph relationship check')
@ -89,6 +97,23 @@ def main():
check_call(cmd)
def get_version(bin_name):
"""Get the version of an installed Kubernetes binary.
:param str bin_name: Name of binary
:return: 3-tuple version (maj, min, patch)
Example::
>>> `get_version('kubelet')
(1, 6, 0)
"""
cmd = '{} --version'.format(bin_name).split()
version_string = check_output(cmd).decode('utf-8')
return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3])
def action_get_or_default(key):
''' Convenience method to manage defaults since actions dont appear to
properly support defaults '''

View File

@ -99,3 +99,9 @@ options:
default: true
description: |
If true the metrics server for Kubernetes will be deployed onto the cluster.
default-storage:
type: string
default: "auto"
description: |
The storage class to make the default storage class. Allowed values are "auto",
"none", "ceph-xfs", "ceph-ext4". Note: Only works in Kubernetes >= 1.10

View File

@ -27,6 +27,7 @@ import ipaddress
from charms.leadership import leader_get, leader_set
from shutil import move
from tempfile import TemporaryDirectory
from pathlib import Path
from shlex import split
@ -666,6 +667,7 @@ def kick_api_server(tls):
@when('kubernetes-master.components.started')
@when('leadership.is_leader')
def configure_cdk_addons():
''' Configure CDK addons '''
remove_state('cdk-addons.configured')
@ -758,6 +760,11 @@ def ceph_storage(ceph_admin):
configuration, and the ceph secret key file used for authentication.
This method will install the client package, and render the requisit files
in order to consume the ceph-storage relation.'''
# deprecated in 1.10 in favor of using CSI
if get_version('kube-apiserver') >= (1, 10):
return
ceph_context = {
'mon_hosts': ceph_admin.mon_hosts(),
'fsid': ceph_admin.fsid(),
@ -817,6 +824,140 @@ def ceph_storage(ceph_admin):
set_state('ceph-storage.configured')
@when('config.changed.default-storage')
def render_ceph():
remove_state('ceph-storage.configured')
@when('ceph-storage.available')
@when('cdk-addons.configured')
@when_not('ceph-storage.configured')
def ceph_csi_storage(ceph_admin):
'''Ceph on kubernetes will require a few things put on the cluster.
This includes a provisioner and storage class. This method will
put those things on the cluster.'''
# Pre-CSI versions still use the old apt version and manual volume
# creation, so skip the CSI stuff
if get_version('kube-apiserver') < (1, 10):
return
if ceph_admin.key():
encoded_key = base64.b64encode(ceph_admin.key().encode('utf-8'))
else:
# We didn't have a key, and cannot proceed. Do not set state and
# allow this method to re-execute
return
ceph_context = {
'mon_hosts': ceph_admin.mon_hosts(),
'fsid': ceph_admin.fsid(),
'auth_supported': ceph_admin.auth(),
'use_syslog': "true",
'ceph_public_network': '',
'ceph_cluster_network': '',
'loglevel': 1,
'hostname': socket.gethostname(),
'admin_key': encoded_key.decode('ascii'),
'kubernetes_key': encoded_key.decode('ascii'),
}
# Install the ceph common utilities.
apt_install(['ceph-common'], fatal=True)
etc_ceph_directory = '/etc/ceph'
if not os.path.isdir(etc_ceph_directory):
os.makedirs(etc_ceph_directory)
charm_ceph_conf = os.path.join(etc_ceph_directory, 'ceph.conf')
# Render the ceph configuration from the ceph conf template
render('ceph.conf', charm_ceph_conf, ceph_context)
config = hookenv.config()
default_storage = config.get('default-storage')
# Create the ceph yamls from templates in cdk-addons
with TemporaryDirectory() as template_path:
addon_path = str(_cdk_addons_template_path())
render('rbd-secrets.yaml',
'{}/rbd-secrets.yaml'.format(template_path), ceph_context,
templates_dir=addon_path)
render('rbdplugin.yaml',
'{}/rbdplugin.yaml'.format(template_path), ceph_context,
templates_dir=addon_path)
render('csi-provisioner.yaml',
'{}/csi-provisioner.yaml'.format(template_path), ceph_context,
templates_dir=addon_path)
render('csi-attacher.yaml',
'{}/csi-attacher.yaml'.format(template_path), ceph_context,
templates_dir=addon_path)
ceph_context['pool_name'] = 'ext4-pool'
ceph_context['fs_type'] = 'ext4'
ceph_context['sc_name'] = 'ceph-ext4'
if default_storage == 'ceph-ext4':
ceph_context['default'] = True
else:
ceph_context['default'] = False
render('rbd-storage-class.yaml',
'{}/rbd-storage-class-ext4.yaml'.format(template_path),
ceph_context, templates_dir=addon_path)
ceph_context['pool_name'] = 'xfs-pool'
ceph_context['fs_type'] = 'ext4'
ceph_context['sc_name'] = 'ceph-xfs'
if default_storage == 'ceph-xfs' or default_storage == 'auto':
ceph_context['default'] = True
else:
ceph_context['default'] = False
render('rbd-storage-class.yaml',
'{}/rbd-storage-class-xfs.yaml'.format(template_path),
ceph_context, templates_dir=addon_path)
try:
# At first glance this is deceptive. The apply stanza will create
# if it doesn't exist, otherwise it will update the entry
cmd = ['kubectl', 'apply', '-f', template_path]
check_call(cmd)
except: # NOQA
# the enlistment in kubernetes failed, return and prepare for
# re-exec
return
# when complete, set a state relating to configuration of the storage
# backend that will allow other modules to hook into this and verify we
# have performed the necessary pre-req steps to interface with a ceph
# deployment.
set_state('ceph-storage.configured')
@when_not('ceph-storage.available')
@when('ceph-storage.configured')
def remove_ceph():
# Pre-CSI versions still use the old apt version and manual volume
# creation, so skip the CSI stuff
if get_version('kube-apiserver') < (1, 10):
return
ceph_things = [['secret', 'csi-ceph-secret'],
['sc', 'ceph-xfs'],
['sc', 'ceph-ext4'],
['serviceaccount', 'csi-rbdplugin'],
['clusterrole', 'csi-rbdplugin'],
['clusterrolebinding', 'csi-rbdplugin'],
['daemonset', 'csi-rbdplugin'],
['serviceaccount', 'csi-attacher'],
['clusterrole', 'external-attacher-runner'],
['clusterrolebinding', 'csi-attacher-role'],
['service', 'csi-attacher'],
['statefulset', 'csi-attacher'],
['serviceaccount', 'csi-provisioner'],
['clusterrole', 'external-provisioner-runner'],
['clusterrolebinding', 'csi-provisioner-role'],
['service', 'csi-provisioner'],
['statefulset', 'csi-provisioner']]
for entity in ceph_things:
cmd = 'kubectl delete {} {}'.format(entity[0], entity[1])
check_call(split(cmd))
@when('nrpe-external-master.available')
@when_not('nrpe-external-master.initial-config')
def initial_nrpe_config(nagios=None):
@ -1581,6 +1722,10 @@ def _daemon_env_path(component):
return _snap_common_path(component) / 'environment'
def _cdk_addons_template_path():
return Path('/snap/cdk-addons/current/templates')
def _write_gcp_snap_config(component):
# gcp requires additional credentials setup
gcp = endpoint_from_flag('endpoint.gcp.ready')