Merge pull request #66523 from hyperbolic2346/mwilson/ceph

Automatic merge from submit-queue (batch tested with PRs 67661, 67497, 66523, 67622, 67632). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Updating ceph to use CSI for k8s >= 1.10

**What this PR does / why we need it**:
Updates the ceph charms to use CSI if the k8s version is >= 1.10
**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
Kubernetes juju charms will now use CSI for ceph.
```

cluster/juju/layers/kubernetes-master/actions.yaml

@@ -1,7 +1,7 @@
 restart:
   description: Restart the Kubernetes master services on demand.
 create-rbd-pv:
-  description: Create RADOS Block Device (RDB) volume in Ceph and creates PersistentVolume.
+  description: Create RADOS Block Device (RDB) volume in Ceph and creates PersistentVolume. Note this is deprecated on Kubernetes >= 1.10 in favor of CSI, where PersistentVolumes are created dynamically to back PersistentVolumeClaims.
   params:
     name:
       type: string

cluster/juju/layers/kubernetes-master/actions/create-rbd-pv

@@ -38,6 +38,14 @@ def main():
     this script thinks the environment is 'sane' enough to provision volumes.
     '''
 
+    # k8s >= 1.10 uses CSI and doesn't directly create persistent volumes
+    if get_version('kube-apiserver') >= (1, 10):
+       print('This action is deprecated in favor of CSI creation of persistent volumes')
+       print('in Kubernetes >= 1.10. Just create the PVC and a PV will be created')
+       print('for you.')
+       action_fail('Deprecated, just create PVC.')
+       return
+
     # validate relationship pre-reqs before additional steps can be taken
     if not validate_relation():
         print('Failed ceph relationship check')
@@ -89,6 +97,23 @@ def main():
         check_call(cmd)
 
 
+def get_version(bin_name):
+    """Get the version of an installed Kubernetes binary.
+
+    :param str bin_name: Name of binary
+    :return: 3-tuple version (maj, min, patch)
+
+    Example::
+
+        >>> `get_version('kubelet')
+        (1, 6, 0)
+
+    """
+    cmd = '{} --version'.format(bin_name).split()
+    version_string = check_output(cmd).decode('utf-8')
+    return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3])
+
+
 def action_get_or_default(key):
     ''' Convenience method to manage defaults since actions dont appear to
     properly support defaults '''

cluster/juju/layers/kubernetes-master/config.yaml

@@ -147,3 +147,9 @@ options:
     default: true
     description: |
       If true the metrics server for Kubernetes will be deployed onto the cluster.
+  default-storage:
+    type: string
+    default: "auto"
+    description: |
+      The storage class to make the default storage class. Allowed values are "auto",
+      "none", "ceph-xfs", "ceph-ext4". Note: Only works in Kubernetes >= 1.10

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -28,6 +28,7 @@ import ipaddress
 from charms.leadership import leader_get, leader_set
 
 from shutil import move
+from tempfile import TemporaryDirectory
 
 from pathlib import Path
 from shlex import split
@@ -711,7 +712,8 @@ def kick_api_server(tls):
     tls_client.reset_certificate_write_flag('server')
 
 
-@when('kubernetes-master.components.started')
+@when_any('kubernetes-master.components.started', 'ceph-storage.configured')
+@when('leadership.is_leader')
 def configure_cdk_addons():
     ''' Configure CDK addons '''
     remove_state('cdk-addons.configured')
@@ -723,6 +725,21 @@ def configure_cdk_addons():
     dbEnabled = str(hookenv.config('enable-dashboard-addons')).lower()
     dnsEnabled = str(hookenv.config('enable-kube-dns')).lower()
     metricsEnabled = str(hookenv.config('enable-metrics')).lower()
+    if (is_state('ceph-storage.configured') and
+            get_version('kube-apiserver') >= (1, 10)):
+        cephEnabled = "true"
+    else:
+        cephEnabled = "false"
+    ceph_ep = endpoint_from_flag('ceph-storage.available')
+    ceph = {}
+    default_storage = ''
+    if ceph_ep:
+        b64_ceph_key = base64.b64encode(ceph_ep.key().encode('utf-8'))
+        ceph['admin_key'] = b64_ceph_key.decode('ascii')
+        ceph['kubernetes_key'] = b64_ceph_key.decode('ascii')
+        ceph['mon_hosts'] = ceph_ep.mon_hosts()
+        default_storage = hookenv.config('default-storage')
+
     args = [
         'arch=' + arch(),
         'dns-ip=' + get_deprecated_dns_ip(),
@@ -731,7 +748,12 @@ def configure_cdk_addons():
         'enable-dashboard=' + dbEnabled,
         'enable-kube-dns=' + dnsEnabled,
         'enable-metrics=' + metricsEnabled,
-        'enable-gpu=' + str(gpuEnable).lower()
+        'enable-gpu=' + str(gpuEnable).lower(),
+        'enable-ceph=' + cephEnabled,
+        'ceph-admin-key=' + (ceph.get('admin_key', '')),
+        'ceph-kubernetes-key=' + (ceph.get('admin_key', '')),
+        'ceph-mon-hosts="' + (ceph.get('mon_hosts', '')) + '"',
+        'default-storage=' + default_storage,
     ]
     check_call(['snap', 'set', 'cdk-addons'] + args)
     if not addons_ready():
@@ -806,6 +828,15 @@ def ceph_storage(ceph_admin):
     configuration, and the ceph secret key file used for authentication.
     This method will install the client package, and render the requisit files
     in order to consume the ceph-storage relation.'''
+
+    # deprecated in 1.10 in favor of using CSI
+    if get_version('kube-apiserver') >= (1, 10):
+        # this is actually false, but by setting this flag we won't keep
+        # running this function for no reason. Also note that we watch this
+        # flag to run cdk-addons.apply.
+        set_state('ceph-storage.configured')
+        return
+
     ceph_context = {
         'mon_hosts': ceph_admin.mon_hosts(),
         'fsid': ceph_admin.fsid(),
@@ -1662,6 +1693,10 @@ def _daemon_env_path(component):
     return _snap_common_path(component) / 'environment'
 
 
+def _cdk_addons_template_path():
+    return Path('/snap/cdk-addons/current/templates')
+
+
 def _write_gcp_snap_config(component):
     # gcp requires additional credentials setup
     gcp = endpoint_from_flag('endpoint.gcp.ready')