From 1b6268b616ba20ad637e5e3de1173e8fb5fa42ec Mon Sep 17 00:00:00 2001 From: Mike Wilson Date: Mon, 16 Jul 2018 14:02:33 -0400 Subject: [PATCH 01/13] Updating ceph to use CSI for k8s >= 1.10 --- .../reactive/kubernetes_master.py | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 8842930d588..fadb163a89d 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -896,6 +896,140 @@ def ceph_storage(ceph_admin): set_state('ceph-storage.configured') +@when('config.changed.default-storage') +def render_ceph(): + remove_state('ceph-storage.configured') + + +@when('ceph-storage.available') +@when('cdk-addons.configured') +@when_not('ceph-storage.configured') +def ceph_csi_storage(ceph_admin): + '''Ceph on kubernetes will require a few things put on the cluster. + This includes a provisioner and storage class. This method will + put those things on the cluster.''' + + # Pre-CSI versions still use the old apt version and manual volume + # creation, so skip the CSI stuff + if get_version('kube-apiserver') < (1, 10): + return + + if ceph_admin.key(): + encoded_key = base64.b64encode(ceph_admin.key().encode('utf-8')) + else: + # We didn't have a key, and cannot proceed. Do not set state and + # allow this method to re-execute + return + + ceph_context = { + 'mon_hosts': ceph_admin.mon_hosts(), + 'fsid': ceph_admin.fsid(), + 'auth_supported': ceph_admin.auth(), + 'use_syslog': "true", + 'ceph_public_network': '', + 'ceph_cluster_network': '', + 'loglevel': 1, + 'hostname': socket.gethostname(), + 'admin_key': encoded_key.decode('ascii'), + 'kubernetes_key': encoded_key.decode('ascii'), + } + + # Install the ceph common utilities. + apt_install(['ceph-common'], fatal=True) + + etc_ceph_directory = '/etc/ceph' + if not os.path.isdir(etc_ceph_directory): + os.makedirs(etc_ceph_directory) + charm_ceph_conf = os.path.join(etc_ceph_directory, 'ceph.conf') + # Render the ceph configuration from the ceph conf template + render('ceph.conf', charm_ceph_conf, ceph_context) + + config = hookenv.config() + default_storage = config.get('default-storage') + + # Create the ceph yamls from templates in cdk-addons + with TemporaryDirectory() as template_path: + addon_path = str(_cdk_addons_template_path()) + render('rbd-secrets.yaml', + '{}/rbd-secrets.yaml'.format(template_path), ceph_context, + templates_dir=addon_path) + render('rbdplugin.yaml', + '{}/rbdplugin.yaml'.format(template_path), ceph_context, + templates_dir=addon_path) + render('csi-provisioner.yaml', + '{}/csi-provisioner.yaml'.format(template_path), ceph_context, + templates_dir=addon_path) + render('csi-attacher.yaml', + '{}/csi-attacher.yaml'.format(template_path), ceph_context, + templates_dir=addon_path) + ceph_context['pool_name'] = 'ext4-pool' + ceph_context['fs_type'] = 'ext4' + ceph_context['sc_name'] = 'ceph-ext4' + if default_storage == 'ceph-ext4': + ceph_context['default'] = True + else: + ceph_context['default'] = False + render('rbd-storage-class.yaml', + '{}/rbd-storage-class-ext4.yaml'.format(template_path), + ceph_context, templates_dir=addon_path) + ceph_context['pool_name'] = 'xfs-pool' + ceph_context['fs_type'] = 'ext4' + ceph_context['sc_name'] = 'ceph-xfs' + if default_storage == 'ceph-xfs' or default_storage == 'auto': + ceph_context['default'] = True + else: + ceph_context['default'] = False + render('rbd-storage-class.yaml', + '{}/rbd-storage-class-xfs.yaml'.format(template_path), + ceph_context, templates_dir=addon_path) + + try: + # At first glance this is deceptive. The apply stanza will create + # if it doesn't exist, otherwise it will update the entry + cmd = ['kubectl', 'apply', '-f', template_path] + check_call(cmd) + except: # NOQA + # the enlistment in kubernetes failed, return and prepare for + # re-exec + return + + # when complete, set a state relating to configuration of the storage + # backend that will allow other modules to hook into this and verify we + # have performed the necessary pre-req steps to interface with a ceph + # deployment. + set_state('ceph-storage.configured') + + +@when_not('ceph-storage.available') +@when('ceph-storage.configured') +def remove_ceph(): + # Pre-CSI versions still use the old apt version and manual volume + # creation, so skip the CSI stuff + if get_version('kube-apiserver') < (1, 10): + return + + ceph_things = [['secret', 'csi-ceph-secret'], + ['sc', 'ceph-xfs'], + ['sc', 'ceph-ext4'], + ['serviceaccount', 'csi-rbdplugin'], + ['clusterrole', 'csi-rbdplugin'], + ['clusterrolebinding', 'csi-rbdplugin'], + ['daemonset', 'csi-rbdplugin'], + ['serviceaccount', 'csi-attacher'], + ['clusterrole', 'external-attacher-runner'], + ['clusterrolebinding', 'csi-attacher-role'], + ['service', 'csi-attacher'], + ['statefulset', 'csi-attacher'], + ['serviceaccount', 'csi-provisioner'], + ['clusterrole', 'external-provisioner-runner'], + ['clusterrolebinding', 'csi-provisioner-role'], + ['service', 'csi-provisioner'], + ['statefulset', 'csi-provisioner']] + for entity in ceph_things: + cmd = 'kubectl delete {} {}'.format(entity[0], entity[1]) + check_call(split(cmd)) + + @when('nrpe-external-master.available') @when_not('nrpe-external-master.initial-config') def initial_nrpe_config(nagios=None): From 800d016dd4448c30e442f772ae246c85bb03463d Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Thu, 26 Jul 2018 06:54:15 -0500 Subject: [PATCH 02/13] new snapd_refresh config to control snapd refresh frequency (#141) * add snapd_refresh config and handlers to k8s-master and -worker * lint readmes * add snapd_refresh doc to the readme; make "max" less specific * adjust wording to note snapd_refresh only affects store snaps --- .../juju/layers/kubernetes-master/README.md | 62 ++++++++++++++----- .../juju/layers/kubernetes-master/config.yaml | 9 +++ .../reactive/kubernetes_master.py | 32 ++++++++++ .../juju/layers/kubernetes-worker/README.md | 35 ++++++++++- .../juju/layers/kubernetes-worker/config.yaml | 9 +++ .../reactive/kubernetes_worker.py | 34 ++++++++++ 6 files changed, 164 insertions(+), 17 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/README.md b/cluster/juju/layers/kubernetes-master/README.md index c1cc84a6cf0..6279a54cfc9 100644 --- a/cluster/juju/layers/kubernetes-master/README.md +++ b/cluster/juju/layers/kubernetes-master/README.md @@ -1,19 +1,19 @@ # Kubernetes-master -[Kubernetes](http://kubernetes.io/) is an open source system for managing +[Kubernetes](http://kubernetes.io/) is an open source system for managing application containers across a cluster of hosts. The Kubernetes project was -started by Google in 2014, combining the experience of running production +started by Google in 2014, combining the experience of running production workloads combined with best practices from the community. The Kubernetes project defines some new terms that may be unfamiliar to users -or operators. For more information please refer to the concept guide in the +or operators. For more information please refer to the concept guide in the [getting started guide](https://kubernetes.io/docs/home/). -This charm is an encapsulation of the Kubernetes master processes and the +This charm is an encapsulation of the Kubernetes master processes and the operations to run on any cloud for the entire lifecycle of the cluster. This charm is built from other charm layers using the Juju reactive framework. -The other layers focus on specific subset of operations making this layer +The other layers focus on specific subset of operations making this layer specific to operations of Kubernetes master processes. # Deployment @@ -23,15 +23,15 @@ charms to model a complete Kubernetes cluster. A Kubernetes cluster needs a distributed key value store such as [Etcd](https://coreos.com/etcd/) and the kubernetes-worker charm which delivers the Kubernetes node services. A cluster requires a Software Defined Network (SDN) and Transport Layer Security (TLS) so -the components in a cluster communicate securely. +the components in a cluster communicate securely. -Please take a look at the [Canonical Distribution of Kubernetes](https://jujucharms.com/canonical-kubernetes/) -or the [Kubernetes core](https://jujucharms.com/kubernetes-core/) bundles for +Please take a look at the [Canonical Distribution of Kubernetes](https://jujucharms.com/canonical-kubernetes/) +or the [Kubernetes core](https://jujucharms.com/kubernetes-core/) bundles for examples of complete models of Kubernetes clusters. # Resources -The kubernetes-master charm takes advantage of the [Juju Resources](https://jujucharms.com/docs/2.0/developer-resources) +The kubernetes-master charm takes advantage of the [Juju Resources](https://jujucharms.com/docs/2.0/developer-resources) feature to deliver the Kubernetes software. In deployments on public clouds the Charm Store provides the resource to the @@ -40,9 +40,41 @@ firewall rules may not be able to contact the Charm Store. In these network restricted environments the resource can be uploaded to the model by the Juju operator. +#### Snap Refresh + +The kubernetes resources used by this charm are snap packages. When not +specified during deployment, these resources come from the public store. By +default, the `snapd` daemon will refresh all snaps installed from the store +four (4) times per day. A charm configuration option is provided for operators +to control this refresh frequency. + +>NOTE: this is a global configuration option and will affect the refresh +time for all snaps installed on a system. + +Examples: + +```sh +## refresh kubernetes-master snaps every tuesday +juju config kubernetes-master snapd_refresh="tue" + +## refresh snaps at 11pm on the last (5th) friday of the month +juju config kubernetes-master snapd_refresh="fri5,23:00" + +## delay the refresh as long as possible +juju config kubernetes-master snapd_refresh="max" + +## use the system default refresh timer +juju config kubernetes-master snapd_refresh="" +``` + +For more information on the possible values for `snapd_refresh`, see the +*refresh.timer* section in the [system options][] documentation. + +[system options]: https://forum.snapcraft.io/t/system-options/87 + # Configuration -This charm supports some configuration options to set up a Kubernetes cluster +This charm supports some configuration options to set up a Kubernetes cluster that works in your environment: #### dns_domain @@ -61,14 +93,14 @@ Enable RBAC and Node authorisation. # DNS for the cluster The DNS add-on allows the pods to have a DNS names in addition to IP addresses. -The Kubernetes cluster DNS server (based off the SkyDNS library) supports -forward lookups (A records), service lookups (SRV records) and reverse IP +The Kubernetes cluster DNS server (based off the SkyDNS library) supports +forward lookups (A records), service lookups (SRV records) and reverse IP address lookups (PTR records). More information about the DNS can be obtained from the [Kubernetes DNS admin guide](http://kubernetes.io/docs/admin/dns/). # Actions -The kubernetes-master charm models a few one time operations called +The kubernetes-master charm models a few one time operations called [Juju actions](https://jujucharms.com/docs/stable/actions) that can be run by Juju users. @@ -80,7 +112,7 @@ requires a relation to the ceph-mon charm before it can create the volume. #### restart -This action restarts the master processes `kube-apiserver`, +This action restarts the master processes `kube-apiserver`, `kube-controller-manager`, and `kube-scheduler` when the user needs a restart. # More information @@ -93,7 +125,7 @@ This action restarts the master processes `kube-apiserver`, # Contact The kubernetes-master charm is free and open source operations created -by the containers team at Canonical. +by the containers team at Canonical. Canonical also offers enterprise support and customization services. Please refer to the [Kubernetes product page](https://www.ubuntu.com/cloud/kubernetes) diff --git a/cluster/juju/layers/kubernetes-master/config.yaml b/cluster/juju/layers/kubernetes-master/config.yaml index 01171c058fd..e6acd542526 100644 --- a/cluster/juju/layers/kubernetes-master/config.yaml +++ b/cluster/juju/layers/kubernetes-master/config.yaml @@ -147,6 +147,15 @@ options: default: true description: | If true the metrics server for Kubernetes will be deployed onto the cluster. + snapd_refresh: + default: "max" + type: string + description: | + How often snapd handles updates for installed snaps. Setting an empty + string will check 4x per day. Set to "max" to delay the refresh as long + as possible. You may also set a custom string as described in the + 'refresh.timer' section here: + https://forum.snapcraft.io/t/system-options/87 default-storage: type: string default: "auto" diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index fadb163a89d..2f9a34fa7a4 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -427,6 +427,38 @@ def set_app_version(): hookenv.application_version_set(version.split(b' v')[-1].rstrip()) +@when('kubernetes-master.snaps.installed') +@when('snap.refresh.set') +@when('leadership.is_leader') +def process_snapd_timer(): + ''' Set the snapd refresh timer on the leader so all cluster members + (present and future) will refresh near the same time. ''' + # Get the current snapd refresh timer; we know layer-snap has set this + # when the 'snap.refresh.set' flag is present. + timer = snap.get(snapname='core', key='refresh.timer').decode('utf-8') + + # The first time through, data_changed will be true. Subsequent calls + # should only update leader data if something changed. + if data_changed('master_snapd_refresh', timer): + hookenv.log('setting snapd_refresh timer to: {}'.format(timer)) + leader_set({'snapd_refresh': timer}) + + +@when('kubernetes-master.snaps.installed') +@when('snap.refresh.set') +@when('leadership.changed.snapd_refresh') +@when_not('leadership.is_leader') +def set_snapd_timer(): + ''' Set the snapd refresh.timer on non-leader cluster members. ''' + # NB: This method should only be run when 'snap.refresh.set' is present. + # Layer-snap will always set a core refresh.timer, which may not be the + # same as our leader. Gating with 'snap.refresh.set' ensures layer-snap + # has finished and we are free to set our config to the leader's timer. + timer = leader_get('snapd_refresh') + hookenv.log('setting snapd_refresh timer to: {}'.format(timer)) + snap.set_refresh_timer(timer) + + @hookenv.atexit def set_final_status(): ''' Set the final status of the charm as we leave hook execution ''' diff --git a/cluster/juju/layers/kubernetes-worker/README.md b/cluster/juju/layers/kubernetes-worker/README.md index 965199eaf75..a661696f653 100644 --- a/cluster/juju/layers/kubernetes-worker/README.md +++ b/cluster/juju/layers/kubernetes-worker/README.md @@ -27,6 +27,38 @@ To add additional compute capacity to your Kubernetes workers, you may join any related kubernetes-master, and enlist themselves as ready once the deployment is complete. +## Snap Configuration + +The kubernetes resources used by this charm are snap packages. When not +specified during deployment, these resources come from the public store. By +default, the `snapd` daemon will refresh all snaps installed from the store +four (4) times per day. A charm configuration option is provided for operators +to control this refresh frequency. + +>NOTE: this is a global configuration option and will affect the refresh +time for all snaps installed on a system. + +Examples: + +```sh +## refresh kubernetes-worker snaps every tuesday +juju config kubernetes-worker snapd_refresh="tue" + +## refresh snaps at 11pm on the last (5th) friday of the month +juju config kubernetes-worker snapd_refresh="fri5,23:00" + +## delay the refresh as long as possible +juju config kubernetes-worker snapd_refresh="max" + +## use the system default refresh timer +juju config kubernetes-worker snapd_refresh="" +``` + +For more information on the possible values for `snapd_refresh`, see the +*refresh.timer* section in the [system options][] documentation. + +[system options]: https://forum.snapcraft.io/t/system-options/87 + ## Operational actions The kubernetes-worker charm supports the following Operational Actions: @@ -89,7 +121,7 @@ service is not reachable. Note: When debugging connection issues with NodePort services, its important to first check the kube-proxy service on the worker units. If kube-proxy is not running, the associated port-mapping will not be configured in the iptables -rulechains. +rulechains. If you need to close the NodePort once a workload has been terminated, you can follow the same steps inversely. @@ -97,4 +129,3 @@ follow the same steps inversely. ``` juju run --application kubernetes-worker close-port 30510 ``` - diff --git a/cluster/juju/layers/kubernetes-worker/config.yaml b/cluster/juju/layers/kubernetes-worker/config.yaml index 7123fc21b7b..048cf9eb816 100644 --- a/cluster/juju/layers/kubernetes-worker/config.yaml +++ b/cluster/juju/layers/kubernetes-worker/config.yaml @@ -80,3 +80,12 @@ options: description: | Docker image to use for the default backend. Auto will select an image based on architecture. + snapd_refresh: + default: "max" + type: string + description: | + How often snapd handles updates for installed snaps. Setting an empty + string will check 4x per day. Set to "max" to delay the refresh as long + as possible. You may also set a custom string as described in the + 'refresh.timer' section here: + https://forum.snapcraft.io/t/system-options/87 diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index 215fa7a3fea..d2af6f58e40 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -22,6 +22,8 @@ import shutil import subprocess import time +from charms.leadership import leader_get, leader_set + from pathlib import Path from shlex import split from subprocess import check_call, check_output @@ -289,6 +291,38 @@ def set_app_version(): hookenv.application_version_set(version.split(b' v')[-1].rstrip()) +@when('kubernetes-worker.snaps.installed') +@when('snap.refresh.set') +@when('leadership.is_leader') +def process_snapd_timer(): + ''' Set the snapd refresh timer on the leader so all cluster members + (present and future) will refresh near the same time. ''' + # Get the current snapd refresh timer; we know layer-snap has set this + # when the 'snap.refresh.set' flag is present. + timer = snap.get(snapname='core', key='refresh.timer').decode('utf-8') + + # The first time through, data_changed will be true. Subsequent calls + # should only update leader data if something changed. + if data_changed('worker_snapd_refresh', timer): + hookenv.log('setting snapd_refresh timer to: {}'.format(timer)) + leader_set({'snapd_refresh': timer}) + + +@when('kubernetes-worker.snaps.installed') +@when('snap.refresh.set') +@when('leadership.changed.snapd_refresh') +@when_not('leadership.is_leader') +def set_snapd_timer(): + ''' Set the snapd refresh.timer on non-leader cluster members. ''' + # NB: This method should only be run when 'snap.refresh.set' is present. + # Layer-snap will always set a core refresh.timer, which may not be the + # same as our leader. Gating with 'snap.refresh.set' ensures layer-snap + # has finished and we are free to set our config to the leader's timer. + timer = leader_get('snapd_refresh') + hookenv.log('setting snapd_refresh timer to: {}'.format(timer)) + snap.set_refresh_timer(timer) + + @when('kubernetes-worker.snaps.installed') @when_not('kube-control.dns.available') def notify_user_transient_status(): From 876859bb9704d26a3513e34ed3f3f896e9ab7acd Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Wed, 1 Aug 2018 13:00:49 +0300 Subject: [PATCH 03/13] Add leadership layer to kubernetes-worker --- cluster/juju/layers/kubernetes-worker/layer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/juju/layers/kubernetes-worker/layer.yaml b/cluster/juju/layers/kubernetes-worker/layer.yaml index af247e80f0a..e18d12dab40 100644 --- a/cluster/juju/layers/kubernetes-worker/layer.yaml +++ b/cluster/juju/layers/kubernetes-worker/layer.yaml @@ -3,6 +3,7 @@ includes: - 'layer:basic' - 'layer:debug' - 'layer:snap' + - 'layer:leadership' - 'layer:docker' - 'layer:metrics' - 'layer:nagios' From e19de54a4b1d1e7b4d50e6af7c5eaa2e5d3d5556 Mon Sep 17 00:00:00 2001 From: Mike Wilson Date: Wed, 1 Aug 2018 11:05:32 -0400 Subject: [PATCH 04/13] Changing ceph CSI to use cdk-addons for template rendering --- .../reactive/kubernetes_master.py | 134 ------------------ 1 file changed, 134 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 2f9a34fa7a4..245eeebc4cd 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -928,140 +928,6 @@ def ceph_storage(ceph_admin): set_state('ceph-storage.configured') -@when('config.changed.default-storage') -def render_ceph(): - remove_state('ceph-storage.configured') - - -@when('ceph-storage.available') -@when('cdk-addons.configured') -@when_not('ceph-storage.configured') -def ceph_csi_storage(ceph_admin): - '''Ceph on kubernetes will require a few things put on the cluster. - This includes a provisioner and storage class. This method will - put those things on the cluster.''' - - # Pre-CSI versions still use the old apt version and manual volume - # creation, so skip the CSI stuff - if get_version('kube-apiserver') < (1, 10): - return - - if ceph_admin.key(): - encoded_key = base64.b64encode(ceph_admin.key().encode('utf-8')) - else: - # We didn't have a key, and cannot proceed. Do not set state and - # allow this method to re-execute - return - - ceph_context = { - 'mon_hosts': ceph_admin.mon_hosts(), - 'fsid': ceph_admin.fsid(), - 'auth_supported': ceph_admin.auth(), - 'use_syslog': "true", - 'ceph_public_network': '', - 'ceph_cluster_network': '', - 'loglevel': 1, - 'hostname': socket.gethostname(), - 'admin_key': encoded_key.decode('ascii'), - 'kubernetes_key': encoded_key.decode('ascii'), - } - - # Install the ceph common utilities. - apt_install(['ceph-common'], fatal=True) - - etc_ceph_directory = '/etc/ceph' - if not os.path.isdir(etc_ceph_directory): - os.makedirs(etc_ceph_directory) - charm_ceph_conf = os.path.join(etc_ceph_directory, 'ceph.conf') - # Render the ceph configuration from the ceph conf template - render('ceph.conf', charm_ceph_conf, ceph_context) - - config = hookenv.config() - default_storage = config.get('default-storage') - - # Create the ceph yamls from templates in cdk-addons - with TemporaryDirectory() as template_path: - addon_path = str(_cdk_addons_template_path()) - render('rbd-secrets.yaml', - '{}/rbd-secrets.yaml'.format(template_path), ceph_context, - templates_dir=addon_path) - render('rbdplugin.yaml', - '{}/rbdplugin.yaml'.format(template_path), ceph_context, - templates_dir=addon_path) - render('csi-provisioner.yaml', - '{}/csi-provisioner.yaml'.format(template_path), ceph_context, - templates_dir=addon_path) - render('csi-attacher.yaml', - '{}/csi-attacher.yaml'.format(template_path), ceph_context, - templates_dir=addon_path) - ceph_context['pool_name'] = 'ext4-pool' - ceph_context['fs_type'] = 'ext4' - ceph_context['sc_name'] = 'ceph-ext4' - if default_storage == 'ceph-ext4': - ceph_context['default'] = True - else: - ceph_context['default'] = False - render('rbd-storage-class.yaml', - '{}/rbd-storage-class-ext4.yaml'.format(template_path), - ceph_context, templates_dir=addon_path) - ceph_context['pool_name'] = 'xfs-pool' - ceph_context['fs_type'] = 'ext4' - ceph_context['sc_name'] = 'ceph-xfs' - if default_storage == 'ceph-xfs' or default_storage == 'auto': - ceph_context['default'] = True - else: - ceph_context['default'] = False - render('rbd-storage-class.yaml', - '{}/rbd-storage-class-xfs.yaml'.format(template_path), - ceph_context, templates_dir=addon_path) - - try: - # At first glance this is deceptive. The apply stanza will create - # if it doesn't exist, otherwise it will update the entry - cmd = ['kubectl', 'apply', '-f', template_path] - check_call(cmd) - except: # NOQA - # the enlistment in kubernetes failed, return and prepare for - # re-exec - return - - # when complete, set a state relating to configuration of the storage - # backend that will allow other modules to hook into this and verify we - # have performed the necessary pre-req steps to interface with a ceph - # deployment. - set_state('ceph-storage.configured') - - -@when_not('ceph-storage.available') -@when('ceph-storage.configured') -def remove_ceph(): - # Pre-CSI versions still use the old apt version and manual volume - # creation, so skip the CSI stuff - if get_version('kube-apiserver') < (1, 10): - return - - ceph_things = [['secret', 'csi-ceph-secret'], - ['sc', 'ceph-xfs'], - ['sc', 'ceph-ext4'], - ['serviceaccount', 'csi-rbdplugin'], - ['clusterrole', 'csi-rbdplugin'], - ['clusterrolebinding', 'csi-rbdplugin'], - ['daemonset', 'csi-rbdplugin'], - ['serviceaccount', 'csi-attacher'], - ['clusterrole', 'external-attacher-runner'], - ['clusterrolebinding', 'csi-attacher-role'], - ['service', 'csi-attacher'], - ['statefulset', 'csi-attacher'], - ['serviceaccount', 'csi-provisioner'], - ['clusterrole', 'external-provisioner-runner'], - ['clusterrolebinding', 'csi-provisioner-role'], - ['service', 'csi-provisioner'], - ['statefulset', 'csi-provisioner']] - for entity in ceph_things: - cmd = 'kubectl delete {} {}'.format(entity[0], entity[1]) - check_call(split(cmd)) - - @when('nrpe-external-master.available') @when_not('nrpe-external-master.initial-config') def initial_nrpe_config(nagios=None): From 2f5735372dda1561823729f83b8e6488ef60648a Mon Sep 17 00:00:00 2001 From: George Kraft Date: Wed, 8 Aug 2018 02:24:09 -0500 Subject: [PATCH 05/13] juju: Use KubeletConfiguration on Kubelet 1.10+ (#143) --- .../reactive/kubernetes_worker.py | 69 ++++++++++++++----- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index d2af6f58e40..eb814e08730 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -21,6 +21,7 @@ import random import shutil import subprocess import time +import yaml from charms.leadership import leader_get, leader_set @@ -684,29 +685,10 @@ def configure_kubelet(dns, ingress_ip): kubelet_opts['kubeconfig'] = kubeconfig_path kubelet_opts['network-plugin'] = 'cni' kubelet_opts['v'] = '0' - kubelet_opts['address'] = '0.0.0.0' - kubelet_opts['port'] = '10250' - kubelet_opts['cluster-domain'] = dns['domain'] - kubelet_opts['anonymous-auth'] = 'false' - kubelet_opts['client-ca-file'] = ca_cert_path - kubelet_opts['tls-cert-file'] = server_cert_path - kubelet_opts['tls-private-key-file'] = server_key_path kubelet_opts['logtostderr'] = 'true' - kubelet_opts['fail-swap-on'] = 'false' kubelet_opts['node-ip'] = ingress_ip - - if (dns['enable-kube-dns']): - kubelet_opts['cluster-dns'] = dns['sdn-ip'] - - # set --allow-privileged flag for kubelet kubelet_opts['allow-privileged'] = set_privileged() - if is_state('kubernetes-worker.gpu.enabled'): - hookenv.log('Adding ' - '--feature-gates=DevicePlugins=true ' - 'to kubelet') - kubelet_opts['feature-gates'] = 'DevicePlugins=true' - if is_state('endpoint.aws.ready'): kubelet_opts['cloud-provider'] = 'aws' elif is_state('endpoint.gcp.ready'): @@ -718,6 +700,55 @@ def configure_kubelet(dns, ingress_ip): kubelet_opts['cloud-provider'] = 'openstack' kubelet_opts['cloud-config'] = str(cloud_config_path) + if get_version('kubelet') >= (1, 10): + # Put together the KubeletConfiguration data + kubelet_config = { + 'apiVersion': 'kubelet.config.k8s.io/v1beta1', + 'kind': 'KubeletConfiguration', + 'address': '0.0.0.0', + 'authentication': { + 'anonymous': { + 'enabled': False + }, + 'x509': { + 'clientCAFile': ca_cert_path + } + }, + 'clusterDomain': dns['domain'], + 'failSwapOn': False, + 'port': 10250, + 'tlsCertFile': server_cert_path, + 'tlsPrivateKeyFile': server_key_path + } + if dns['enable-kube-dns']: + kubelet_config['clusterDNS'] = [dns['sdn-ip']] + if is_state('kubernetes-worker.gpu.enabled'): + kubelet_config['featureGates'] = { + 'DevicePlugins': True + } + + # Render the file and configure Kubelet to use it + os.makedirs('/root/cdk/kubelet', exist_ok=True) + with open('/root/cdk/kubelet/config.yaml', 'w') as f: + f.write('# Generated by kubernetes-worker charm, do not edit\n') + yaml.dump(kubelet_config, f) + kubelet_opts['config'] = '/root/cdk/kubelet/config.yaml' + else: + # NOTE: This is for 1.9. Once we've dropped 1.9 support, we can remove + # this whole block and the parent if statement. + kubelet_opts['address'] = '0.0.0.0' + kubelet_opts['anonymous-auth'] = 'false' + kubelet_opts['client-ca-file'] = ca_cert_path + kubelet_opts['cluster-domain'] = dns['domain'] + kubelet_opts['fail-swap-on'] = 'false' + kubelet_opts['port'] = '10250' + kubelet_opts['tls-cert-file'] = server_cert_path + kubelet_opts['tls-private-key-file'] = server_key_path + if dns['enable-kube-dns']: + kubelet_opts['cluster-dns'] = dns['sdn-ip'] + if is_state('kubernetes-worker.gpu.enabled'): + kubelet_opts['feature-gates'] = 'DevicePlugins=true' + configure_kubernetes_service('kubelet', kubelet_opts, 'kubelet-extra-args') From 577849693ecbdc0549a639ce69924bd8804cb836 Mon Sep 17 00:00:00 2001 From: George Kraft Date: Thu, 9 Aug 2018 04:25:05 -0500 Subject: [PATCH 06/13] juju: Set kubelet dynamic-config-dir on Kubernetes 1.11+ (#144) --- .../layers/kubernetes-worker/reactive/kubernetes_worker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index eb814e08730..2008efe9de4 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -749,6 +749,9 @@ def configure_kubelet(dns, ingress_ip): if is_state('kubernetes-worker.gpu.enabled'): kubelet_opts['feature-gates'] = 'DevicePlugins=true' + if get_version('kubelet') >= (1, 11): + kubelet_opts['dynamic-config-dir'] = '/root/cdk/kubelet/dynamic-config' + configure_kubernetes_service('kubelet', kubelet_opts, 'kubelet-extra-args') From fed74b7299c32a48e72a0d7905cecb2559372319 Mon Sep 17 00:00:00 2001 From: George Kraft Date: Thu, 16 Aug 2018 06:54:15 -0500 Subject: [PATCH 07/13] juju: Add kubelet-extra-config to kubernetes-worker (#145) --- .../juju/layers/kubernetes-worker/config.yaml | 17 +++++++++++++ .../reactive/kubernetes_worker.py | 25 +++++++++++++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/cluster/juju/layers/kubernetes-worker/config.yaml b/cluster/juju/layers/kubernetes-worker/config.yaml index 048cf9eb816..2f15134f259 100644 --- a/cluster/juju/layers/kubernetes-worker/config.yaml +++ b/cluster/juju/layers/kubernetes-worker/config.yaml @@ -89,3 +89,20 @@ options: as possible. You may also set a custom string as described in the 'refresh.timer' section here: https://forum.snapcraft.io/t/system-options/87 + kubelet-extra-config: + default: "{}" + type: string + description: | + Extra configuration to be passed to kubelet. Any values specified in this + config will be merged into a KubeletConfiguration file that is passed to + the kubelet service via the --config flag. This can be used to override + values provided by the charm. + + Requires Kubernetes 1.10+. + + The value for this config must be a YAML mapping that can be safely + merged with a KubeletConfiguration file. For example: + {evictionHard: {memory.available: 200Mi}} + + For more information about KubeletConfiguration, see upstream docs: + https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/ diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index 2008efe9de4..48dbd8dacbe 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -552,8 +552,9 @@ def apply_node_labels(): @when_any('config.changed.kubelet-extra-args', - 'config.changed.proxy-extra-args') -def extra_args_changed(): + 'config.changed.proxy-extra-args', + 'config.changed.kubelet-extra-config') +def config_changed_requires_restart(): set_state('kubernetes-worker.restart-needed') @@ -674,6 +675,20 @@ def configure_kubernetes_service(service, base_args, extra_args_key): db.set(prev_args_key, args) +def merge_kubelet_extra_config(config, extra_config): + ''' Updates config to include the contents of extra_config. This is done + recursively to allow deeply nested dictionaries to be merged. + + This is destructive: it modifies the config dict that is passed in. + ''' + for k, extra_config_value in extra_config.items(): + if isinstance(extra_config_value, dict): + config_value = config.setdefault(k, {}) + merge_kubelet_extra_config(config_value, extra_config_value) + else: + config[k] = extra_config_value + + def configure_kubelet(dns, ingress_ip): layer_options = layer.options('tls-client') ca_cert_path = layer_options.get('ca_certificate_path') @@ -727,6 +742,12 @@ def configure_kubelet(dns, ingress_ip): 'DevicePlugins': True } + # Add kubelet-extra-config. This needs to happen last so that it + # overrides any config provided by the charm. + kubelet_extra_config = hookenv.config('kubelet-extra-config') + kubelet_extra_config = yaml.load(kubelet_extra_config) + merge_kubelet_extra_config(kubelet_config, kubelet_extra_config) + # Render the file and configure Kubelet to use it os.makedirs('/root/cdk/kubelet', exist_ok=True) with open('/root/cdk/kubelet/config.yaml', 'w') as f: From 0687655c53d830759528f8f57020f6faef222707 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Sat, 18 Aug 2018 09:38:11 -0500 Subject: [PATCH 08/13] add letters to source code to make native vsphere integration (reqs >= 1.12) --- .../juju/layers/kubernetes-master/layer.yaml | 1 + .../layers/kubernetes-master/metadata.yaml | 2 + .../reactive/kubernetes_master.py | 49 +++++++++++++++++-- .../juju/layers/kubernetes-worker/layer.yaml | 1 + .../layers/kubernetes-worker/metadata.yaml | 2 + .../reactive/kubernetes_worker.py | 6 +++ 6 files changed, 58 insertions(+), 3 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/layer.yaml b/cluster/juju/layers/kubernetes-master/layer.yaml index 44488b5ab2b..b8f5dfc0f18 100644 --- a/cluster/juju/layers/kubernetes-master/layer.yaml +++ b/cluster/juju/layers/kubernetes-master/layer.yaml @@ -18,6 +18,7 @@ includes: - 'interface:aws-integration' - 'interface:gcp-integration' - 'interface:openstack-integration' + - 'interface:vsphere-integration' options: basic: packages: diff --git a/cluster/juju/layers/kubernetes-master/metadata.yaml b/cluster/juju/layers/kubernetes-master/metadata.yaml index 8cbc0b50c94..88759653c7d 100644 --- a/cluster/juju/layers/kubernetes-master/metadata.yaml +++ b/cluster/juju/layers/kubernetes-master/metadata.yaml @@ -47,6 +47,8 @@ requires: interface: gcp-integration openstack: interface: openstack-integration + vsphere: + interface: vsphere-integration resources: kubectl: type: file diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 245eeebc4cd..4743a3a60cd 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -511,12 +511,20 @@ def set_final_status(): return req_sent = is_state('kubernetes-master.cloud-request-sent') + # openstack and vsphere have admin perms; cloud req is not required openstack_joined = is_state('endpoint.openstack.joined') - cloud_req = req_sent or openstack_joined + vsphere_joined = is_state('endpoint.vsphere.joined') + cloud_req = req_sent or openstack_joined or vsphere_joined aws_ready = is_state('endpoint.aws.ready') gcp_ready = is_state('endpoint.gcp.ready') openstack_ready = is_state('endpoint.openstack.ready') - cloud_ready = aws_ready or gcp_ready or openstack_ready + vsphere_ready = is_state('endpoint.vsphere.ready') + if vsphere_ready and get_version('kube-apiserver') < (1, 12): + msg = 'vSphere integration requires K8s 1.12 or greater' + hookenv.status_set('blocked', msg) + return + + cloud_ready = aws_ready or gcp_ready or openstack_ready or vsphere_ready if cloud_req and not cloud_ready: hookenv.status_set('waiting', 'waiting for cloud integration') @@ -1369,6 +1377,11 @@ def configure_apiserver(etcd_connection_string): cloud_config_path = _cloud_config_path('kube-apiserver') api_opts['cloud-provider'] = 'openstack' api_opts['cloud-config'] = str(cloud_config_path) + elif (is_state('endpoint.vsphere.ready') and + get_version('kube-apiserver') >= (1, 12)): + cloud_config_path = _cloud_config_path('kube-apiserver') + api_opts['cloud-provider'] = 'vsphere' + api_opts['cloud-config'] = str(cloud_config_path) audit_root = '/root/cdk/audit' os.makedirs(audit_root, exist_ok=True) @@ -1426,6 +1439,11 @@ def configure_controller_manager(): cloud_config_path = _cloud_config_path('kube-controller-manager') controller_opts['cloud-provider'] = 'openstack' controller_opts['cloud-config'] = str(cloud_config_path) + elif (is_state('endpoint.vsphere.ready') and + get_version('kube-apiserver') >= (1, 12)): + cloud_config_path = _cloud_config_path('kube-controller-manager') + controller_opts['cloud-provider'] = 'vsphere' + controller_opts['cloud-config'] = str(cloud_config_path) configure_kubernetes_service('kube-controller-manager', controller_opts, 'controller-manager-extra-args') @@ -1696,7 +1714,8 @@ def clear_requested_integration(): @when_any('endpoint.aws.ready', 'endpoint.gcp.ready', - 'endpoint.openstack.ready') + 'endpoint.openstack.ready', + 'endpoint.vsphere.ready') @when_not('kubernetes-master.restarted-for-cloud') def restart_for_cloud(): if is_state('endpoint.gcp.ready'): @@ -1705,6 +1724,9 @@ def restart_for_cloud(): elif is_state('endpoint.openstack.ready'): _write_openstack_snap_config('kube-apiserver') _write_openstack_snap_config('kube-controller-manager') + elif is_state('endpoint.vsphere.ready'): + _write_vsphere_snap_config('kube-apiserver') + _write_vsphere_snap_config('kube-controller-manager') set_state('kubernetes-master.restarted-for-cloud') remove_state('kubernetes-master.components.started') # force restart @@ -1771,3 +1793,24 @@ def _write_openstack_snap_config(component): 'tenant-name = {}'.format(openstack.project_name), 'domain-name = {}'.format(openstack.user_domain_name), ])) + + +def _write_vsphere_snap_config(component): + # vsphere requires additional cloud config + vsphere = endpoint_from_flag('endpoint.vsphere.ready') + + cloud_config_path = _cloud_config_path(component) + cloud_config_path.write_text('\n'.join([ + '[Global]', + 'user = {}'.format(vsphere.user), + 'password = {}'.format(vsphere.password), + 'insecure-flag = "1"', + 'datacenters = {}'.format(vsphere.datacenter), + '[VirtualCenter "{}"]'.format(vsphere.vsphere_ip), + '[Workspace]', + 'server = {}'.format(vsphere.vsphere_ip), + 'datacenter = "{}"'.format(vsphere.datacenter), + 'default-datastore = "{}"'.format(vsphere.datastore), + '[Disk]', + 'scsicontrollertype = "pvscsi"', + ])) diff --git a/cluster/juju/layers/kubernetes-worker/layer.yaml b/cluster/juju/layers/kubernetes-worker/layer.yaml index e18d12dab40..b3b47c7d869 100644 --- a/cluster/juju/layers/kubernetes-worker/layer.yaml +++ b/cluster/juju/layers/kubernetes-worker/layer.yaml @@ -16,6 +16,7 @@ includes: - 'interface:aws-integration' - 'interface:gcp-integration' - 'interface:openstack-integration' + - 'interface:vsphere-integration' - 'interface:mount' config: deletes: diff --git a/cluster/juju/layers/kubernetes-worker/metadata.yaml b/cluster/juju/layers/kubernetes-worker/metadata.yaml index 3f3a83c47fb..bb19271cc9f 100644 --- a/cluster/juju/layers/kubernetes-worker/metadata.yaml +++ b/cluster/juju/layers/kubernetes-worker/metadata.yaml @@ -36,6 +36,8 @@ requires: interface: gcp-integration openstack: interface: openstack-integration + vsphere: + interface: vsphere-integration nfs: interface: mount provides: diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index 48dbd8dacbe..ed76687980c 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -714,6 +714,10 @@ def configure_kubelet(dns, ingress_ip): cloud_config_path = _cloud_config_path('kubelet') kubelet_opts['cloud-provider'] = 'openstack' kubelet_opts['cloud-config'] = str(cloud_config_path) + elif is_state('endpoint.vsphere.ready'): + # vsphere doesnt need a cloud config on the worker + cloud_config_path = _cloud_config_path('kubelet') + kubelet_opts['cloud-provider'] = 'vsphere' if get_version('kubelet') >= (1, 10): # Put together the KubeletConfiguration data @@ -1193,6 +1197,8 @@ def get_node_name(): cloud_provider = 'gce' elif is_state('endpoint.openstack.ready'): cloud_provider = 'openstack' + elif is_state('endpoint.vsphere.ready'): + cloud_provider = 'vsphere' if cloud_provider == 'aws': return getfqdn().lower() else: From 809dc2c763fc6546df3e1b02c38f9d806aaf5260 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Mon, 20 Aug 2018 05:20:59 +0000 Subject: [PATCH 09/13] add uuid bits to master/worker to make it a bit easier on operators --- .../reactive/kubernetes_master.py | 21 ++++++++++++++++--- .../reactive/kubernetes_worker.py | 9 ++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 4743a3a60cd..a658a3479eb 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -1799,18 +1799,33 @@ def _write_vsphere_snap_config(component): # vsphere requires additional cloud config vsphere = endpoint_from_flag('endpoint.vsphere.ready') + # NB: vsphere provider will ask kube-apiserver and -controller-manager to + # find a uuid from sysfs unless a global config value is set. Our strict + # snaps cannot read sysfs, so let's do it in the charm. An invalid uuid is + # not fatal for storage, but it will muddy the logs; try to get it right. + uuid_file = '/sys/class/dmi/id/product_uuid' + try: + with open(uuid_file, 'r') as f: + uuid = f.read().strip() + except IOError as err: + hookenv.log("Unable to read UUID from sysfs: {}".format(err)) + uuid = 'UNKNOWN' + cloud_config_path = _cloud_config_path(component) cloud_config_path.write_text('\n'.join([ '[Global]', + 'insecure-flag = true', + 'datacenters = "{}"'.format(vsphere.datacenter), + 'vm-uuid = "VMware-{}"'.format(uuid), + '[VirtualCenter "{}"]'.format(vsphere.vsphere_ip), 'user = {}'.format(vsphere.user), 'password = {}'.format(vsphere.password), - 'insecure-flag = "1"', - 'datacenters = {}'.format(vsphere.datacenter), - '[VirtualCenter "{}"]'.format(vsphere.vsphere_ip), '[Workspace]', 'server = {}'.format(vsphere.vsphere_ip), 'datacenter = "{}"'.format(vsphere.datacenter), 'default-datastore = "{}"'.format(vsphere.datastore), + 'folder = "kubernetes"', + 'resourcepool-path = ""', '[Disk]', 'scsicontrollertype = "pvscsi"', ])) diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index ed76687980c..4d285328fc9 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -714,10 +714,15 @@ def configure_kubelet(dns, ingress_ip): cloud_config_path = _cloud_config_path('kubelet') kubelet_opts['cloud-provider'] = 'openstack' kubelet_opts['cloud-config'] = str(cloud_config_path) - elif is_state('endpoint.vsphere.ready'): - # vsphere doesnt need a cloud config on the worker + elif is_state('endpoint.vsphere.joined'): + # vsphere just needs to be joined on the worker (vs 'ready') cloud_config_path = _cloud_config_path('kubelet') kubelet_opts['cloud-provider'] = 'vsphere' + # NB: vsphere maps node product-id to its uuid (no config file needed). + uuid_file = '/sys/class/dmi/id/product_uuid' + with open(uuid_file, 'r') as f: + uuid = f.read().strip() + kubelet_opts['provider-id'] = 'vsphere://{}'.format(uuid) if get_version('kubelet') >= (1, 10): # Put together the KubeletConfiguration data From 99a631fc80f6189b0a9285cfe2a7c18de5cc8e55 Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Thu, 16 Aug 2018 11:39:01 -0400 Subject: [PATCH 10/13] Add Azure Integrator support to k8s charms --- .../juju/layers/kubernetes-master/layer.yaml | 1 + .../layers/kubernetes-master/metadata.yaml | 2 + .../reactive/kubernetes_master.py | 115 ++++++++++--- .../juju/layers/kubernetes-worker/layer.yaml | 1 + .../layers/kubernetes-worker/metadata.yaml | 2 + .../reactive/kubernetes_worker.py | 153 +++++++++++++----- 6 files changed, 209 insertions(+), 65 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/layer.yaml b/cluster/juju/layers/kubernetes-master/layer.yaml index b8f5dfc0f18..b1a1ed22c1a 100644 --- a/cluster/juju/layers/kubernetes-master/layer.yaml +++ b/cluster/juju/layers/kubernetes-master/layer.yaml @@ -19,6 +19,7 @@ includes: - 'interface:gcp-integration' - 'interface:openstack-integration' - 'interface:vsphere-integration' + - 'interface:azure-integration' options: basic: packages: diff --git a/cluster/juju/layers/kubernetes-master/metadata.yaml b/cluster/juju/layers/kubernetes-master/metadata.yaml index 88759653c7d..03746ad95a2 100644 --- a/cluster/juju/layers/kubernetes-master/metadata.yaml +++ b/cluster/juju/layers/kubernetes-master/metadata.yaml @@ -49,6 +49,8 @@ requires: interface: openstack-integration vsphere: interface: vsphere-integration + azure: + interface: azure-integration resources: kubectl: type: file diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index a658a3479eb..1ef69367897 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -467,6 +467,22 @@ def set_final_status(): except NotImplementedError: goal_state = {} + vsphere_joined = is_state('endpoint.vsphere.joined') + azure_joined = is_state('endpoint.azure.joined') + cloud_blocked = is_state('kubernetes-master.cloud.blocked') + if vsphere_joined and cloud_blocked: + hookenv.status_set('blocked', + 'vSphere integration requires K8s 1.12 or greater') + return + if azure_joined and cloud_blocked: + hookenv.status_set('blocked', + 'Azure integration requires K8s 1.11 or greater') + return + + if is_state('kubernetes-master.cloud.pending'): + hookenv.status_set('waiting', 'Waiting for cloud integration') + return + if not is_state('kube-api-endpoint.available'): if 'kube-api-endpoint' in goal_state.get('relations', {}): status = 'waiting' @@ -510,24 +526,6 @@ def set_final_status(): hookenv.status_set('waiting', 'Waiting to retry addon deployment') return - req_sent = is_state('kubernetes-master.cloud-request-sent') - # openstack and vsphere have admin perms; cloud req is not required - openstack_joined = is_state('endpoint.openstack.joined') - vsphere_joined = is_state('endpoint.vsphere.joined') - cloud_req = req_sent or openstack_joined or vsphere_joined - aws_ready = is_state('endpoint.aws.ready') - gcp_ready = is_state('endpoint.gcp.ready') - openstack_ready = is_state('endpoint.openstack.ready') - vsphere_ready = is_state('endpoint.vsphere.ready') - if vsphere_ready and get_version('kube-apiserver') < (1, 12): - msg = 'vSphere integration requires K8s 1.12 or greater' - hookenv.status_set('blocked', msg) - return - - cloud_ready = aws_ready or gcp_ready or openstack_ready or vsphere_ready - if cloud_req and not cloud_ready: - hookenv.status_set('waiting', 'waiting for cloud integration') - if addons_configured and not all_kube_system_pods_running(): hookenv.status_set('waiting', 'Waiting for kube-system pods to start') return @@ -565,7 +563,9 @@ def master_services_down(): @when('etcd.available', 'tls_client.server.certificate.saved', 'authentication.setup') @when('leadership.set.auto_storage_backend') -@when_not('kubernetes-master.components.started') +@when_not('kubernetes-master.components.started', + 'kubernetes-master.cloud.pending', + 'kubernetes-master.cloud.blocked') def start_master(etcd): '''Run the Kubernetes master components.''' hookenv.status_set('maintenance', @@ -1382,6 +1382,10 @@ def configure_apiserver(etcd_connection_string): cloud_config_path = _cloud_config_path('kube-apiserver') api_opts['cloud-provider'] = 'vsphere' api_opts['cloud-config'] = str(cloud_config_path) + elif is_state('endpoint.azure.ready'): + cloud_config_path = _cloud_config_path('kube-apiserver') + api_opts['cloud-provider'] = 'azure' + api_opts['cloud-config'] = str(cloud_config_path) audit_root = '/root/cdk/audit' os.makedirs(audit_root, exist_ok=True) @@ -1444,6 +1448,10 @@ def configure_controller_manager(): cloud_config_path = _cloud_config_path('kube-controller-manager') controller_opts['cloud-provider'] = 'vsphere' controller_opts['cloud-config'] = str(cloud_config_path) + elif is_state('endpoint.azure.ready'): + cloud_config_path = _cloud_config_path('kube-controller-manager') + controller_opts['cloud-provider'] = 'azure' + controller_opts['cloud-config'] = str(cloud_config_path) configure_kubernetes_service('kube-controller-manager', controller_opts, 'controller-manager-extra-args') @@ -1670,7 +1678,25 @@ def clear_cluster_tag_sent(): @when_any('endpoint.aws.joined', - 'endpoint.gcp.joined') + 'endpoint.gcp.joined', + 'endpoint.openstack.joined', + 'endpoint.vsphere.joined', + 'endpoint.azure.joined') +@when_not('kubernetes-master.cloud.ready') +def set_cloud_pending(): + k8s_version = get_version('kube-apiserver') + k8s_1_11 = k8s_version >= (1, 11) + k8s_1_12 = k8s_version >= (1, 12) + vsphere_joined = is_state('endpoint.vsphere.joined') + azure_joined = is_state('endpoint.azure.joined') + if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11): + set_state('kubernetes-master.cloud.blocked') + set_state('kubernetes-master.cloud.pending') + + +@when_any('endpoint.aws.joined', + 'endpoint.gcp.joined', + 'endpoint.azure.joined') @when('leadership.set.cluster_tag') @when_not('kubernetes-master.cloud-request-sent') def request_integration(): @@ -1698,6 +1724,14 @@ def request_integration(): }) cloud.enable_object_storage_management() cloud.enable_security_management() + elif is_state('endpoint.azure.joined'): + cloud = endpoint_from_flag('endpoint.azure.joined') + cloud.tag_instance({ + 'k8s-io-cluster-name': cluster_tag, + 'k8s-io-role-master': 'master', + }) + cloud.enable_object_storage_management() + cloud.enable_security_management() cloud.enable_instance_inspection() cloud.enable_network_management() cloud.enable_dns_management() @@ -1706,18 +1740,27 @@ def request_integration(): @when_none('endpoint.aws.joined', - 'endpoint.gcp.joined') + 'endpoint.gcp.joined', + 'endpoint.openstack.joined', + 'endpoint.vsphere.joined', + 'endpoint.azure.joined') @when('kubernetes-master.cloud-request-sent') def clear_requested_integration(): + remove_state('kubernetes-master.cloud.pending') remove_state('kubernetes-master.cloud-request-sent') + remove_state('kubernetes-master.cloud.blocked') + remove_state('kubernetes-master.cloud.ready') @when_any('endpoint.aws.ready', 'endpoint.gcp.ready', 'endpoint.openstack.ready', - 'endpoint.vsphere.ready') -@when_not('kubernetes-master.restarted-for-cloud') -def restart_for_cloud(): + 'endpoint.vsphere.ready', + 'endpoint.azure.ready') +@when_not('kubernetes-master.cloud.blocked', + 'kubernetes-master.cloud.ready', + 'kubernetes-master.restarted-for-cloud') # compat. TODO: remove +def cloud_ready(): if is_state('endpoint.gcp.ready'): _write_gcp_snap_config('kube-apiserver') _write_gcp_snap_config('kube-controller-manager') @@ -1727,10 +1770,21 @@ def restart_for_cloud(): elif is_state('endpoint.vsphere.ready'): _write_vsphere_snap_config('kube-apiserver') _write_vsphere_snap_config('kube-controller-manager') - set_state('kubernetes-master.restarted-for-cloud') + elif is_state('endpoint.azure.ready'): + _write_azure_snap_config('kube-apiserver') + _write_azure_snap_config('kube-controller-manager') + remove_state('kubernetes-master.cloud.pending') + set_state('kubernetes-master.cloud.ready') remove_state('kubernetes-master.components.started') # force restart +@when('kubernetes-master.restarted-for-cloud') +@when_not('kubernetes-master.cloud.ready') +def convert_cloud_flag(): + remove_state('kubernetes-master.restarted-for-cloud') + set_state('kubernetes-master.cloud.ready') + + def _snap_common_path(component): return Path('/var/snap/{}/common'.format(component)) @@ -1829,3 +1883,14 @@ def _write_vsphere_snap_config(component): '[Disk]', 'scsicontrollertype = "pvscsi"', ])) + + +def _write_azure_snap_config(component): + azure = endpoint_from_flag('endpoint.azure.ready') + cloud_config_path = _cloud_config_path(component) + cloud_config_path.write_text(json.dumps({ + 'useInstanceMetadata': True, + 'useManagedIdentityExtension': True, + 'resourceGroup': azure.resource_group, + 'subscriptionId': azure.subscription_id, + })) diff --git a/cluster/juju/layers/kubernetes-worker/layer.yaml b/cluster/juju/layers/kubernetes-worker/layer.yaml index b3b47c7d869..e7958853ae5 100644 --- a/cluster/juju/layers/kubernetes-worker/layer.yaml +++ b/cluster/juju/layers/kubernetes-worker/layer.yaml @@ -17,6 +17,7 @@ includes: - 'interface:gcp-integration' - 'interface:openstack-integration' - 'interface:vsphere-integration' + - 'interface:azure-integration' - 'interface:mount' config: deletes: diff --git a/cluster/juju/layers/kubernetes-worker/metadata.yaml b/cluster/juju/layers/kubernetes-worker/metadata.yaml index bb19271cc9f..d4040fb449b 100644 --- a/cluster/juju/layers/kubernetes-worker/metadata.yaml +++ b/cluster/juju/layers/kubernetes-worker/metadata.yaml @@ -38,6 +38,8 @@ requires: interface: openstack-integration vsphere: interface: vsphere-integration + azure: + interface: azure-integration nfs: interface: mount provides: diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index 4d285328fc9..e00ef541c85 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -192,13 +192,6 @@ def channel_changed(): set_upgrade_needed() -@when('kubernetes-worker.snaps.upgrade-needed') -@when_not('kubernetes-worker.snaps.upgrade-specified') -def upgrade_needed_status(): - msg = 'Needs manual upgrade, run the upgrade action' - hookenv.status_set('blocked', msg) - - @when('kubernetes-worker.snaps.upgrade-specified') def install_snaps(): channel = hookenv.config('channel') @@ -324,27 +317,42 @@ def set_snapd_timer(): snap.set_refresh_timer(timer) -@when('kubernetes-worker.snaps.installed') -@when_not('kube-control.dns.available') -def notify_user_transient_status(): - ''' Notify to the user we are in a transient state and the application - is still converging. Potentially remotely, or we may be in a detached loop - wait state ''' - - # During deployment the worker has to start kubelet without cluster dns - # configured. If this is the first unit online in a service pool waiting - # to self host the dns pod, and configure itself to query the dns service - # declared in the kube-system namespace - - hookenv.status_set('waiting', 'Waiting for cluster DNS.') - - -@when('kubernetes-worker.snaps.installed', - 'kube-control.dns.available') -@when_not('kubernetes-worker.snaps.upgrade-needed') -def charm_status(kube_control): +@hookenv.atexit +def charm_status(): '''Update the status message with the current status of kubelet.''' - update_kubelet_status() + vsphere_joined = is_state('endpoint.vsphere.joined') + azure_joined = is_state('endpoint.azure.joined') + cloud_blocked = is_state('kubernetes-worker.cloud.blocked') + if vsphere_joined and cloud_blocked: + hookenv.status_set('blocked', + 'vSphere integration requires K8s 1.12 or greater') + return + if azure_joined and cloud_blocked: + hookenv.status_set('blocked', + 'Azure integration requires K8s 1.11 or greater') + return + if is_state('kubernetes-worker.cloud.pending'): + hookenv.status_set('waiting', 'Waiting for cloud integration') + return + if not is_state('kube-control.dns.available'): + # During deployment the worker has to start kubelet without cluster dns + # configured. If this is the first unit online in a service pool + # waiting to self host the dns pod, and configure itself to query the + # dns service declared in the kube-system namespace + hookenv.status_set('waiting', 'Waiting for cluster DNS.') + return + if is_state('kubernetes-worker.snaps.upgrade-specified'): + hookenv.status_set('waiting', 'Upgrade pending') + return + if is_state('kubernetes-worker.snaps.upgrade-needed'): + hookenv.status_set('blocked', + 'Needs manual upgrade, run the upgrade action') + return + if is_state('kubernetes-worker.snaps.installed'): + update_kubelet_status() + return + else: + pass # will have been set by snap layer or other handler def update_kubelet_status(): @@ -429,6 +437,8 @@ def watch_for_changes(kube_api, kube_control, cni): 'kube-control.dns.available', 'kube-control.auth.available', 'cni.available', 'kubernetes-worker.restart-needed', 'worker.auth.bootstrapped') +@when_not('kubernetes-worker.cloud.pending', + 'kubernetes-worker.cloud.blocked') def start_worker(kube_api, kube_control, auth_control, cni): ''' Start kubelet using the provided API and DNS info.''' servers = get_kube_api_servers(kube_api) @@ -723,6 +733,12 @@ def configure_kubelet(dns, ingress_ip): with open(uuid_file, 'r') as f: uuid = f.read().strip() kubelet_opts['provider-id'] = 'vsphere://{}'.format(uuid) + elif is_state('endpoint.azure.ready'): + azure = endpoint_from_flag('endpoint.azure.ready') + cloud_config_path = _cloud_config_path('kubelet') + kubelet_opts['cloud-provider'] = 'azure' + kubelet_opts['cloud-config'] = str(cloud_config_path) + kubelet_opts['provider-id'] = azure.vm_id if get_version('kubelet') >= (1, 10): # Put together the KubeletConfiguration data @@ -892,10 +908,10 @@ def launch_default_ingress_controller(): 'ingress-ssl-chain-completion') context['ingress_image'] = config.get('nginx-image') if context['ingress_image'] == "" or context['ingress_image'] == "auto": - images = {'amd64': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1', # noqa - 'arm64': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.16.1', # noqa - 's390x': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-s390x:0.16.1', # noqa - 'ppc64el': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-ppc64le:0.16.1', # noqa + images = {'amd64': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1', # noqa + 'arm64': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.16.1', # noqa + 's390x': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-s390x:0.16.1', # noqa + 'ppc64el': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-ppc64le:0.16.1', # noqa } context['ingress_image'] = images.get(context['arch'], images['amd64']) if get_version('kubelet') < (1, 9): @@ -1204,6 +1220,8 @@ def get_node_name(): cloud_provider = 'openstack' elif is_state('endpoint.vsphere.ready'): cloud_provider = 'vsphere' + elif is_state('endpoint.azure.ready'): + cloud_provider = 'azure' if cloud_provider == 'aws': return getfqdn().lower() else: @@ -1247,7 +1265,25 @@ def remove_label(label): @when_any('endpoint.aws.joined', - 'endpoint.gcp.joined') + 'endpoint.gcp.joined', + 'endpoint.openstack.joined', + 'endpoint.vsphere.joined', + 'endpoint.azure.joined') +@when_not('kubernetes-worker.cloud.ready') +def set_cloud_pending(): + k8s_version = get_version('kubelet') + k8s_1_11 = k8s_version >= (1, 11) + k8s_1_12 = k8s_version >= (1, 12) + vsphere_joined = is_state('endpoint.vsphere.joined') + azure_joined = is_state('endpoint.azure.joined') + if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11): + set_state('kubernetes-worker.cloud.blocked') + set_state('kubernetes-worker.cloud.pending') + + +@when_any('endpoint.aws.joined', + 'endpoint.gcp.joined', + 'endpoint.azure.joined') @when('kube-control.cluster_tag.available') @when_not('kubernetes-worker.cloud-request-sent') def request_integration(): @@ -1272,29 +1308,55 @@ def request_integration(): 'k8s-io-cluster-name': cluster_tag, }) cloud.enable_object_storage_management() + elif is_state('endpoint.azure.joined'): + cloud = endpoint_from_flag('endpoint.azure.joined') + cloud.tag_instance({ + 'k8s-io-cluster-name': cluster_tag, + }) + cloud.enable_object_storage_management() cloud.enable_instance_inspection() cloud.enable_dns_management() set_state('kubernetes-worker.cloud-request-sent') - hookenv.status_set('waiting', 'waiting for cloud integration') + hookenv.status_set('waiting', 'Waiting for cloud integration') @when_none('endpoint.aws.joined', - 'endpoint.gcp.joined') -def clear_requested_integration(): + 'endpoint.gcp.joined', + 'endpoint.openstack.joined', + 'endpoint.vsphere.joined', + 'endpoint.azure.joined') +def clear_cloud_flags(): + remove_state('kubernetes-worker.cloud.pending') remove_state('kubernetes-worker.cloud-request-sent') + remove_state('kubernetes-worker.cloud.blocked') + remove_state('kubernetes-worker.cloud.ready') @when_any('endpoint.aws.ready', 'endpoint.gcp.ready', - 'endpoint.openstack.ready') -@when_not('kubernetes-worker.restarted-for-cloud') -def restart_for_cloud(): + 'endpoint.openstack.ready', + 'endpoint.vsphere.ready', + 'endpoint.azure.ready') +@when_not('kubernetes-worker.cloud.blocked', + 'kubernetes-worker.cloud.ready', + 'kubernetes-worker.restarted-for-cloud') # compat. TODO: remove +def cloud_ready(): + remove_state('kubernetes-worker.cloud.pending') if is_state('endpoint.gcp.ready'): _write_gcp_snap_config('kubelet') elif is_state('endpoint.openstack.ready'): _write_openstack_snap_config('kubelet') - set_state('kubernetes-worker.restarted-for-cloud') - set_state('kubernetes-worker.restart-needed') + elif is_state('endpoint.azure.ready'): + _write_azure_snap_config('kubelet') + set_state('kubernetes-worker.cloud.ready') + set_state('kubernetes-worker.restart-needed') # force restart + + +@when('kubernetes-master.restarted-for-cloud') +@when_not('kubernetes-master.cloud.ready') +def convert_cloud_flag(): + remove_state('kubernetes-worker.restarted-for-cloud') + set_state('kubernetes-worker.cloud.ready') def _snap_common_path(component): @@ -1357,6 +1419,17 @@ def _write_openstack_snap_config(component): ])) +def _write_azure_snap_config(component): + azure = endpoint_from_flag('endpoint.azure.ready') + cloud_config_path = _cloud_config_path(component) + cloud_config_path.write_text(json.dumps({ + 'useInstanceMetadata': True, + 'useManagedIdentityExtension': True, + 'resourceGroup': azure.resource_group, + 'subscriptionId': azure.subscription_id, + })) + + def get_first_mount(mount_relation): mount_relation_list = mount_relation.mounts() if mount_relation_list and len(mount_relation_list) > 0: From ef10295e892e142679309dda6f9d3e96e7d417da Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Wed, 22 Aug 2018 12:03:16 -0400 Subject: [PATCH 11/13] Add missing config for Azure integration --- .../layers/kubernetes-master/reactive/kubernetes_master.py | 7 ++++++- .../layers/kubernetes-worker/reactive/kubernetes_worker.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 1ef69367897..89a58475907 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -1891,6 +1891,11 @@ def _write_azure_snap_config(component): cloud_config_path.write_text(json.dumps({ 'useInstanceMetadata': True, 'useManagedIdentityExtension': True, - 'resourceGroup': azure.resource_group, 'subscriptionId': azure.subscription_id, + 'resourceGroup': azure.resource_group, + 'location': azure.resource_group_location, + 'vnetName': azure.vnet_name, + 'vnetResourceGroup': azure.vnet_resource_group, + 'subnetName': azure.subnet_name, + 'securityGroupName': azure.security_group_name, })) diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index e00ef541c85..3bf60106b46 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -1425,8 +1425,13 @@ def _write_azure_snap_config(component): cloud_config_path.write_text(json.dumps({ 'useInstanceMetadata': True, 'useManagedIdentityExtension': True, - 'resourceGroup': azure.resource_group, 'subscriptionId': azure.subscription_id, + 'resourceGroup': azure.resource_group, + 'location': azure.resource_group_location, + 'vnetName': azure.vnet_name, + 'vnetResourceGroup': azure.vnet_resource_group, + 'subnetName': azure.subnet_name, + 'securityGroupName': azure.security_group_name, })) From 161540eb318f6829884b7e85465f192610d6346c Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Wed, 22 Aug 2018 12:17:19 -0400 Subject: [PATCH 12/13] Fixes and improvements per review --- .../reactive/kubernetes_master.py | 26 ++++++++++--------- .../reactive/kubernetes_worker.py | 23 +++++++++------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 89a58475907..b9ca89ef32a 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -106,6 +106,15 @@ def check_for_upgrade_needed(): '''An upgrade charm event was triggered by Juju, react to that here.''' hookenv.status_set('maintenance', 'Checking resources') + # migrate to new flags + if is_state('kubernetes-master.restarted-for-cloud'): + remove_state('kubernetes-master.restarted-for-cloud') + set_state('kubernetes-master.cloud.ready') + if is_state('kubernetes-master.cloud-request-sent'): + # minor change, just for consistency + remove_state('kubernetes-master.cloud-request-sent') + set_state('kubernetes-master.cloud.request-sent') + migrate_from_pre_snaps() add_rbac_roles() set_state('reconfigure.authentication.setup') @@ -1691,6 +1700,8 @@ def set_cloud_pending(): azure_joined = is_state('endpoint.azure.joined') if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11): set_state('kubernetes-master.cloud.blocked') + else: + remove_state('kubernetes-master.cloud.blocked') set_state('kubernetes-master.cloud.pending') @@ -1744,10 +1755,9 @@ def request_integration(): 'endpoint.openstack.joined', 'endpoint.vsphere.joined', 'endpoint.azure.joined') -@when('kubernetes-master.cloud-request-sent') -def clear_requested_integration(): +def clear_cloud_flags(): remove_state('kubernetes-master.cloud.pending') - remove_state('kubernetes-master.cloud-request-sent') + remove_state('kubernetes-master.cloud.request-sent') remove_state('kubernetes-master.cloud.blocked') remove_state('kubernetes-master.cloud.ready') @@ -1758,8 +1768,7 @@ def clear_requested_integration(): 'endpoint.vsphere.ready', 'endpoint.azure.ready') @when_not('kubernetes-master.cloud.blocked', - 'kubernetes-master.cloud.ready', - 'kubernetes-master.restarted-for-cloud') # compat. TODO: remove + 'kubernetes-master.cloud.ready') def cloud_ready(): if is_state('endpoint.gcp.ready'): _write_gcp_snap_config('kube-apiserver') @@ -1778,13 +1787,6 @@ def cloud_ready(): remove_state('kubernetes-master.components.started') # force restart -@when('kubernetes-master.restarted-for-cloud') -@when_not('kubernetes-master.cloud.ready') -def convert_cloud_flag(): - remove_state('kubernetes-master.restarted-for-cloud') - set_state('kubernetes-master.cloud.ready') - - def _snap_common_path(component): return Path('/var/snap/{}/common'.format(component)) diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index 3bf60106b46..59302a2c660 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -64,6 +64,15 @@ db = unitdata.kv() @hook('upgrade-charm') def upgrade_charm(): + # migrate to new flags + if is_state('kubernetes-worker.restarted-for-cloud'): + remove_state('kubernetes-worker.restarted-for-cloud') + set_state('kubernetes-worker.cloud.ready') + if is_state('kubernetes-worker.cloud-request-sent'): + # minor change, just for consistency + remove_state('kubernetes-worker.cloud-request-sent') + set_state('kubernetes-worker.cloud.request-sent') + # Trigger removal of PPA docker installation if it was previously set. set_state('config.changed.install_from_upstream') hookenv.atexit(remove_state, 'config.changed.install_from_upstream') @@ -1278,6 +1287,8 @@ def set_cloud_pending(): azure_joined = is_state('endpoint.azure.joined') if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11): set_state('kubernetes-worker.cloud.blocked') + else: + remove_state('kubernetes-worker.cloud.blocked') set_state('kubernetes-worker.cloud.pending') @@ -1327,7 +1338,7 @@ def request_integration(): 'endpoint.azure.joined') def clear_cloud_flags(): remove_state('kubernetes-worker.cloud.pending') - remove_state('kubernetes-worker.cloud-request-sent') + remove_state('kubernetes-worker.cloud.request-sent') remove_state('kubernetes-worker.cloud.blocked') remove_state('kubernetes-worker.cloud.ready') @@ -1338,8 +1349,7 @@ def clear_cloud_flags(): 'endpoint.vsphere.ready', 'endpoint.azure.ready') @when_not('kubernetes-worker.cloud.blocked', - 'kubernetes-worker.cloud.ready', - 'kubernetes-worker.restarted-for-cloud') # compat. TODO: remove + 'kubernetes-worker.cloud.ready') def cloud_ready(): remove_state('kubernetes-worker.cloud.pending') if is_state('endpoint.gcp.ready'): @@ -1352,13 +1362,6 @@ def cloud_ready(): set_state('kubernetes-worker.restart-needed') # force restart -@when('kubernetes-master.restarted-for-cloud') -@when_not('kubernetes-master.cloud.ready') -def convert_cloud_flag(): - remove_state('kubernetes-worker.restarted-for-cloud') - set_state('kubernetes-worker.cloud.ready') - - def _snap_common_path(component): return Path('/var/snap/{}/common'.format(component)) From 7c2f5e328f51de16d40e01e6a437a30ea01296ff Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Wed, 22 Aug 2018 15:50:49 -0400 Subject: [PATCH 13/13] Fix missed cloud-request-sent flags --- .../layers/kubernetes-master/reactive/kubernetes_master.py | 4 ++-- .../layers/kubernetes-worker/reactive/kubernetes_worker.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index b9ca89ef32a..6652aeafb12 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -1709,7 +1709,7 @@ def set_cloud_pending(): 'endpoint.gcp.joined', 'endpoint.azure.joined') @when('leadership.set.cluster_tag') -@when_not('kubernetes-master.cloud-request-sent') +@when_not('kubernetes-master.cloud.request-sent') def request_integration(): hookenv.status_set('maintenance', 'requesting cloud integration') cluster_tag = leader_get('cluster_tag') @@ -1747,7 +1747,7 @@ def request_integration(): cloud.enable_network_management() cloud.enable_dns_management() cloud.enable_block_storage_management() - set_state('kubernetes-master.cloud-request-sent') + set_state('kubernetes-master.cloud.request-sent') @when_none('endpoint.aws.joined', diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index 59302a2c660..52d4a4d0eb6 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -1296,7 +1296,7 @@ def set_cloud_pending(): 'endpoint.gcp.joined', 'endpoint.azure.joined') @when('kube-control.cluster_tag.available') -@when_not('kubernetes-worker.cloud-request-sent') +@when_not('kubernetes-worker.cloud.request-sent') def request_integration(): hookenv.status_set('maintenance', 'requesting cloud integration') kube_control = endpoint_from_flag('kube-control.cluster_tag.available') @@ -1327,7 +1327,7 @@ def request_integration(): cloud.enable_object_storage_management() cloud.enable_instance_inspection() cloud.enable_dns_management() - set_state('kubernetes-worker.cloud-request-sent') + set_state('kubernetes-worker.cloud.request-sent') hookenv.status_set('waiting', 'Waiting for cloud integration')