Add Azure Integrator support to k8s charms

cluster/juju/layers/kubernetes-master/layer.yaml

@@ -19,6 +19,7 @@ includes:
   - 'interface:gcp-integration'
   - 'interface:openstack-integration'
   - 'interface:vsphere-integration'
+  - 'interface:azure-integration'
 options:
   basic:
     packages:

cluster/juju/layers/kubernetes-master/metadata.yaml

@@ -49,6 +49,8 @@ requires:
     interface: openstack-integration
   vsphere:
     interface: vsphere-integration
+  azure:
+    interface: azure-integration
 resources:
   kubectl:
     type: file

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -467,6 +467,22 @@ def set_final_status():
     except NotImplementedError:
         goal_state = {}
 
+    vsphere_joined = is_state('endpoint.vsphere.joined')
+    azure_joined = is_state('endpoint.azure.joined')
+    cloud_blocked = is_state('kubernetes-master.cloud.blocked')
+    if vsphere_joined and cloud_blocked:
+        hookenv.status_set('blocked',
+                           'vSphere integration requires K8s 1.12 or greater')
+        return
+    if azure_joined and cloud_blocked:
+        hookenv.status_set('blocked',
+                           'Azure integration requires K8s 1.11 or greater')
+        return
+
+    if is_state('kubernetes-master.cloud.pending'):
+        hookenv.status_set('waiting', 'Waiting for cloud integration')
+        return
+
     if not is_state('kube-api-endpoint.available'):
         if 'kube-api-endpoint' in goal_state.get('relations', {}):
             status = 'waiting'
@@ -510,24 +526,6 @@ def set_final_status():
         hookenv.status_set('waiting', 'Waiting to retry addon deployment')
         return
 
-    req_sent = is_state('kubernetes-master.cloud-request-sent')
-    # openstack and vsphere have admin perms; cloud req is not required
-    openstack_joined = is_state('endpoint.openstack.joined')
-    vsphere_joined = is_state('endpoint.vsphere.joined')
-    cloud_req = req_sent or openstack_joined or vsphere_joined
-    aws_ready = is_state('endpoint.aws.ready')
-    gcp_ready = is_state('endpoint.gcp.ready')
-    openstack_ready = is_state('endpoint.openstack.ready')
-    vsphere_ready = is_state('endpoint.vsphere.ready')
-    if vsphere_ready and get_version('kube-apiserver') < (1, 12):
-        msg = 'vSphere integration requires K8s 1.12 or greater'
-        hookenv.status_set('blocked', msg)
-        return
-
-    cloud_ready = aws_ready or gcp_ready or openstack_ready or vsphere_ready
-    if cloud_req and not cloud_ready:
-        hookenv.status_set('waiting', 'waiting for cloud integration')
-
     if addons_configured and not all_kube_system_pods_running():
         hookenv.status_set('waiting', 'Waiting for kube-system pods to start')
         return
@@ -565,7 +563,9 @@ def master_services_down():
 @when('etcd.available', 'tls_client.server.certificate.saved',
       'authentication.setup')
 @when('leadership.set.auto_storage_backend')
-@when_not('kubernetes-master.components.started')
+@when_not('kubernetes-master.components.started',
+          'kubernetes-master.cloud.pending',
+          'kubernetes-master.cloud.blocked')
 def start_master(etcd):
     '''Run the Kubernetes master components.'''
     hookenv.status_set('maintenance',
@@ -1382,6 +1382,10 @@ def configure_apiserver(etcd_connection_string):
         cloud_config_path = _cloud_config_path('kube-apiserver')
         api_opts['cloud-provider'] = 'vsphere'
         api_opts['cloud-config'] = str(cloud_config_path)
+    elif is_state('endpoint.azure.ready'):
+        cloud_config_path = _cloud_config_path('kube-apiserver')
+        api_opts['cloud-provider'] = 'azure'
+        api_opts['cloud-config'] = str(cloud_config_path)
 
     audit_root = '/root/cdk/audit'
     os.makedirs(audit_root, exist_ok=True)
@@ -1444,6 +1448,10 @@ def configure_controller_manager():
         cloud_config_path = _cloud_config_path('kube-controller-manager')
         controller_opts['cloud-provider'] = 'vsphere'
         controller_opts['cloud-config'] = str(cloud_config_path)
+    elif is_state('endpoint.azure.ready'):
+        cloud_config_path = _cloud_config_path('kube-controller-manager')
+        controller_opts['cloud-provider'] = 'azure'
+        controller_opts['cloud-config'] = str(cloud_config_path)
 
     configure_kubernetes_service('kube-controller-manager', controller_opts,
                                  'controller-manager-extra-args')
@@ -1670,7 +1678,25 @@ def clear_cluster_tag_sent():
 
 
 @when_any('endpoint.aws.joined',
-          'endpoint.gcp.joined')
+          'endpoint.gcp.joined',
+          'endpoint.openstack.joined',
+          'endpoint.vsphere.joined',
+          'endpoint.azure.joined')
+@when_not('kubernetes-master.cloud.ready')
+def set_cloud_pending():
+    k8s_version = get_version('kube-apiserver')
+    k8s_1_11 = k8s_version >= (1, 11)
+    k8s_1_12 = k8s_version >= (1, 12)
+    vsphere_joined = is_state('endpoint.vsphere.joined')
+    azure_joined = is_state('endpoint.azure.joined')
+    if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11):
+        set_state('kubernetes-master.cloud.blocked')
+    set_state('kubernetes-master.cloud.pending')
+
+
+@when_any('endpoint.aws.joined',
+          'endpoint.gcp.joined',
+          'endpoint.azure.joined')
 @when('leadership.set.cluster_tag')
 @when_not('kubernetes-master.cloud-request-sent')
 def request_integration():
@@ -1698,6 +1724,14 @@ def request_integration():
         })
         cloud.enable_object_storage_management()
         cloud.enable_security_management()
+    elif is_state('endpoint.azure.joined'):
+        cloud = endpoint_from_flag('endpoint.azure.joined')
+        cloud.tag_instance({
+            'k8s-io-cluster-name': cluster_tag,
+            'k8s-io-role-master': 'master',
+        })
+        cloud.enable_object_storage_management()
+        cloud.enable_security_management()
     cloud.enable_instance_inspection()
     cloud.enable_network_management()
     cloud.enable_dns_management()
@@ -1706,18 +1740,27 @@ def request_integration():
 
 
 @when_none('endpoint.aws.joined',
-           'endpoint.gcp.joined')
+           'endpoint.gcp.joined',
+           'endpoint.openstack.joined',
+           'endpoint.vsphere.joined',
+           'endpoint.azure.joined')
 @when('kubernetes-master.cloud-request-sent')
 def clear_requested_integration():
+    remove_state('kubernetes-master.cloud.pending')
     remove_state('kubernetes-master.cloud-request-sent')
+    remove_state('kubernetes-master.cloud.blocked')
+    remove_state('kubernetes-master.cloud.ready')
 
 
 @when_any('endpoint.aws.ready',
           'endpoint.gcp.ready',
           'endpoint.openstack.ready',
-          'endpoint.vsphere.ready')
+          'endpoint.vsphere.ready',
-@when_not('kubernetes-master.restarted-for-cloud')
+          'endpoint.azure.ready')
-def restart_for_cloud():
+@when_not('kubernetes-master.cloud.blocked',
+          'kubernetes-master.cloud.ready',
+          'kubernetes-master.restarted-for-cloud')  # compat. TODO: remove
+def cloud_ready():
     if is_state('endpoint.gcp.ready'):
         _write_gcp_snap_config('kube-apiserver')
         _write_gcp_snap_config('kube-controller-manager')
@@ -1727,10 +1770,21 @@ def restart_for_cloud():
     elif is_state('endpoint.vsphere.ready'):
         _write_vsphere_snap_config('kube-apiserver')
         _write_vsphere_snap_config('kube-controller-manager')
-    set_state('kubernetes-master.restarted-for-cloud')
+    elif is_state('endpoint.azure.ready'):
+        _write_azure_snap_config('kube-apiserver')
+        _write_azure_snap_config('kube-controller-manager')
+    remove_state('kubernetes-master.cloud.pending')
+    set_state('kubernetes-master.cloud.ready')
     remove_state('kubernetes-master.components.started')  # force restart
 
 
+@when('kubernetes-master.restarted-for-cloud')
+@when_not('kubernetes-master.cloud.ready')
+def convert_cloud_flag():
+    remove_state('kubernetes-master.restarted-for-cloud')
+    set_state('kubernetes-master.cloud.ready')
+
+
 def _snap_common_path(component):
     return Path('/var/snap/{}/common'.format(component))
 
@@ -1829,3 +1883,14 @@ def _write_vsphere_snap_config(component):
         '[Disk]',
         'scsicontrollertype = "pvscsi"',
     ]))
+
+
+def _write_azure_snap_config(component):
+    azure = endpoint_from_flag('endpoint.azure.ready')
+    cloud_config_path = _cloud_config_path(component)
+    cloud_config_path.write_text(json.dumps({
+        'useInstanceMetadata': True,
+        'useManagedIdentityExtension': True,
+        'resourceGroup': azure.resource_group,
+        'subscriptionId': azure.subscription_id,
+    }))

cluster/juju/layers/kubernetes-worker/layer.yaml

@@ -17,6 +17,7 @@ includes:
   - 'interface:gcp-integration'
   - 'interface:openstack-integration'
   - 'interface:vsphere-integration'
+  - 'interface:azure-integration'
   - 'interface:mount'
 config:
   deletes:

cluster/juju/layers/kubernetes-worker/metadata.yaml

@@ -38,6 +38,8 @@ requires:
     interface: openstack-integration
   vsphere:
     interface: vsphere-integration
+  azure:
+    interface: azure-integration
   nfs:
     interface: mount
 provides:

cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py

@@ -192,13 +192,6 @@ def channel_changed():
     set_upgrade_needed()
 
 
-@when('kubernetes-worker.snaps.upgrade-needed')
-@when_not('kubernetes-worker.snaps.upgrade-specified')
-def upgrade_needed_status():
-    msg = 'Needs manual upgrade, run the upgrade action'
-    hookenv.status_set('blocked', msg)
-
-
 @when('kubernetes-worker.snaps.upgrade-specified')
 def install_snaps():
     channel = hookenv.config('channel')
@@ -324,27 +317,42 @@ def set_snapd_timer():
     snap.set_refresh_timer(timer)
 
 
-@when('kubernetes-worker.snaps.installed')
+@hookenv.atexit
-@when_not('kube-control.dns.available')
+def charm_status():
-def notify_user_transient_status():
-    ''' Notify to the user we are in a transient state and the application
-    is still converging. Potentially remotely, or we may be in a detached loop
-    wait state '''
-
-    # During deployment the worker has to start kubelet without cluster dns
-    # configured. If this is the first unit online in a service pool waiting
-    # to self host the dns pod, and configure itself to query the dns service
-    # declared in the kube-system namespace
-
-    hookenv.status_set('waiting', 'Waiting for cluster DNS.')
-
-
-@when('kubernetes-worker.snaps.installed',
-      'kube-control.dns.available')
-@when_not('kubernetes-worker.snaps.upgrade-needed')
-def charm_status(kube_control):
     '''Update the status message with the current status of kubelet.'''
+    vsphere_joined = is_state('endpoint.vsphere.joined')
+    azure_joined = is_state('endpoint.azure.joined')
+    cloud_blocked = is_state('kubernetes-worker.cloud.blocked')
+    if vsphere_joined and cloud_blocked:
+        hookenv.status_set('blocked',
+                           'vSphere integration requires K8s 1.12 or greater')
+        return
+    if azure_joined and cloud_blocked:
+        hookenv.status_set('blocked',
+                           'Azure integration requires K8s 1.11 or greater')
+        return
+    if is_state('kubernetes-worker.cloud.pending'):
+        hookenv.status_set('waiting', 'Waiting for cloud integration')
+        return
+    if not is_state('kube-control.dns.available'):
+        # During deployment the worker has to start kubelet without cluster dns
+        # configured. If this is the first unit online in a service pool
+        # waiting to self host the dns pod, and configure itself to query the
+        # dns service declared in the kube-system namespace
+        hookenv.status_set('waiting', 'Waiting for cluster DNS.')
+        return
+    if is_state('kubernetes-worker.snaps.upgrade-specified'):
+        hookenv.status_set('waiting', 'Upgrade pending')
+        return
+    if is_state('kubernetes-worker.snaps.upgrade-needed'):
+        hookenv.status_set('blocked',
+                           'Needs manual upgrade, run the upgrade action')
+        return
+    if is_state('kubernetes-worker.snaps.installed'):
         update_kubelet_status()
+        return
+    else:
+        pass  # will have been set by snap layer or other handler
 
 
 def update_kubelet_status():
@@ -429,6 +437,8 @@ def watch_for_changes(kube_api, kube_control, cni):
       'kube-control.dns.available', 'kube-control.auth.available',
       'cni.available', 'kubernetes-worker.restart-needed',
       'worker.auth.bootstrapped')
+@when_not('kubernetes-worker.cloud.pending',
+          'kubernetes-worker.cloud.blocked')
 def start_worker(kube_api, kube_control, auth_control, cni):
     ''' Start kubelet using the provided API and DNS info.'''
     servers = get_kube_api_servers(kube_api)
@@ -723,6 +733,12 @@ def configure_kubelet(dns, ingress_ip):
         with open(uuid_file, 'r') as f:
             uuid = f.read().strip()
         kubelet_opts['provider-id'] = 'vsphere://{}'.format(uuid)
+    elif is_state('endpoint.azure.ready'):
+        azure = endpoint_from_flag('endpoint.azure.ready')
+        cloud_config_path = _cloud_config_path('kubelet')
+        kubelet_opts['cloud-provider'] = 'azure'
+        kubelet_opts['cloud-config'] = str(cloud_config_path)
+        kubelet_opts['provider-id'] = azure.vm_id
 
     if get_version('kubelet') >= (1, 10):
         # Put together the KubeletConfiguration data
@@ -1204,6 +1220,8 @@ def get_node_name():
         cloud_provider = 'openstack'
     elif is_state('endpoint.vsphere.ready'):
         cloud_provider = 'vsphere'
+    elif is_state('endpoint.azure.ready'):
+        cloud_provider = 'azure'
     if cloud_provider == 'aws':
         return getfqdn().lower()
     else:
@@ -1247,7 +1265,25 @@ def remove_label(label):
 
 
 @when_any('endpoint.aws.joined',
-          'endpoint.gcp.joined')
+          'endpoint.gcp.joined',
+          'endpoint.openstack.joined',
+          'endpoint.vsphere.joined',
+          'endpoint.azure.joined')
+@when_not('kubernetes-worker.cloud.ready')
+def set_cloud_pending():
+    k8s_version = get_version('kubelet')
+    k8s_1_11 = k8s_version >= (1, 11)
+    k8s_1_12 = k8s_version >= (1, 12)
+    vsphere_joined = is_state('endpoint.vsphere.joined')
+    azure_joined = is_state('endpoint.azure.joined')
+    if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11):
+        set_state('kubernetes-worker.cloud.blocked')
+    set_state('kubernetes-worker.cloud.pending')
+
+
+@when_any('endpoint.aws.joined',
+          'endpoint.gcp.joined',
+          'endpoint.azure.joined')
 @when('kube-control.cluster_tag.available')
 @when_not('kubernetes-worker.cloud-request-sent')
 def request_integration():
@@ -1272,29 +1308,55 @@ def request_integration():
             'k8s-io-cluster-name': cluster_tag,
         })
         cloud.enable_object_storage_management()
+    elif is_state('endpoint.azure.joined'):
+        cloud = endpoint_from_flag('endpoint.azure.joined')
+        cloud.tag_instance({
+            'k8s-io-cluster-name': cluster_tag,
+        })
+        cloud.enable_object_storage_management()
     cloud.enable_instance_inspection()
     cloud.enable_dns_management()
     set_state('kubernetes-worker.cloud-request-sent')
-    hookenv.status_set('waiting', 'waiting for cloud integration')
+    hookenv.status_set('waiting', 'Waiting for cloud integration')
 
 
 @when_none('endpoint.aws.joined',
-           'endpoint.gcp.joined')
+           'endpoint.gcp.joined',
-def clear_requested_integration():
+           'endpoint.openstack.joined',
+           'endpoint.vsphere.joined',
+           'endpoint.azure.joined')
+def clear_cloud_flags():
+    remove_state('kubernetes-worker.cloud.pending')
     remove_state('kubernetes-worker.cloud-request-sent')
+    remove_state('kubernetes-worker.cloud.blocked')
+    remove_state('kubernetes-worker.cloud.ready')
 
 
 @when_any('endpoint.aws.ready',
           'endpoint.gcp.ready',
-          'endpoint.openstack.ready')
+          'endpoint.openstack.ready',
-@when_not('kubernetes-worker.restarted-for-cloud')
+          'endpoint.vsphere.ready',
-def restart_for_cloud():
+          'endpoint.azure.ready')
+@when_not('kubernetes-worker.cloud.blocked',
+          'kubernetes-worker.cloud.ready',
+          'kubernetes-worker.restarted-for-cloud')  # compat. TODO: remove
+def cloud_ready():
+    remove_state('kubernetes-worker.cloud.pending')
     if is_state('endpoint.gcp.ready'):
         _write_gcp_snap_config('kubelet')
     elif is_state('endpoint.openstack.ready'):
         _write_openstack_snap_config('kubelet')
-    set_state('kubernetes-worker.restarted-for-cloud')
+    elif is_state('endpoint.azure.ready'):
-    set_state('kubernetes-worker.restart-needed')
+        _write_azure_snap_config('kubelet')
+    set_state('kubernetes-worker.cloud.ready')
+    set_state('kubernetes-worker.restart-needed')  # force restart
+
+
+@when('kubernetes-master.restarted-for-cloud')
+@when_not('kubernetes-master.cloud.ready')
+def convert_cloud_flag():
+    remove_state('kubernetes-worker.restarted-for-cloud')
+    set_state('kubernetes-worker.cloud.ready')
 
 
 def _snap_common_path(component):
@@ -1357,6 +1419,17 @@ def _write_openstack_snap_config(component):
     ]))
 
 
+def _write_azure_snap_config(component):
+    azure = endpoint_from_flag('endpoint.azure.ready')
+    cloud_config_path = _cloud_config_path(component)
+    cloud_config_path.write_text(json.dumps({
+        'useInstanceMetadata': True,
+        'useManagedIdentityExtension': True,
+        'resourceGroup': azure.resource_group,
+        'subscriptionId': azure.subscription_id,
+    }))
+
+
 def get_first_mount(mount_relation):
     mount_relation_list = mount_relation.mounts()
     if mount_relation_list and len(mount_relation_list) > 0: