diff --git a/cluster/juju/layers/kubernetes-master/layer.yaml b/cluster/juju/layers/kubernetes-master/layer.yaml index b8f5dfc0f18..b1a1ed22c1a 100644 --- a/cluster/juju/layers/kubernetes-master/layer.yaml +++ b/cluster/juju/layers/kubernetes-master/layer.yaml @@ -19,6 +19,7 @@ includes: - 'interface:gcp-integration' - 'interface:openstack-integration' - 'interface:vsphere-integration' + - 'interface:azure-integration' options: basic: packages: diff --git a/cluster/juju/layers/kubernetes-master/metadata.yaml b/cluster/juju/layers/kubernetes-master/metadata.yaml index 88759653c7d..03746ad95a2 100644 --- a/cluster/juju/layers/kubernetes-master/metadata.yaml +++ b/cluster/juju/layers/kubernetes-master/metadata.yaml @@ -49,6 +49,8 @@ requires: interface: openstack-integration vsphere: interface: vsphere-integration + azure: + interface: azure-integration resources: kubectl: type: file diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index a658a3479eb..1ef69367897 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -467,6 +467,22 @@ def set_final_status(): except NotImplementedError: goal_state = {} + vsphere_joined = is_state('endpoint.vsphere.joined') + azure_joined = is_state('endpoint.azure.joined') + cloud_blocked = is_state('kubernetes-master.cloud.blocked') + if vsphere_joined and cloud_blocked: + hookenv.status_set('blocked', + 'vSphere integration requires K8s 1.12 or greater') + return + if azure_joined and cloud_blocked: + hookenv.status_set('blocked', + 'Azure integration requires K8s 1.11 or greater') + return + + if is_state('kubernetes-master.cloud.pending'): + hookenv.status_set('waiting', 'Waiting for cloud integration') + return + if not is_state('kube-api-endpoint.available'): if 'kube-api-endpoint' in goal_state.get('relations', {}): status = 'waiting' @@ -510,24 +526,6 @@ def set_final_status(): hookenv.status_set('waiting', 'Waiting to retry addon deployment') return - req_sent = is_state('kubernetes-master.cloud-request-sent') - # openstack and vsphere have admin perms; cloud req is not required - openstack_joined = is_state('endpoint.openstack.joined') - vsphere_joined = is_state('endpoint.vsphere.joined') - cloud_req = req_sent or openstack_joined or vsphere_joined - aws_ready = is_state('endpoint.aws.ready') - gcp_ready = is_state('endpoint.gcp.ready') - openstack_ready = is_state('endpoint.openstack.ready') - vsphere_ready = is_state('endpoint.vsphere.ready') - if vsphere_ready and get_version('kube-apiserver') < (1, 12): - msg = 'vSphere integration requires K8s 1.12 or greater' - hookenv.status_set('blocked', msg) - return - - cloud_ready = aws_ready or gcp_ready or openstack_ready or vsphere_ready - if cloud_req and not cloud_ready: - hookenv.status_set('waiting', 'waiting for cloud integration') - if addons_configured and not all_kube_system_pods_running(): hookenv.status_set('waiting', 'Waiting for kube-system pods to start') return @@ -565,7 +563,9 @@ def master_services_down(): @when('etcd.available', 'tls_client.server.certificate.saved', 'authentication.setup') @when('leadership.set.auto_storage_backend') -@when_not('kubernetes-master.components.started') +@when_not('kubernetes-master.components.started', + 'kubernetes-master.cloud.pending', + 'kubernetes-master.cloud.blocked') def start_master(etcd): '''Run the Kubernetes master components.''' hookenv.status_set('maintenance', @@ -1382,6 +1382,10 @@ def configure_apiserver(etcd_connection_string): cloud_config_path = _cloud_config_path('kube-apiserver') api_opts['cloud-provider'] = 'vsphere' api_opts['cloud-config'] = str(cloud_config_path) + elif is_state('endpoint.azure.ready'): + cloud_config_path = _cloud_config_path('kube-apiserver') + api_opts['cloud-provider'] = 'azure' + api_opts['cloud-config'] = str(cloud_config_path) audit_root = '/root/cdk/audit' os.makedirs(audit_root, exist_ok=True) @@ -1444,6 +1448,10 @@ def configure_controller_manager(): cloud_config_path = _cloud_config_path('kube-controller-manager') controller_opts['cloud-provider'] = 'vsphere' controller_opts['cloud-config'] = str(cloud_config_path) + elif is_state('endpoint.azure.ready'): + cloud_config_path = _cloud_config_path('kube-controller-manager') + controller_opts['cloud-provider'] = 'azure' + controller_opts['cloud-config'] = str(cloud_config_path) configure_kubernetes_service('kube-controller-manager', controller_opts, 'controller-manager-extra-args') @@ -1670,7 +1678,25 @@ def clear_cluster_tag_sent(): @when_any('endpoint.aws.joined', - 'endpoint.gcp.joined') + 'endpoint.gcp.joined', + 'endpoint.openstack.joined', + 'endpoint.vsphere.joined', + 'endpoint.azure.joined') +@when_not('kubernetes-master.cloud.ready') +def set_cloud_pending(): + k8s_version = get_version('kube-apiserver') + k8s_1_11 = k8s_version >= (1, 11) + k8s_1_12 = k8s_version >= (1, 12) + vsphere_joined = is_state('endpoint.vsphere.joined') + azure_joined = is_state('endpoint.azure.joined') + if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11): + set_state('kubernetes-master.cloud.blocked') + set_state('kubernetes-master.cloud.pending') + + +@when_any('endpoint.aws.joined', + 'endpoint.gcp.joined', + 'endpoint.azure.joined') @when('leadership.set.cluster_tag') @when_not('kubernetes-master.cloud-request-sent') def request_integration(): @@ -1698,6 +1724,14 @@ def request_integration(): }) cloud.enable_object_storage_management() cloud.enable_security_management() + elif is_state('endpoint.azure.joined'): + cloud = endpoint_from_flag('endpoint.azure.joined') + cloud.tag_instance({ + 'k8s-io-cluster-name': cluster_tag, + 'k8s-io-role-master': 'master', + }) + cloud.enable_object_storage_management() + cloud.enable_security_management() cloud.enable_instance_inspection() cloud.enable_network_management() cloud.enable_dns_management() @@ -1706,18 +1740,27 @@ def request_integration(): @when_none('endpoint.aws.joined', - 'endpoint.gcp.joined') + 'endpoint.gcp.joined', + 'endpoint.openstack.joined', + 'endpoint.vsphere.joined', + 'endpoint.azure.joined') @when('kubernetes-master.cloud-request-sent') def clear_requested_integration(): + remove_state('kubernetes-master.cloud.pending') remove_state('kubernetes-master.cloud-request-sent') + remove_state('kubernetes-master.cloud.blocked') + remove_state('kubernetes-master.cloud.ready') @when_any('endpoint.aws.ready', 'endpoint.gcp.ready', 'endpoint.openstack.ready', - 'endpoint.vsphere.ready') -@when_not('kubernetes-master.restarted-for-cloud') -def restart_for_cloud(): + 'endpoint.vsphere.ready', + 'endpoint.azure.ready') +@when_not('kubernetes-master.cloud.blocked', + 'kubernetes-master.cloud.ready', + 'kubernetes-master.restarted-for-cloud') # compat. TODO: remove +def cloud_ready(): if is_state('endpoint.gcp.ready'): _write_gcp_snap_config('kube-apiserver') _write_gcp_snap_config('kube-controller-manager') @@ -1727,10 +1770,21 @@ def restart_for_cloud(): elif is_state('endpoint.vsphere.ready'): _write_vsphere_snap_config('kube-apiserver') _write_vsphere_snap_config('kube-controller-manager') - set_state('kubernetes-master.restarted-for-cloud') + elif is_state('endpoint.azure.ready'): + _write_azure_snap_config('kube-apiserver') + _write_azure_snap_config('kube-controller-manager') + remove_state('kubernetes-master.cloud.pending') + set_state('kubernetes-master.cloud.ready') remove_state('kubernetes-master.components.started') # force restart +@when('kubernetes-master.restarted-for-cloud') +@when_not('kubernetes-master.cloud.ready') +def convert_cloud_flag(): + remove_state('kubernetes-master.restarted-for-cloud') + set_state('kubernetes-master.cloud.ready') + + def _snap_common_path(component): return Path('/var/snap/{}/common'.format(component)) @@ -1829,3 +1883,14 @@ def _write_vsphere_snap_config(component): '[Disk]', 'scsicontrollertype = "pvscsi"', ])) + + +def _write_azure_snap_config(component): + azure = endpoint_from_flag('endpoint.azure.ready') + cloud_config_path = _cloud_config_path(component) + cloud_config_path.write_text(json.dumps({ + 'useInstanceMetadata': True, + 'useManagedIdentityExtension': True, + 'resourceGroup': azure.resource_group, + 'subscriptionId': azure.subscription_id, + })) diff --git a/cluster/juju/layers/kubernetes-worker/layer.yaml b/cluster/juju/layers/kubernetes-worker/layer.yaml index b3b47c7d869..e7958853ae5 100644 --- a/cluster/juju/layers/kubernetes-worker/layer.yaml +++ b/cluster/juju/layers/kubernetes-worker/layer.yaml @@ -17,6 +17,7 @@ includes: - 'interface:gcp-integration' - 'interface:openstack-integration' - 'interface:vsphere-integration' + - 'interface:azure-integration' - 'interface:mount' config: deletes: diff --git a/cluster/juju/layers/kubernetes-worker/metadata.yaml b/cluster/juju/layers/kubernetes-worker/metadata.yaml index bb19271cc9f..d4040fb449b 100644 --- a/cluster/juju/layers/kubernetes-worker/metadata.yaml +++ b/cluster/juju/layers/kubernetes-worker/metadata.yaml @@ -38,6 +38,8 @@ requires: interface: openstack-integration vsphere: interface: vsphere-integration + azure: + interface: azure-integration nfs: interface: mount provides: diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index 4d285328fc9..e00ef541c85 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -192,13 +192,6 @@ def channel_changed(): set_upgrade_needed() -@when('kubernetes-worker.snaps.upgrade-needed') -@when_not('kubernetes-worker.snaps.upgrade-specified') -def upgrade_needed_status(): - msg = 'Needs manual upgrade, run the upgrade action' - hookenv.status_set('blocked', msg) - - @when('kubernetes-worker.snaps.upgrade-specified') def install_snaps(): channel = hookenv.config('channel') @@ -324,27 +317,42 @@ def set_snapd_timer(): snap.set_refresh_timer(timer) -@when('kubernetes-worker.snaps.installed') -@when_not('kube-control.dns.available') -def notify_user_transient_status(): - ''' Notify to the user we are in a transient state and the application - is still converging. Potentially remotely, or we may be in a detached loop - wait state ''' - - # During deployment the worker has to start kubelet without cluster dns - # configured. If this is the first unit online in a service pool waiting - # to self host the dns pod, and configure itself to query the dns service - # declared in the kube-system namespace - - hookenv.status_set('waiting', 'Waiting for cluster DNS.') - - -@when('kubernetes-worker.snaps.installed', - 'kube-control.dns.available') -@when_not('kubernetes-worker.snaps.upgrade-needed') -def charm_status(kube_control): +@hookenv.atexit +def charm_status(): '''Update the status message with the current status of kubelet.''' - update_kubelet_status() + vsphere_joined = is_state('endpoint.vsphere.joined') + azure_joined = is_state('endpoint.azure.joined') + cloud_blocked = is_state('kubernetes-worker.cloud.blocked') + if vsphere_joined and cloud_blocked: + hookenv.status_set('blocked', + 'vSphere integration requires K8s 1.12 or greater') + return + if azure_joined and cloud_blocked: + hookenv.status_set('blocked', + 'Azure integration requires K8s 1.11 or greater') + return + if is_state('kubernetes-worker.cloud.pending'): + hookenv.status_set('waiting', 'Waiting for cloud integration') + return + if not is_state('kube-control.dns.available'): + # During deployment the worker has to start kubelet without cluster dns + # configured. If this is the first unit online in a service pool + # waiting to self host the dns pod, and configure itself to query the + # dns service declared in the kube-system namespace + hookenv.status_set('waiting', 'Waiting for cluster DNS.') + return + if is_state('kubernetes-worker.snaps.upgrade-specified'): + hookenv.status_set('waiting', 'Upgrade pending') + return + if is_state('kubernetes-worker.snaps.upgrade-needed'): + hookenv.status_set('blocked', + 'Needs manual upgrade, run the upgrade action') + return + if is_state('kubernetes-worker.snaps.installed'): + update_kubelet_status() + return + else: + pass # will have been set by snap layer or other handler def update_kubelet_status(): @@ -429,6 +437,8 @@ def watch_for_changes(kube_api, kube_control, cni): 'kube-control.dns.available', 'kube-control.auth.available', 'cni.available', 'kubernetes-worker.restart-needed', 'worker.auth.bootstrapped') +@when_not('kubernetes-worker.cloud.pending', + 'kubernetes-worker.cloud.blocked') def start_worker(kube_api, kube_control, auth_control, cni): ''' Start kubelet using the provided API and DNS info.''' servers = get_kube_api_servers(kube_api) @@ -723,6 +733,12 @@ def configure_kubelet(dns, ingress_ip): with open(uuid_file, 'r') as f: uuid = f.read().strip() kubelet_opts['provider-id'] = 'vsphere://{}'.format(uuid) + elif is_state('endpoint.azure.ready'): + azure = endpoint_from_flag('endpoint.azure.ready') + cloud_config_path = _cloud_config_path('kubelet') + kubelet_opts['cloud-provider'] = 'azure' + kubelet_opts['cloud-config'] = str(cloud_config_path) + kubelet_opts['provider-id'] = azure.vm_id if get_version('kubelet') >= (1, 10): # Put together the KubeletConfiguration data @@ -892,10 +908,10 @@ def launch_default_ingress_controller(): 'ingress-ssl-chain-completion') context['ingress_image'] = config.get('nginx-image') if context['ingress_image'] == "" or context['ingress_image'] == "auto": - images = {'amd64': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1', # noqa - 'arm64': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.16.1', # noqa - 's390x': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-s390x:0.16.1', # noqa - 'ppc64el': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-ppc64le:0.16.1', # noqa + images = {'amd64': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1', # noqa + 'arm64': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.16.1', # noqa + 's390x': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-s390x:0.16.1', # noqa + 'ppc64el': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-ppc64le:0.16.1', # noqa } context['ingress_image'] = images.get(context['arch'], images['amd64']) if get_version('kubelet') < (1, 9): @@ -1204,6 +1220,8 @@ def get_node_name(): cloud_provider = 'openstack' elif is_state('endpoint.vsphere.ready'): cloud_provider = 'vsphere' + elif is_state('endpoint.azure.ready'): + cloud_provider = 'azure' if cloud_provider == 'aws': return getfqdn().lower() else: @@ -1247,7 +1265,25 @@ def remove_label(label): @when_any('endpoint.aws.joined', - 'endpoint.gcp.joined') + 'endpoint.gcp.joined', + 'endpoint.openstack.joined', + 'endpoint.vsphere.joined', + 'endpoint.azure.joined') +@when_not('kubernetes-worker.cloud.ready') +def set_cloud_pending(): + k8s_version = get_version('kubelet') + k8s_1_11 = k8s_version >= (1, 11) + k8s_1_12 = k8s_version >= (1, 12) + vsphere_joined = is_state('endpoint.vsphere.joined') + azure_joined = is_state('endpoint.azure.joined') + if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11): + set_state('kubernetes-worker.cloud.blocked') + set_state('kubernetes-worker.cloud.pending') + + +@when_any('endpoint.aws.joined', + 'endpoint.gcp.joined', + 'endpoint.azure.joined') @when('kube-control.cluster_tag.available') @when_not('kubernetes-worker.cloud-request-sent') def request_integration(): @@ -1272,29 +1308,55 @@ def request_integration(): 'k8s-io-cluster-name': cluster_tag, }) cloud.enable_object_storage_management() + elif is_state('endpoint.azure.joined'): + cloud = endpoint_from_flag('endpoint.azure.joined') + cloud.tag_instance({ + 'k8s-io-cluster-name': cluster_tag, + }) + cloud.enable_object_storage_management() cloud.enable_instance_inspection() cloud.enable_dns_management() set_state('kubernetes-worker.cloud-request-sent') - hookenv.status_set('waiting', 'waiting for cloud integration') + hookenv.status_set('waiting', 'Waiting for cloud integration') @when_none('endpoint.aws.joined', - 'endpoint.gcp.joined') -def clear_requested_integration(): + 'endpoint.gcp.joined', + 'endpoint.openstack.joined', + 'endpoint.vsphere.joined', + 'endpoint.azure.joined') +def clear_cloud_flags(): + remove_state('kubernetes-worker.cloud.pending') remove_state('kubernetes-worker.cloud-request-sent') + remove_state('kubernetes-worker.cloud.blocked') + remove_state('kubernetes-worker.cloud.ready') @when_any('endpoint.aws.ready', 'endpoint.gcp.ready', - 'endpoint.openstack.ready') -@when_not('kubernetes-worker.restarted-for-cloud') -def restart_for_cloud(): + 'endpoint.openstack.ready', + 'endpoint.vsphere.ready', + 'endpoint.azure.ready') +@when_not('kubernetes-worker.cloud.blocked', + 'kubernetes-worker.cloud.ready', + 'kubernetes-worker.restarted-for-cloud') # compat. TODO: remove +def cloud_ready(): + remove_state('kubernetes-worker.cloud.pending') if is_state('endpoint.gcp.ready'): _write_gcp_snap_config('kubelet') elif is_state('endpoint.openstack.ready'): _write_openstack_snap_config('kubelet') - set_state('kubernetes-worker.restarted-for-cloud') - set_state('kubernetes-worker.restart-needed') + elif is_state('endpoint.azure.ready'): + _write_azure_snap_config('kubelet') + set_state('kubernetes-worker.cloud.ready') + set_state('kubernetes-worker.restart-needed') # force restart + + +@when('kubernetes-master.restarted-for-cloud') +@when_not('kubernetes-master.cloud.ready') +def convert_cloud_flag(): + remove_state('kubernetes-worker.restarted-for-cloud') + set_state('kubernetes-worker.cloud.ready') def _snap_common_path(component): @@ -1357,6 +1419,17 @@ def _write_openstack_snap_config(component): ])) +def _write_azure_snap_config(component): + azure = endpoint_from_flag('endpoint.azure.ready') + cloud_config_path = _cloud_config_path(component) + cloud_config_path.write_text(json.dumps({ + 'useInstanceMetadata': True, + 'useManagedIdentityExtension': True, + 'resourceGroup': azure.resource_group, + 'subscriptionId': azure.subscription_id, + })) + + def get_first_mount(mount_relation): mount_relation_list = mount_relation.mounts() if mount_relation_list and len(mount_relation_list) > 0: