RBAC work on PoC

cluster/juju/layers/kubernetes-master/config.yaml

@@ -40,3 +40,7 @@ options:
         runtime-config=batch/v2alpha1=true profiling=true
       will result in kube-apiserver being run with the following options:
         --runtime-config=batch/v2alpha1=true --profiling=true
+  enable-rbac:
+    type: boolean
+    default: True
+    description: Enable RBAC authorization mode.

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -209,12 +209,10 @@ def setup_leader_authentication():
     if not get_keys_from_leader(keys) \
             or is_state('reconfigure.authentication.setup'):
         last_pass = get_password('basic_auth.csv', 'admin')
-        setup_basic_auth(last_pass, 'admin', 'admin')
+        setup_basic_auth(last_pass, 'admin', 'admin', 'system:masters')
 
         if not os.path.isfile(known_tokens):
-            setup_tokens(None, 'admin', 'admin')
+            touch(known_tokens)
-            setup_tokens(None, 'kubelet', 'kubelet')
-            setup_tokens(None, 'kube_proxy', 'kube_proxy')
 
         # Generate the default service account token key
         os.makedirs('/root/cdk', exist_ok=True)
@@ -400,19 +398,63 @@ def send_cluster_dns_detail(kube_control):
 
 
 @when('kube-control.auth.requested')
-@when('authentication.setup')
+@when('snap.installed.kubectl')
 @when('leadership.is_leader')
-def send_tokens(kube_control):
+def create_service_configs(kube_control):
-    """Send the tokens to the workers."""
+    """Create the users for kubelet"""
-    kubelet_token = get_token('kubelet')
+    # generate the username/pass for the requesting unit
-    proxy_token = get_token('kube_proxy')
+    proxy_token = get_token('system:kube-proxy')
-    admin_token = get_token('admin')
+    if not proxy_token:
+        setup_tokens(None, 'system:kube-proxy', 'kube-proxy')
+        proxy_token = get_token('system:kube-proxy')
+
+    client_token = get_token('admin')
+    if not client_token:
+        setup_tokens(None, 'admin', 'admin', "system:masters")
+        client_token = get_token('admin')
 
-    # Send the data
     requests = kube_control.auth_user()
     for request in requests:
-        kube_control.sign_auth_request(request[0], kubelet_token,
+        username = request[1]['user']
-                                       proxy_token, admin_token)
+        group = request[1]['group']
+        kubelet_token = get_token(username)
+        if not kubelet_token:
+            # Usernames have to be in the form of system:node:<hostname>
+            userid = "kubelet-{}".format(request[0].split('/')[1])
+            setup_tokens(None, username, userid, group)
+            kubelet_token = get_token(username)
+
+        kube_control.sign_auth_request(request[0], username,
+                                       kubelet_token, proxy_token, client_token)
+
+    host.service_restart('snap.kube-apiserver.daemon')
+    remove_state('authentication.setup')
+
+
+@when('kube-control.departed')
+@when('leadership.is_leader')
+def flush_auth_for_departed(kube_control):
+    ''' Unit has left the cluster and needs to have its authentication
+    tokens removed from the token registry '''
+    token_auth_file = '/root/cdk/known_tokens.csv'
+    departing_unit = kube_control.flush_departed()
+    userid = "kubelet-{}".format(departing_unit.split('/')[1])
+    known_tokens = open(token_auth_file, 'r').readlines()
+    for line in known_tokens[:]:
+        haystack = line.split(',')
+        # skip the entry if we dont have token,user,id,groups format
+        if len(haystack) < 4:
+            continue
+        if haystack[2] == userid:
+            hookenv.log('Found unit {} in token auth. Removing auth'
+                        ' token.'.format(userid))
+            known_tokens.remove(line)
+    # atomically rewrite the file minus any scrubbed units
+    hookenv.log('Rewriting token auth file: {}'.format(token_auth_file))
+    with open(token_auth_file, 'w') as fp:
+        fp.writelines(known_tokens)
+    # Trigger rebroadcast of auth files for followers
+    remove_state('autentication.setup')
 
 
 @when_not('kube-control.connected')
@@ -492,6 +534,7 @@ def addons_ready():
 
     """
     try:
+        apply_rbac()
         check_call(['cdk-addons.apply'])
         return True
     except CalledProcessError:
@@ -614,6 +657,52 @@ def initial_nrpe_config(nagios=None):
     update_nrpe_config(nagios)
 
 
+@when('config.changed.enable-rbac',
+      'kubernetes-master.components.started')
+def enable_rbac_config():
+    config = hookenv.config()
+    if data_changed('rbac-flag', str(config.get('enable-rbac'))):
+        remove_state('kubernetes-master.components.started')
+
+
+def apply_rbac():
+    # TODO(kjackal): we should be checking if rbac is already applied
+    config = hookenv.config()
+    if is_state('leadership.is_leader'):
+        if config.get('enable-rbac'):
+            try:
+                cmd = ['kubectl', 'apply', '-f', 'templates/heapster-rbac.yaml']
+                check_output(cmd).decode('utf-8')
+            except CalledProcessError:
+                hookenv.log('Failed to apply heapster rbac rules')
+            try:
+                cmd = ['kubectl', 'apply', '-f', 'templates/nginx-ingress-controller-rbac.yml']
+                check_output(cmd).decode('utf-8')
+            except CalledProcessError:
+                hookenv.log('Failed to apply heapster rbac rules')
+
+            # TODO(kjackal): The follwoing is wrong and imposes security risk. What we should be doing is
+            # update the add-ons to include an rbac enabled dashboard
+            try:
+                cmd = "kubectl create clusterrolebinding add-on-cluster-admin  --clusterrole=cluster-admin" \
+                      " --serviceaccount=kube-system:default".split(' ')
+                check_output(cmd).decode('utf-8')
+            except CalledProcessError:
+                hookenv.log('Failed to elevate credentials')
+
+        else:
+            try:
+                cmd = ['kubectl', 'delete', '-f', 'templates/heapster-rbac.yaml']
+                check_output(cmd).decode('utf-8')
+            except CalledProcessError:
+                hookenv.log('Failed to delete heapster rbac rules')
+            try:
+                cmd = ['kubectl', 'delete', '-f', 'templates/nginx-ingress-controller-rbac.yml']
+                check_output(cmd).decode('utf-8')
+            except CalledProcessError:
+                hookenv.log('Failed to apply heapster rbac rules')
+
+
 @when('kubernetes-master.components.started')
 @when('nrpe-external-master.available')
 @when_any('config.changed.nagios_context',
@@ -965,6 +1054,12 @@ def configure_apiserver():
         'DefaultTolerationSeconds'
     ]
 
+    if hookenv.config('enable-rbac'):
+        admission_control.append('NodeRestriction')
+        api_opts.add('authorization-mode', 'Node,RBAC', strict=True)
+    else:
+        api_opts.add('authorization-mode', 'AlwaysAllow', strict=True)
+
     if get_version('kube-apiserver') < (1, 6):
         hookenv.log('Removing DefaultTolerationSeconds from admission-control')
         admission_control.remove('DefaultTolerationSeconds')
@@ -1020,7 +1115,8 @@ def configure_scheduler():
     set_state('kube-scheduler.do-restart')
 
 
-def setup_basic_auth(password=None, username='admin', uid='admin'):
+def setup_basic_auth(password=None, username='admin', uid='admin',
+                     groups=None):
     '''Create the htacces file and the tokens.'''
     root_cdk = '/root/cdk'
     if not os.path.isdir(root_cdk):
@@ -1029,10 +1125,14 @@ def setup_basic_auth(password=None, username='admin', uid='admin'):
     if not password:
         password = token_generator()
     with open(htaccess, 'w') as stream:
+        if groups:
+            stream.write('{0},{1},{2},"{3}"'.format(password,
+                                                    username, uid, groups))
+        else:
             stream.write('{0},{1},{2}'.format(password, username, uid))
 
 
-def setup_tokens(token, username, user):
+def setup_tokens(token, username, user, groups=None):
     '''Create a token file for kubernetes authentication.'''
     root_cdk = '/root/cdk'
     if not os.path.isdir(root_cdk):
@@ -1041,6 +1141,10 @@ def setup_tokens(token, username, user):
     if not token:
         token = token_generator()
     with open(known_tokens, 'a') as stream:
+        if groups:
+            stream.write('{0},{1},{2},"{3}"\n'.format(token,
+                                                    username, user, groups))
+        else:
             stream.write('{0},{1},{2}\n'.format(token, username, user))
 
 
@@ -1107,3 +1211,10 @@ def apiserverVersion():
     cmd = 'kube-apiserver --version'.split()
     version_string = check_output(cmd).decode('utf-8')
     return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3])
+
+
+def touch(fname):
+    try:
+        os.utime(fname, None)
+    except OSError:
+        open(fname, 'a').close()

cluster/juju/layers/kubernetes-master/templates/heapster-rbac.yaml

@@ -0,0 +1,58 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: heapster-binding
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:heapster
+subjects:
+- kind: ServiceAccount
+  name: heapster
+  namespace: kube-system
+---
+# Heapster's pod_nanny monitors the heapster deployment & its pod(s), and scales
+# the resources of the deployment if necessary.
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: system:pod-nanny
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+- apiGroups:
+  - "extensions"
+  resources:
+  - deployments
+  verbs:
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: heapster-binding
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: system:pod-nanny
+subjects:
+- kind: ServiceAccount
+  name: heapster
+  namespace: kube-system
+---

cluster/juju/layers/kubernetes-master/templates/nginx-ingress-controller-rbac.yml

@@ -0,0 +1,127 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nginx-ingress-serviceaccount
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: nginx-ingress-clusterrole
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - endpoints
+      - nodes
+      - pods
+      - secrets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "extensions"
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+        - events
+    verbs:
+        - create
+        - patch
+  - apiGroups:
+      - "extensions"
+    resources:
+      - ingresses/status
+    verbs:
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: nginx-ingress-role
+  namespace: kube-system
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - pods
+      - secrets
+      - namespaces
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    resourceNames:
+      # Defaults to "<election-id>-<ingress-class>"
+      # Here: "<ingress-controller-leader>-<nginx>"
+      # This has to be adapted if you change either parameter
+      # when launching the nginx-ingress-controller.
+      - "ingress-controller-leader-nginx"
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - get
+      - create
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: nginx-ingress-role-nisa-binding
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: nginx-ingress-role
+subjects:
+  - kind: ServiceAccount
+    name: nginx-ingress-serviceaccount
+    namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: nginx-ingress-clusterrole-nisa-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nginx-ingress-clusterrole
+subjects:
+  - kind: ServiceAccount
+    name: nginx-ingress-serviceaccount
+    namespace: kube-system

cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py

@@ -47,6 +47,8 @@ from charmhelpers.contrib.charmsupport import nrpe
 nrpe.Check.shortname_re = '[\.A-Za-z0-9-_]+$'
 
 kubeconfig_path = '/root/cdk/kubeconfig'
+kubeproxyconfig_path = '/root/cdk/kubeproxyconfig'
+kubeclientconfig_path = '/root/.kube/config'
 
 os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin')
 
@@ -319,7 +321,8 @@ def watch_for_changes(kube_api, kube_control, cni):
       'tls_client.client.key.saved', 'tls_client.server.certificate.saved',
       'tls_client.server.key.saved',
       'kube-control.dns.available', 'kube-control.auth.available',
-      'cni.available', 'kubernetes-worker.restart-needed')
+      'cni.available', 'kubernetes-worker.restart-needed',
+      'worker.auth.bootstrapped')
 def start_worker(kube_api, kube_control, auth_control, cni):
     ''' Start kubelet using the provided API and DNS info.'''
     servers = get_kube_api_servers(kube_api)
@@ -335,7 +338,7 @@ def start_worker(kube_api, kube_control, auth_control, cni):
         hookenv.log('Waiting for cluster cidr.')
         return
 
-    creds = kube_control.get_auth_credentials()
+    creds = db.get('credentials')
     data_changed('kube-control.creds', creds)
 
     # set --allow-privileged flag for kubelet
@@ -389,6 +392,8 @@ def render_and_launch_ingress():
                          '/root/cdk/addons/default-http-backend.yaml')
         kubectl_manifest('delete',
                          '/root/cdk/addons/ingress-replication-controller.yaml')  # noqa
+        kubectl_manifest('delete',
+                         '/root/cdk/addons/ingress-replication-controller-service.yaml')  # noqa
         hookenv.close_port(80)
         hookenv.close_port(443)
 
@@ -458,11 +463,13 @@ def create_config(server, creds):
     cmd = ['chown', '-R', 'ubuntu:ubuntu', '/home/ubuntu/.kube']
     check_call(cmd)
     # Create kubernetes configuration in the default location for root.
-    create_kubeconfig('/root/.kube/config', server, ca,
+    create_kubeconfig(kubeclientconfig_path, server, ca,
                       token=creds['client_token'], user='root')
     # Create kubernetes configuration for kubelet, and kube-proxy services.
     create_kubeconfig(kubeconfig_path, server, ca,
                       token=creds['kubelet_token'], user='kubelet')
+    create_kubeconfig(kubeproxyconfig_path, server, ca,
+                      token=creds['proxy_token'], user='kube-proxy')
 
 
 def configure_worker_services(api_servers, dns, cluster_cidr):
@@ -491,7 +498,7 @@ def configure_worker_services(api_servers, dns, cluster_cidr):
 
     kube_proxy_opts = FlagManager('kube-proxy')
     kube_proxy_opts.add('cluster-cidr', cluster_cidr)
-    kube_proxy_opts.add('kubeconfig', kubeconfig_path)
+    kube_proxy_opts.add('kubeconfig', kubeproxyconfig_path)
     kube_proxy_opts.add('logtostderr', 'true')
     kube_proxy_opts.add('v', '0')
     kube_proxy_opts.add('master', random.choice(api_servers), strict=True)
@@ -556,7 +563,7 @@ def launch_default_ingress_controller():
 
     # Render the default http backend (404) replicationcontroller manifest
     manifest = addon_path.format('default-http-backend.yaml')
-    render('default-http-backend.yaml', manifest, context)
+    render('default-backend.yml', manifest, context)
     hookenv.log('Creating the default http backend.')
     try:
         kubectl('apply', '-f', manifest)
@@ -574,7 +581,7 @@ def launch_default_ingress_controller():
         context['ingress_image'] = \
             "docker.io/cdkbot/nginx-ingress-controller-s390x:0.9.0-beta.13"
     manifest = addon_path.format('ingress-replication-controller.yaml')
-    render('ingress-replication-controller.yaml', manifest, context)
+    render('nginx-ingress-controller.yml', manifest, context)
     hookenv.log('Creating the ingress replication controller.')
     try:
         kubectl('apply', '-f', manifest)
@@ -584,6 +591,17 @@ def launch_default_ingress_controller():
         hookenv.close_port(80)
         hookenv.close_port(443)
         return
+    manifest = addon_path.format('ingress-replication-controller-service.yaml')
+    render('nginx-ingress-controller-service.yml', manifest, context)
+    hookenv.log('Creating the ingress replication controller service.')
+    try:
+        kubectl('apply', '-f', manifest)
+    except CalledProcessError as e:
+        hookenv.log(e)
+        hookenv.log('Failed to create ingress controller service. Will attempt again next update.')  # noqa
+        hookenv.close_port(80)
+        hookenv.close_port(443)
+        return
 
     set_state('kubernetes-worker.ingress.available')
     hookenv.open_port(80)
@@ -613,7 +631,7 @@ def get_kube_api_servers(kube_api):
 def kubectl(*args):
     ''' Run a kubectl cli command with a config file. Returns stdout and throws
     an error if the command fails. '''
-    command = ['kubectl', '--kubeconfig=' + kubeconfig_path] + list(args)
+    command = ['kubectl', '--kubeconfig=' + kubeclientconfig_path] + list(args)
     hookenv.log('Executing {}'.format(command))
     return check_output(command)
 
@@ -821,7 +839,10 @@ def request_kubelet_and_proxy_credentials(kube_control):
 def catch_change_in_creds(kube_control):
     """Request a service restart in case credential updates were detected."""
     creds = kube_control.get_auth_credentials()
-    if data_changed('kube-control.creds', creds):
+    nodeuser = 'system:node:{}'.format(gethostname())
+    if data_changed('kube-control.creds', creds) and creds['user'] == nodeuser:
+        db.set('credentials', creds)
+        set_state('worker.auth.bootstrapped')
         set_state('kubernetes-worker.restart-needed')
 
 

cluster/juju/layers/kubernetes-worker/templates/default-backend.yml

@@ -0,0 +1,51 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: default-http-backend
+  labels:
+    k8s-app: default-http-backend
+  namespace: kube-system
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: default-http-backend
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+      - name: default-http-backend
+        # Any image is permissable as long as:
+        # 1. It serves a 404 page at /
+        # 2. It serves 200 on a /healthz endpoint
+        image: gcr.io/google_containers/defaultbackend:1.0
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        ports:
+        - containerPort: 8080
+        resources:
+          limits:
+            cpu: 10m
+            memory: 20Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: default-http-backend
+  namespace: kube-system
+  labels:
+    k8s-app: default-http-backend
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+  selector:
+    k8s-app: default-http-backend

cluster/juju/layers/kubernetes-worker/templates/nginx-ingress-controller-service.yml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-ingress
+  namespace: kube-system
+spec:
+# Can also use LoadBalancer type
+  type: NodePort
+  ports:
+  - name: http
+    port: 8080
+    nodePort: 30080
+    targetPort: 80
+    protocol: TCP
+  selector:
+    k8s-app: nginx-ingress-controller

cluster/juju/layers/kubernetes-worker/templates/nginx-ingress-controller.yml

@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-load-balancer-conf
+  namespace: kube-system
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: nginx-ingress-controller
+  labels:
+    k8s-app: nginx-ingress-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+        k8s-app: nginx-ingress-controller
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-ingress-controller
+      annotations:
+        prometheus.io/port: '10254'
+        prometheus.io/scrape: 'true'
+    spec:
+      # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration
+      # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host
+      # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used
+      # like with kubeadm
+      hostNetwork: true
+      terminationGracePeriodSeconds: 60
+      serviceAccountName: nginx-ingress-serviceaccount
+      containers:
+      - name: nginx-ingress-controller
+        image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.11
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        args:
+        - /nginx-ingress-controller
+        - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
+        - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf
+        ports:
+        - containerPort: 80
+          hostPort: 80
+        - containerPort: 443
+          hostPort: 443