From ca7f8973f79cfb549896d65fc6a88d298df37155 Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Thu, 3 Aug 2017 15:26:41 +0300 Subject: [PATCH] RBAC work on PoC --- .../juju/layers/kubernetes-master/config.yaml | 4 + .../reactive/kubernetes_master.py | 145 ++++++++++++++++-- .../templates/heapster-rbac.yaml | 58 +++++++ .../nginx-ingress-controller-rbac.yml | 127 +++++++++++++++ .../reactive/kubernetes_worker.py | 37 ++++- .../templates/default-backend.yml | 51 ++++++ .../nginx-ingress-controller-service.yml | 16 ++ .../templates/nginx-ingress-controller.yml | 66 ++++++++ 8 files changed, 479 insertions(+), 25 deletions(-) create mode 100644 cluster/juju/layers/kubernetes-master/templates/heapster-rbac.yaml create mode 100644 cluster/juju/layers/kubernetes-master/templates/nginx-ingress-controller-rbac.yml create mode 100644 cluster/juju/layers/kubernetes-worker/templates/default-backend.yml create mode 100644 cluster/juju/layers/kubernetes-worker/templates/nginx-ingress-controller-service.yml create mode 100644 cluster/juju/layers/kubernetes-worker/templates/nginx-ingress-controller.yml diff --git a/cluster/juju/layers/kubernetes-master/config.yaml b/cluster/juju/layers/kubernetes-master/config.yaml index c328a43751c..aca3f31ea0a 100644 --- a/cluster/juju/layers/kubernetes-master/config.yaml +++ b/cluster/juju/layers/kubernetes-master/config.yaml @@ -40,3 +40,7 @@ options: runtime-config=batch/v2alpha1=true profiling=true will result in kube-apiserver being run with the following options: --runtime-config=batch/v2alpha1=true --profiling=true + enable-rbac: + type: boolean + default: True + description: Enable RBAC authorization mode. \ No newline at end of file diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index b6430637d47..f910020e633 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -209,12 +209,10 @@ def setup_leader_authentication(): if not get_keys_from_leader(keys) \ or is_state('reconfigure.authentication.setup'): last_pass = get_password('basic_auth.csv', 'admin') - setup_basic_auth(last_pass, 'admin', 'admin') + setup_basic_auth(last_pass, 'admin', 'admin', 'system:masters') if not os.path.isfile(known_tokens): - setup_tokens(None, 'admin', 'admin') - setup_tokens(None, 'kubelet', 'kubelet') - setup_tokens(None, 'kube_proxy', 'kube_proxy') + touch(known_tokens) # Generate the default service account token key os.makedirs('/root/cdk', exist_ok=True) @@ -400,19 +398,63 @@ def send_cluster_dns_detail(kube_control): @when('kube-control.auth.requested') -@when('authentication.setup') +@when('snap.installed.kubectl') @when('leadership.is_leader') -def send_tokens(kube_control): - """Send the tokens to the workers.""" - kubelet_token = get_token('kubelet') - proxy_token = get_token('kube_proxy') - admin_token = get_token('admin') +def create_service_configs(kube_control): + """Create the users for kubelet""" + # generate the username/pass for the requesting unit + proxy_token = get_token('system:kube-proxy') + if not proxy_token: + setup_tokens(None, 'system:kube-proxy', 'kube-proxy') + proxy_token = get_token('system:kube-proxy') + + client_token = get_token('admin') + if not client_token: + setup_tokens(None, 'admin', 'admin', "system:masters") + client_token = get_token('admin') - # Send the data requests = kube_control.auth_user() for request in requests: - kube_control.sign_auth_request(request[0], kubelet_token, - proxy_token, admin_token) + username = request[1]['user'] + group = request[1]['group'] + kubelet_token = get_token(username) + if not kubelet_token: + # Usernames have to be in the form of system:node: + userid = "kubelet-{}".format(request[0].split('/')[1]) + setup_tokens(None, username, userid, group) + kubelet_token = get_token(username) + + kube_control.sign_auth_request(request[0], username, + kubelet_token, proxy_token, client_token) + + host.service_restart('snap.kube-apiserver.daemon') + remove_state('authentication.setup') + + +@when('kube-control.departed') +@when('leadership.is_leader') +def flush_auth_for_departed(kube_control): + ''' Unit has left the cluster and needs to have its authentication + tokens removed from the token registry ''' + token_auth_file = '/root/cdk/known_tokens.csv' + departing_unit = kube_control.flush_departed() + userid = "kubelet-{}".format(departing_unit.split('/')[1]) + known_tokens = open(token_auth_file, 'r').readlines() + for line in known_tokens[:]: + haystack = line.split(',') + # skip the entry if we dont have token,user,id,groups format + if len(haystack) < 4: + continue + if haystack[2] == userid: + hookenv.log('Found unit {} in token auth. Removing auth' + ' token.'.format(userid)) + known_tokens.remove(line) + # atomically rewrite the file minus any scrubbed units + hookenv.log('Rewriting token auth file: {}'.format(token_auth_file)) + with open(token_auth_file, 'w') as fp: + fp.writelines(known_tokens) + # Trigger rebroadcast of auth files for followers + remove_state('autentication.setup') @when_not('kube-control.connected') @@ -492,6 +534,7 @@ def addons_ready(): """ try: + apply_rbac() check_call(['cdk-addons.apply']) return True except CalledProcessError: @@ -614,6 +657,52 @@ def initial_nrpe_config(nagios=None): update_nrpe_config(nagios) +@when('config.changed.enable-rbac', + 'kubernetes-master.components.started') +def enable_rbac_config(): + config = hookenv.config() + if data_changed('rbac-flag', str(config.get('enable-rbac'))): + remove_state('kubernetes-master.components.started') + + +def apply_rbac(): + # TODO(kjackal): we should be checking if rbac is already applied + config = hookenv.config() + if is_state('leadership.is_leader'): + if config.get('enable-rbac'): + try: + cmd = ['kubectl', 'apply', '-f', 'templates/heapster-rbac.yaml'] + check_output(cmd).decode('utf-8') + except CalledProcessError: + hookenv.log('Failed to apply heapster rbac rules') + try: + cmd = ['kubectl', 'apply', '-f', 'templates/nginx-ingress-controller-rbac.yml'] + check_output(cmd).decode('utf-8') + except CalledProcessError: + hookenv.log('Failed to apply heapster rbac rules') + + # TODO(kjackal): The follwoing is wrong and imposes security risk. What we should be doing is + # update the add-ons to include an rbac enabled dashboard + try: + cmd = "kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin" \ + " --serviceaccount=kube-system:default".split(' ') + check_output(cmd).decode('utf-8') + except CalledProcessError: + hookenv.log('Failed to elevate credentials') + + else: + try: + cmd = ['kubectl', 'delete', '-f', 'templates/heapster-rbac.yaml'] + check_output(cmd).decode('utf-8') + except CalledProcessError: + hookenv.log('Failed to delete heapster rbac rules') + try: + cmd = ['kubectl', 'delete', '-f', 'templates/nginx-ingress-controller-rbac.yml'] + check_output(cmd).decode('utf-8') + except CalledProcessError: + hookenv.log('Failed to apply heapster rbac rules') + + @when('kubernetes-master.components.started') @when('nrpe-external-master.available') @when_any('config.changed.nagios_context', @@ -965,6 +1054,12 @@ def configure_apiserver(): 'DefaultTolerationSeconds' ] + if hookenv.config('enable-rbac'): + admission_control.append('NodeRestriction') + api_opts.add('authorization-mode', 'Node,RBAC', strict=True) + else: + api_opts.add('authorization-mode', 'AlwaysAllow', strict=True) + if get_version('kube-apiserver') < (1, 6): hookenv.log('Removing DefaultTolerationSeconds from admission-control') admission_control.remove('DefaultTolerationSeconds') @@ -1020,7 +1115,8 @@ def configure_scheduler(): set_state('kube-scheduler.do-restart') -def setup_basic_auth(password=None, username='admin', uid='admin'): +def setup_basic_auth(password=None, username='admin', uid='admin', + groups=None): '''Create the htacces file and the tokens.''' root_cdk = '/root/cdk' if not os.path.isdir(root_cdk): @@ -1029,10 +1125,14 @@ def setup_basic_auth(password=None, username='admin', uid='admin'): if not password: password = token_generator() with open(htaccess, 'w') as stream: - stream.write('{0},{1},{2}'.format(password, username, uid)) + if groups: + stream.write('{0},{1},{2},"{3}"'.format(password, + username, uid, groups)) + else: + stream.write('{0},{1},{2}'.format(password, username, uid)) -def setup_tokens(token, username, user): +def setup_tokens(token, username, user, groups=None): '''Create a token file for kubernetes authentication.''' root_cdk = '/root/cdk' if not os.path.isdir(root_cdk): @@ -1041,7 +1141,11 @@ def setup_tokens(token, username, user): if not token: token = token_generator() with open(known_tokens, 'a') as stream: - stream.write('{0},{1},{2}\n'.format(token, username, user)) + if groups: + stream.write('{0},{1},{2},"{3}"\n'.format(token, + username, user, groups)) + else: + stream.write('{0},{1},{2}\n'.format(token, username, user)) def get_password(csv_fname, user): @@ -1107,3 +1211,10 @@ def apiserverVersion(): cmd = 'kube-apiserver --version'.split() version_string = check_output(cmd).decode('utf-8') return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3]) + + +def touch(fname): + try: + os.utime(fname, None) + except OSError: + open(fname, 'a').close() \ No newline at end of file diff --git a/cluster/juju/layers/kubernetes-master/templates/heapster-rbac.yaml b/cluster/juju/layers/kubernetes-master/templates/heapster-rbac.yaml new file mode 100644 index 00000000000..58fa1b9921b --- /dev/null +++ b/cluster/juju/layers/kubernetes-master/templates/heapster-rbac.yaml @@ -0,0 +1,58 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: heapster-binding + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:heapster +subjects: +- kind: ServiceAccount + name: heapster + namespace: kube-system +--- +# Heapster's pod_nanny monitors the heapster deployment & its pod(s), and scales +# the resources of the deployment if necessary. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: system:pod-nanny + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - "extensions" + resources: + - deployments + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: heapster-binding + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system:pod-nanny +subjects: +- kind: ServiceAccount + name: heapster + namespace: kube-system +--- diff --git a/cluster/juju/layers/kubernetes-master/templates/nginx-ingress-controller-rbac.yml b/cluster/juju/layers/kubernetes-master/templates/nginx-ingress-controller-rbac.yml new file mode 100644 index 00000000000..696f4c6e076 --- /dev/null +++ b/cluster/juju/layers/kubernetes-master/templates/nginx-ingress-controller-rbac.yml @@ -0,0 +1,127 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nginx-ingress-serviceaccount + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: nginx-ingress-clusterrole +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + resources: + - ingresses/status + verbs: + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: nginx-ingress-role + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + # Defaults to "-" + # Here: "-" + # This has to be adapted if you change either parameter + # when launching the nginx-ingress-controller. + - "ingress-controller-leader-nginx" + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: nginx-ingress-role-nisa-binding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-ingress-role +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: nginx-ingress-clusterrole-nisa-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx-ingress-clusterrole +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount + namespace: kube-system diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index 2f5707790b6..60129618b7a 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -47,6 +47,8 @@ from charmhelpers.contrib.charmsupport import nrpe nrpe.Check.shortname_re = '[\.A-Za-z0-9-_]+$' kubeconfig_path = '/root/cdk/kubeconfig' +kubeproxyconfig_path = '/root/cdk/kubeproxyconfig' +kubeclientconfig_path = '/root/.kube/config' os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin') @@ -319,7 +321,8 @@ def watch_for_changes(kube_api, kube_control, cni): 'tls_client.client.key.saved', 'tls_client.server.certificate.saved', 'tls_client.server.key.saved', 'kube-control.dns.available', 'kube-control.auth.available', - 'cni.available', 'kubernetes-worker.restart-needed') + 'cni.available', 'kubernetes-worker.restart-needed', + 'worker.auth.bootstrapped') def start_worker(kube_api, kube_control, auth_control, cni): ''' Start kubelet using the provided API and DNS info.''' servers = get_kube_api_servers(kube_api) @@ -335,7 +338,7 @@ def start_worker(kube_api, kube_control, auth_control, cni): hookenv.log('Waiting for cluster cidr.') return - creds = kube_control.get_auth_credentials() + creds = db.get('credentials') data_changed('kube-control.creds', creds) # set --allow-privileged flag for kubelet @@ -389,6 +392,8 @@ def render_and_launch_ingress(): '/root/cdk/addons/default-http-backend.yaml') kubectl_manifest('delete', '/root/cdk/addons/ingress-replication-controller.yaml') # noqa + kubectl_manifest('delete', + '/root/cdk/addons/ingress-replication-controller-service.yaml') # noqa hookenv.close_port(80) hookenv.close_port(443) @@ -458,11 +463,13 @@ def create_config(server, creds): cmd = ['chown', '-R', 'ubuntu:ubuntu', '/home/ubuntu/.kube'] check_call(cmd) # Create kubernetes configuration in the default location for root. - create_kubeconfig('/root/.kube/config', server, ca, + create_kubeconfig(kubeclientconfig_path, server, ca, token=creds['client_token'], user='root') # Create kubernetes configuration for kubelet, and kube-proxy services. create_kubeconfig(kubeconfig_path, server, ca, token=creds['kubelet_token'], user='kubelet') + create_kubeconfig(kubeproxyconfig_path, server, ca, + token=creds['proxy_token'], user='kube-proxy') def configure_worker_services(api_servers, dns, cluster_cidr): @@ -491,7 +498,7 @@ def configure_worker_services(api_servers, dns, cluster_cidr): kube_proxy_opts = FlagManager('kube-proxy') kube_proxy_opts.add('cluster-cidr', cluster_cidr) - kube_proxy_opts.add('kubeconfig', kubeconfig_path) + kube_proxy_opts.add('kubeconfig', kubeproxyconfig_path) kube_proxy_opts.add('logtostderr', 'true') kube_proxy_opts.add('v', '0') kube_proxy_opts.add('master', random.choice(api_servers), strict=True) @@ -556,7 +563,7 @@ def launch_default_ingress_controller(): # Render the default http backend (404) replicationcontroller manifest manifest = addon_path.format('default-http-backend.yaml') - render('default-http-backend.yaml', manifest, context) + render('default-backend.yml', manifest, context) hookenv.log('Creating the default http backend.') try: kubectl('apply', '-f', manifest) @@ -574,7 +581,7 @@ def launch_default_ingress_controller(): context['ingress_image'] = \ "docker.io/cdkbot/nginx-ingress-controller-s390x:0.9.0-beta.13" manifest = addon_path.format('ingress-replication-controller.yaml') - render('ingress-replication-controller.yaml', manifest, context) + render('nginx-ingress-controller.yml', manifest, context) hookenv.log('Creating the ingress replication controller.') try: kubectl('apply', '-f', manifest) @@ -584,6 +591,17 @@ def launch_default_ingress_controller(): hookenv.close_port(80) hookenv.close_port(443) return + manifest = addon_path.format('ingress-replication-controller-service.yaml') + render('nginx-ingress-controller-service.yml', manifest, context) + hookenv.log('Creating the ingress replication controller service.') + try: + kubectl('apply', '-f', manifest) + except CalledProcessError as e: + hookenv.log(e) + hookenv.log('Failed to create ingress controller service. Will attempt again next update.') # noqa + hookenv.close_port(80) + hookenv.close_port(443) + return set_state('kubernetes-worker.ingress.available') hookenv.open_port(80) @@ -613,7 +631,7 @@ def get_kube_api_servers(kube_api): def kubectl(*args): ''' Run a kubectl cli command with a config file. Returns stdout and throws an error if the command fails. ''' - command = ['kubectl', '--kubeconfig=' + kubeconfig_path] + list(args) + command = ['kubectl', '--kubeconfig=' + kubeclientconfig_path] + list(args) hookenv.log('Executing {}'.format(command)) return check_output(command) @@ -821,7 +839,10 @@ def request_kubelet_and_proxy_credentials(kube_control): def catch_change_in_creds(kube_control): """Request a service restart in case credential updates were detected.""" creds = kube_control.get_auth_credentials() - if data_changed('kube-control.creds', creds): + nodeuser = 'system:node:{}'.format(gethostname()) + if data_changed('kube-control.creds', creds) and creds['user'] == nodeuser: + db.set('credentials', creds) + set_state('worker.auth.bootstrapped') set_state('kubernetes-worker.restart-needed') diff --git a/cluster/juju/layers/kubernetes-worker/templates/default-backend.yml b/cluster/juju/layers/kubernetes-worker/templates/default-backend.yml new file mode 100644 index 00000000000..3c40989a31e --- /dev/null +++ b/cluster/juju/layers/kubernetes-worker/templates/default-backend.yml @@ -0,0 +1,51 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: default-http-backend + labels: + k8s-app: default-http-backend + namespace: kube-system +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: default-http-backend + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: default-http-backend + # Any image is permissable as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.0 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: default-http-backend + namespace: kube-system + labels: + k8s-app: default-http-backend +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + k8s-app: default-http-backend diff --git a/cluster/juju/layers/kubernetes-worker/templates/nginx-ingress-controller-service.yml b/cluster/juju/layers/kubernetes-worker/templates/nginx-ingress-controller-service.yml new file mode 100644 index 00000000000..ad8b79df13d --- /dev/null +++ b/cluster/juju/layers/kubernetes-worker/templates/nginx-ingress-controller-service.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-ingress + namespace: kube-system +spec: +# Can also use LoadBalancer type + type: NodePort + ports: + - name: http + port: 8080 + nodePort: 30080 + targetPort: 80 + protocol: TCP + selector: + k8s-app: nginx-ingress-controller diff --git a/cluster/juju/layers/kubernetes-worker/templates/nginx-ingress-controller.yml b/cluster/juju/layers/kubernetes-worker/templates/nginx-ingress-controller.yml new file mode 100644 index 00000000000..206fc3b6577 --- /dev/null +++ b/cluster/juju/layers/kubernetes-worker/templates/nginx-ingress-controller.yml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-load-balancer-conf + namespace: kube-system +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-ingress-controller + labels: + k8s-app: nginx-ingress-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: nginx-ingress-controller + template: + metadata: + labels: + k8s-app: nginx-ingress-controller + annotations: + prometheus.io/port: '10254' + prometheus.io/scrape: 'true' + spec: + # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration + # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host + # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used + # like with kubeadm + hostNetwork: true + terminationGracePeriodSeconds: 60 + serviceAccountName: nginx-ingress-serviceaccount + containers: + - name: nginx-ingress-controller + image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.11 + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 1 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend + - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf + ports: + - containerPort: 80 + hostPort: 80 + - containerPort: 443 + hostPort: 443