Merge pull request #55185 from Cynerva/gkk/scheduler-extra-args

Automatic merge from submit-queue (batch tested with PRs 53866, 54852, 55178, 55185, 55130). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add extra-args configs for scheduler and controller-manager to kubernetes-master charm

**What this PR does / why we need it**:

This adds `controller-manager-extra-args` and `scheduler-extra-args` config options to the kubernetes-master charm.

These allow operators of Juju-based Kubernetes clusters to pass arbitrary arguments to kube-controller-manager and kube-scheduler as needed.

**Release note**:

```release-note
Add extra-args configs for scheduler and controller-manager to kubernetes-master charm
```
This commit is contained in:
Kubernetes Submit Queue 2017-11-06 22:20:35 -08:00 committed by GitHub
commit de53b19024
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 93 additions and 55 deletions

View File

@ -46,6 +46,24 @@ options:
runtime-config=batch/v2alpha1=true profiling=true
will result in kube-apiserver being run with the following options:
--runtime-config=batch/v2alpha1=true --profiling=true
controller-manager-extra-args:
type: string
default: ""
description: |
Space separated list of flags and key=value pairs that will be passed as arguments to
kube-controller-manager. For example a value like this:
runtime-config=batch/v2alpha1=true profiling=true
will result in kube-controller-manager being run with the following options:
--runtime-config=batch/v2alpha1=true --profiling=true
scheduler-extra-args:
type: string
default: ""
description: |
Space separated list of flags and key=value pairs that will be passed as arguments to
kube-scheduler. For example a value like this:
runtime-config=batch/v2alpha1=true profiling=true
will result in kube-scheduler being run with the following options:
--runtime-config=batch/v2alpha1=true --profiling=true
authorization-mode:
type: string
default: "AlwaysAllow"

View File

@ -772,6 +772,18 @@ def on_config_api_extra_args_change():
configure_apiserver()
@when('config.changed.controller-manager-extra-args')
@when('kubernetes-master.components.started')
def on_config_controller_manager_extra_args_change():
configure_controller_manager()
@when('config.changed.scheduler-extra-args')
@when('kubernetes-master.components.started')
def on_config_scheduler_extra_args_change():
configure_scheduler()
@when('kube-control.gpu.available')
@when('kubernetes-master.components.started')
@when_not('kubernetes-master.gpu.enabled')
@ -975,27 +987,27 @@ def handle_etcd_relation(reldata):
api_opts.add('etcd-servers', connection_string, strict=True)
def get_config_args():
def get_config_args(key):
db = unitdata.kv()
old_config_args = db.get('api-extra-args', [])
old_config_args = db.get(key, [])
# We have to convert them to tuples becuase we use sets
old_config_args = [tuple(i) for i in old_config_args]
new_config_args = []
new_config_arg_names = []
for arg in hookenv.config().get('api-extra-args', '').split():
for arg in hookenv.config().get(key, '').split():
new_config_arg_names.append(arg.split('=', 1)[0])
if len(arg.split('=', 1)) == 1: # handle flags ie. --profiling
new_config_args.append(tuple([arg, 'true']))
else:
new_config_args.append(tuple(arg.split('=', 1)))
hookenv.log('Handling "api-extra-args" option.')
hookenv.log('Handling "%s" option.' % key)
hookenv.log('Old arguments: {}'.format(old_config_args))
hookenv.log('New arguments: {}'.format(new_config_args))
if set(new_config_args) == set(old_config_args):
return (new_config_args, [])
# Store new args
db.set('api-extra-args', new_config_args)
db.set(key, new_config_args)
to_add = set(new_config_args)
to_remove = set(old_config_args) - set(new_config_args)
# Extract option names only
@ -1003,23 +1015,40 @@ def get_config_args():
return (to_add, to_remove)
def configure_apiserver():
# TODO: investigate if it's possible to use config file to store args
# https://github.com/juju-solutions/bundle-canonical-kubernetes/issues/315
def configure_kubernetes_service(service, base_args, extra_args_key):
# Handle api-extra-args config option
to_add, to_remove = get_config_args()
to_add, to_remove = get_config_args(extra_args_key)
api_opts = FlagManager('kube-apiserver')
flag_manager = FlagManager(service)
# Remove arguments that are no longer provided as config option
# this allows them to be reverted to charm defaults
for arg in to_remove:
hookenv.log('Removing option: {}'.format(arg))
api_opts.destroy(arg)
# We need to "unset" options by settig their value to "null" string
cmd = ['snap', 'set', 'kube-apiserver', '{}=null'.format(arg)]
flag_manager.destroy(arg)
# We need to "unset" options by setting their value to "null" string
cmd = ['snap', 'set', service, '{}=null'.format(arg)]
check_call(cmd)
# Add base arguments
for k, v in base_args.items():
flag_manager.add(k, v, strict=True)
# Add operator-provided arguments, this allows operators
# to override defaults
for arg in to_add:
hookenv.log('Adding option: {} {}'.format(arg[0], arg[1]))
# Make sure old value is gone
flag_manager.destroy(arg[0])
flag_manager.add(arg[0], arg[1], strict=True)
cmd = ['snap', 'set', service] + flag_manager.to_s().split(' ')
check_call(cmd)
def configure_apiserver():
api_opts = {}
# Get the tls paths from the layer data.
layer_options = layer.options('tls-client')
ca_cert_path = layer_options.get('ca_certificate_path')
@ -1029,25 +1058,25 @@ def configure_apiserver():
server_key_path = layer_options.get('server_key_path')
if is_privileged():
api_opts.add('allow-privileged', 'true', strict=True)
api_opts['allow-privileged'] = 'true'
set_state('kubernetes-master.privileged')
else:
api_opts.add('allow-privileged', 'false', strict=True)
api_opts['allow-privileged'] = 'false'
remove_state('kubernetes-master.privileged')
# Handle static options for now
api_opts.add('service-cluster-ip-range', service_cidr())
api_opts.add('min-request-timeout', '300')
api_opts.add('v', '4')
api_opts.add('tls-cert-file', server_cert_path)
api_opts.add('tls-private-key-file', server_key_path)
api_opts.add('kubelet-certificate-authority', ca_cert_path)
api_opts.add('kubelet-client-certificate', client_cert_path)
api_opts.add('kubelet-client-key', client_key_path)
api_opts.add('logtostderr', 'true')
api_opts.add('insecure-bind-address', '127.0.0.1')
api_opts.add('insecure-port', '8080')
api_opts.add('storage-backend', 'etcd2') # FIXME: add etcd3 support
api_opts['service-cluster-ip-range'] = service_cidr()
api_opts['min-request-timeout'] = '300'
api_opts['v'] = '4'
api_opts['tls-cert-file'] = server_cert_path
api_opts['tls-private-key-file'] = server_key_path
api_opts['kubelet-certificate-authority'] = ca_cert_path
api_opts['kubelet-client-certificate'] = client_cert_path
api_opts['kubelet-client-key'] = client_key_path
api_opts['logtostderr'] = 'true'
api_opts['insecure-bind-address'] = '127.0.0.1'
api_opts['insecure-port'] = '8080'
api_opts['storage-backend'] = 'etcd2' # FIXME: add etcd3 support
admission_control = [
'Initializers',
@ -1062,7 +1091,7 @@ def configure_apiserver():
if 'Node' in auth_mode:
admission_control.append('NodeRestriction')
api_opts.add('authorization-mode', auth_mode, strict=True)
api_opts['authorization-mode'] = auth_mode
if get_version('kube-apiserver') < (1, 6):
hookenv.log('Removing DefaultTolerationSeconds from admission-control')
@ -1070,52 +1099,43 @@ def configure_apiserver():
if get_version('kube-apiserver') < (1, 7):
hookenv.log('Removing Initializers from admission-control')
admission_control.remove('Initializers')
api_opts.add('admission-control', ','.join(admission_control), strict=True)
api_opts['admission-control'] = ','.join(admission_control)
# Add operator-provided arguments, this allows operators
# to override defaults
for arg in to_add:
hookenv.log('Adding option: {} {}'.format(arg[0], arg[1]))
# Make sure old value is gone
api_opts.destroy(arg[0])
api_opts.add(arg[0], arg[1])
configure_kubernetes_service('kube-apiserver', api_opts, 'api-extra-args')
cmd = ['snap', 'set', 'kube-apiserver'] + api_opts.to_s().split(' ')
check_call(cmd)
set_state('kube-apiserver.do-restart')
def configure_controller_manager():
controller_opts = FlagManager('kube-controller-manager')
controller_opts = {}
# Get the tls paths from the layer data.
layer_options = layer.options('tls-client')
ca_cert_path = layer_options.get('ca_certificate_path')
# Default to 3 minute resync. TODO: Make this configureable?
controller_opts.add('min-resync-period', '3m')
controller_opts.add('v', '2')
controller_opts.add('root-ca-file', ca_cert_path)
controller_opts.add('logtostderr', 'true')
controller_opts.add('master', 'http://127.0.0.1:8080')
controller_opts['min-resync-period'] = '3m'
controller_opts['v'] = '2'
controller_opts['root-ca-file'] = ca_cert_path
controller_opts['logtostderr'] = 'true'
controller_opts['master'] = 'http://127.0.0.1:8080'
configure_kubernetes_service('kube-controller-manager', controller_opts,
'controller-manager-extra-args')
cmd = (
['snap', 'set', 'kube-controller-manager'] +
controller_opts.to_s().split(' ')
)
check_call(cmd)
set_state('kube-controller-manager.do-restart')
def configure_scheduler():
scheduler_opts = FlagManager('kube-scheduler')
scheduler_opts = {}
scheduler_opts.add('v', '2')
scheduler_opts.add('logtostderr', 'true')
scheduler_opts.add('master', 'http://127.0.0.1:8080')
scheduler_opts['v'] = '2'
scheduler_opts['logtostderr'] = 'true'
scheduler_opts['master'] = 'http://127.0.0.1:8080'
configure_kubernetes_service('kube-scheduler', scheduler_opts,
'scheduler-extra-args')
cmd = ['snap', 'set', 'kube-scheduler'] + scheduler_opts.to_s().split(' ')
check_call(cmd)
set_state('kube-scheduler.do-restart')