mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-21 17:48:01 +00:00
Upgrade addon-manager with kubectl apply
This commit is contained in:
Zihong Zheng
parent
b359034817
commit
0e4e54a6b6
@ -1,520 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Copyright 2015 The Kubernetes Authors.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
# The business logic for whether a given object should be created
|
|
||||||
# was already enforced by salt, and /etc/kubernetes/addons is the
|
|
||||||
# managed result is of that. Start everything below that directory.
|
|
||||||
|
|
||||||
# Parameters
|
|
||||||
# $1 path to add-ons
|
|
||||||
|
|
||||||
|
|
||||||
# LIMITATIONS
|
|
||||||
# 1. Controllers are not updated unless their name is changed
|
|
||||||
# 3. Services will not be updated unless their name is changed,
|
|
||||||
# but for services we actually want updates without name change.
|
|
||||||
# 4. Json files are not handled at all. Currently addons must be
|
|
||||||
# in yaml files
|
|
||||||
# 5. Exit code is probably not always correct (I haven't checked
|
|
||||||
# carefully if it works in 100% cases)
|
|
||||||
# 6. There are no unittests
|
|
||||||
# 8. Will not work if the total length of paths to addons is greater than
|
|
||||||
# bash can handle. Probably it is not a problem: ARG_MAX=2097152 on GCE.
|
|
||||||
# 9. Performance issue: yaml files are read many times in a single execution.
|
|
||||||
|
|
||||||
# cosmetic improvements to be done
|
|
||||||
# 1. Improve the log function; add timestamp, file name, etc.
|
|
||||||
# 2. Logging doesn't work from files that print things out.
|
|
||||||
# 3. Kubectl prints the output to stderr (the output should be captured and then
|
|
||||||
# logged)
|
|
||||||
|
|
||||||
# global config
|
|
||||||
KUBECTL=${TEST_KUBECTL:-} # substitute for tests
|
|
||||||
KUBECTL=${KUBECTL:-${KUBECTL_BIN:-}}
|
|
||||||
KUBECTL=${KUBECTL:-/usr/local/bin/kubectl}
|
|
||||||
if [[ ! -x ${KUBECTL} ]]; then
|
|
||||||
echo "ERROR: kubectl command (${KUBECTL}) not found or is not executable" 1>&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
KUBECTL_OPTS=${KUBECTL_OPTS:-}
|
|
||||||
|
|
||||||
# If an add-on definition is incorrect, or a definition has just disappeared
|
|
||||||
# from the local directory, the script will still keep on retrying.
|
|
||||||
# The script does not end until all retries are done, so
|
|
||||||
# one invalid manifest may block updates of other add-ons.
|
|
||||||
# Be careful how you set these parameters
|
|
||||||
NUM_TRIES=1 # will be updated based on input parameters
|
|
||||||
DELAY_AFTER_ERROR_SEC=${TEST_DELAY_AFTER_ERROR_SEC:=10}
|
|
||||||
|
|
||||||
|
|
||||||
# remember that you can't log from functions that print some output (because
|
|
||||||
# logs are also printed on stdout)
|
|
||||||
# $1 level
|
|
||||||
# $2 message
|
|
||||||
function log() {
|
|
||||||
# manage log levels manually here
|
|
||||||
|
|
||||||
# add the timestamp if you find it useful
|
|
||||||
case $1 in
|
|
||||||
DB3 )
|
|
||||||
# echo "$1: $2"
|
|
||||||
;;
|
|
||||||
DB2 )
|
|
||||||
# echo "$1: $2"
|
|
||||||
;;
|
|
||||||
DBG )
|
|
||||||
# echo "$1: $2"
|
|
||||||
;;
|
|
||||||
INFO )
|
|
||||||
echo "$1: $2"
|
|
||||||
;;
|
|
||||||
WRN )
|
|
||||||
echo "$1: $2"
|
|
||||||
;;
|
|
||||||
ERR )
|
|
||||||
echo "$1: $2"
|
|
||||||
;;
|
|
||||||
* )
|
|
||||||
echo "INVALID_LOG_LEVEL $1: $2"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
#$1 yaml file path
|
|
||||||
function get-object-kind-from-file() {
|
|
||||||
# prints to stdout, so log cannot be used
|
|
||||||
#WARNING: only yaml is supported
|
|
||||||
cat $1 | python -c '''
|
|
||||||
try:
|
|
||||||
import pipes,sys,yaml
|
|
||||||
y = yaml.load(sys.stdin)
|
|
||||||
labels = y["metadata"]["labels"]
|
|
||||||
if ("kubernetes.io/cluster-service", "true") not in labels.iteritems():
|
|
||||||
# all add-ons must have the label "kubernetes.io/cluster-service".
|
|
||||||
# Otherwise we are ignoring them (the update will not work anyway)
|
|
||||||
print "ERROR"
|
|
||||||
else:
|
|
||||||
print y["kind"]
|
|
||||||
except Exception, ex:
|
|
||||||
print "ERROR"
|
|
||||||
'''
|
|
||||||
}
|
|
||||||
|
|
||||||
# $1 yaml file path
|
|
||||||
# returns a string of the form <namespace>/<name> (we call it nsnames)
|
|
||||||
function get-object-nsname-from-file() {
|
|
||||||
# prints to stdout, so log cannot be used
|
|
||||||
#WARNING: only yaml is supported
|
|
||||||
#addons that do not specify a namespace are assumed to be in "default".
|
|
||||||
cat $1 | python -c '''
|
|
||||||
try:
|
|
||||||
import pipes,sys,yaml
|
|
||||||
y = yaml.load(sys.stdin)
|
|
||||||
labels = y["metadata"]["labels"]
|
|
||||||
if ("kubernetes.io/cluster-service", "true") not in labels.iteritems():
|
|
||||||
# all add-ons must have the label "kubernetes.io/cluster-service".
|
|
||||||
# Otherwise we are ignoring them (the update will not work anyway)
|
|
||||||
print "ERROR"
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
print "%s/%s" % (y["metadata"]["namespace"], y["metadata"]["name"])
|
|
||||||
except Exception, ex:
|
|
||||||
print "/%s" % y["metadata"]["name"]
|
|
||||||
except Exception, ex:
|
|
||||||
print "ERROR"
|
|
||||||
'''
|
|
||||||
}
|
|
||||||
|
|
||||||
# $1 addon directory path
|
|
||||||
# $2 addon type (e.g. ReplicationController)
|
|
||||||
# echoes the string with paths to files containing addon for the given type
|
|
||||||
# works only for yaml files (!) (ignores json files)
|
|
||||||
function get-addon-paths-from-disk() {
|
|
||||||
# prints to stdout, so log cannot be used
|
|
||||||
local -r addon_dir=$1
|
|
||||||
local -r obj_type=$2
|
|
||||||
local kind
|
|
||||||
local file_path
|
|
||||||
for file_path in $(find ${addon_dir} -name \*.yaml); do
|
|
||||||
kind=$(get-object-kind-from-file ${file_path})
|
|
||||||
# WARNING: assumption that the topmost indentation is zero (I'm not sure yaml allows for topmost indentation)
|
|
||||||
if [[ "${kind}" == "${obj_type}" ]]; then
|
|
||||||
echo ${file_path}
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# waits for all subprocesses
|
|
||||||
# returns 0 if all of them were successful and 1 otherwise
|
|
||||||
function wait-for-jobs() {
|
|
||||||
local rv=0
|
|
||||||
local pid
|
|
||||||
for pid in $(jobs -p); do
|
|
||||||
wait ${pid}
|
|
||||||
if [[ $? -ne 0 ]]; then
|
|
||||||
rv=1;
|
|
||||||
log ERR "error in pid ${pid}"
|
|
||||||
fi
|
|
||||||
log DB2 "pid ${pid} completed, current error code: ${rv}"
|
|
||||||
done
|
|
||||||
return ${rv}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function run-until-success() {
|
|
||||||
local -r command=$1
|
|
||||||
local tries=$2
|
|
||||||
local -r delay=$3
|
|
||||||
local -r command_name=$1
|
|
||||||
while [ ${tries} -gt 0 ]; do
|
|
||||||
log DBG "executing: '$command'"
|
|
||||||
# let's give the command as an argument to bash -c, so that we can use
|
|
||||||
# && and || inside the command itself
|
|
||||||
/bin/bash -c "${command}" && \
|
|
||||||
log DB3 "== Successfully executed ${command_name} at $(date -Is) ==" && \
|
|
||||||
return 0
|
|
||||||
let tries=tries-1
|
|
||||||
log INFO "== Failed to execute ${command_name} at $(date -Is). ${tries} tries remaining. =="
|
|
||||||
sleep ${delay}
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# $1 object type
|
|
||||||
# returns a list of <namespace>/<name> pairs (nsnames)
|
|
||||||
function get-addon-nsnames-from-server() {
|
|
||||||
local -r obj_type=$1
|
|
||||||
"${KUBECTL}" "${KUBECTL_OPTS}" get "${obj_type}" --all-namespaces -o go-template="{{range.items}}{{.metadata.namespace}}/{{.metadata.name}} {{end}}" -l kubernetes.io/cluster-service=true | sed 's/<no value>//g'
|
|
||||||
}
|
|
||||||
|
|
||||||
# returns the characters after the last separator (including)
|
|
||||||
# If the separator is empty or if it doesn't appear in the string,
|
|
||||||
# an empty string is printed
|
|
||||||
# $1 input string
|
|
||||||
# $2 separator (must be single character, or empty)
|
|
||||||
function get-suffix() {
|
|
||||||
# prints to stdout, so log cannot be used
|
|
||||||
local -r input_string=$1
|
|
||||||
local -r separator=$2
|
|
||||||
local suffix
|
|
||||||
|
|
||||||
if [[ "${separator}" == "" ]]; then
|
|
||||||
echo ""
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "${input_string}" == *"${separator}"* ]]; then
|
|
||||||
suffix=$(echo "${input_string}" | rev | cut -d "${separator}" -f1 | rev)
|
|
||||||
echo "${separator}${suffix}"
|
|
||||||
else
|
|
||||||
echo ""
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# returns the characters up to the last '-' (without it)
|
|
||||||
# $1 input string
|
|
||||||
# $2 separator
|
|
||||||
function get-basename() {
|
|
||||||
# prints to stdout, so log cannot be used
|
|
||||||
local -r input_string=$1
|
|
||||||
local -r separator=$2
|
|
||||||
local suffix
|
|
||||||
suffix="$(get-suffix ${input_string} ${separator})"
|
|
||||||
# this will strip the suffix (if matches)
|
|
||||||
echo ${input_string%$suffix}
|
|
||||||
}
|
|
||||||
|
|
||||||
function delete-object() {
|
|
||||||
local -r obj_type=$1
|
|
||||||
local -r namespace=$2
|
|
||||||
local -r obj_name=$3
|
|
||||||
log INFO "Deleting ${obj_type} ${namespace}/${obj_name}"
|
|
||||||
|
|
||||||
run-until-success "${KUBECTL} ${KUBECTL_OPTS} delete --namespace=${namespace} ${obj_type} ${obj_name}" ${NUM_TRIES} ${DELAY_AFTER_ERROR_SEC}
|
|
||||||
}
|
|
||||||
|
|
||||||
function create-object() {
|
|
||||||
local -r obj_type=$1
|
|
||||||
local -r file_path=$2
|
|
||||||
|
|
||||||
local nsname_from_file
|
|
||||||
nsname_from_file=$(get-object-nsname-from-file ${file_path})
|
|
||||||
if [[ "${nsname_from_file}" == "ERROR" ]]; then
|
|
||||||
log INFO "Cannot read object name from ${file_path}. Ignoring"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
IFS='/' read namespace obj_name <<< "${nsname_from_file}"
|
|
||||||
|
|
||||||
log INFO "Creating new ${obj_type} from file ${file_path} in namespace ${namespace}, name: ${obj_name}"
|
|
||||||
# this will keep on failing if the ${file_path} disappeared in the meantime.
|
|
||||||
# Do not use too many retries.
|
|
||||||
if [[ -n "${namespace}" ]]; then
|
|
||||||
run-until-success "${KUBECTL} ${KUBECTL_OPTS} create --namespace=${namespace} -f ${file_path}" ${NUM_TRIES} ${DELAY_AFTER_ERROR_SEC}
|
|
||||||
else
|
|
||||||
run-until-success "${KUBECTL} ${KUBECTL_OPTS} create -f ${file_path}" ${NUM_TRIES} ${DELAY_AFTER_ERROR_SEC}
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function update-object() {
|
|
||||||
local -r obj_type=$1
|
|
||||||
local -r namespace=$2
|
|
||||||
local -r obj_name=$3
|
|
||||||
local -r file_path=$4
|
|
||||||
log INFO "updating the ${obj_type} ${namespace}/${obj_name} with the new definition ${file_path}"
|
|
||||||
delete-object ${obj_type} ${namespace} ${obj_name}
|
|
||||||
create-object ${obj_type} ${file_path}
|
|
||||||
}
|
|
||||||
|
|
||||||
# deletes the objects from the server
|
|
||||||
# $1 object type
|
|
||||||
# $2 a list of object nsnames
|
|
||||||
function delete-objects() {
|
|
||||||
local -r obj_type=$1
|
|
||||||
local -r obj_nsnames=$2
|
|
||||||
local namespace
|
|
||||||
local obj_name
|
|
||||||
for nsname in ${obj_nsnames}; do
|
|
||||||
IFS='/' read namespace obj_name <<< "${nsname}"
|
|
||||||
delete-object ${obj_type} ${namespace} ${obj_name} &
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# creates objects from the given files
|
|
||||||
# $1 object type
|
|
||||||
# $2 a list of paths to definition files
|
|
||||||
function create-objects() {
|
|
||||||
local -r obj_type=$1
|
|
||||||
local -r file_paths=$2
|
|
||||||
local file_path
|
|
||||||
for file_path in ${file_paths}; do
|
|
||||||
# Remember that the file may have disappear by now
|
|
||||||
# But we don't want to check it here because
|
|
||||||
# such race condition may always happen after
|
|
||||||
# we check it. Let's have the race
|
|
||||||
# condition happen a bit more often so that
|
|
||||||
# we see that our tests pass anyway.
|
|
||||||
create-object ${obj_type} ${file_path} &
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# updates objects
|
|
||||||
# $1 object type
|
|
||||||
# $2 a list of update specifications
|
|
||||||
# each update specification is a ';' separated pair: <nsname>;<file path>
|
|
||||||
function update-objects() {
|
|
||||||
local -r obj_type=$1 # ignored
|
|
||||||
local -r update_spec=$2
|
|
||||||
local objdesc
|
|
||||||
local nsname
|
|
||||||
local obj_name
|
|
||||||
local namespace
|
|
||||||
|
|
||||||
for objdesc in ${update_spec}; do
|
|
||||||
IFS=';' read nsname file_path <<< "${objdesc}"
|
|
||||||
IFS='/' read namespace obj_name <<< "${nsname}"
|
|
||||||
|
|
||||||
update-object ${obj_type} ${namespace} ${obj_name} ${file_path} &
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# Global variables set by function match-objects.
|
|
||||||
nsnames_for_delete="" # a list of object nsnames to be deleted
|
|
||||||
for_update="" # a list of pairs <nsname>;<filePath> for objects that should be updated
|
|
||||||
nsnames_for_ignore="" # a list of object nsnames that will be ignored
|
|
||||||
new_files="" # a list of file paths that weren't matched by any existing objects (these objects must be created now)
|
|
||||||
|
|
||||||
|
|
||||||
# $1 path to files with objects
|
|
||||||
# $2 object type in the API (ReplicationController or Service)
|
|
||||||
# $3 name separator (single character or empty)
|
|
||||||
function match-objects() {
|
|
||||||
local -r addon_dir=$1
|
|
||||||
local -r obj_type=$2
|
|
||||||
local -r separator=$3
|
|
||||||
|
|
||||||
# output variables (globals)
|
|
||||||
nsnames_for_delete=""
|
|
||||||
for_update=""
|
|
||||||
nsnames_for_ignore=""
|
|
||||||
new_files=""
|
|
||||||
|
|
||||||
addon_nsnames_on_server=$(get-addon-nsnames-from-server "${obj_type}")
|
|
||||||
# if the api server is unavailable then abandon the update for this cycle
|
|
||||||
if [[ $? -ne 0 ]]; then
|
|
||||||
log ERR "unable to query ${obj_type} - exiting"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
addon_paths_in_files=$(get-addon-paths-from-disk "${addon_dir}" "${obj_type}")
|
|
||||||
|
|
||||||
log DB2 "addon_nsnames_on_server=${addon_nsnames_on_server}"
|
|
||||||
log DB2 "addon_paths_in_files=${addon_paths_in_files}"
|
|
||||||
|
|
||||||
local matched_files=""
|
|
||||||
|
|
||||||
local basensname_on_server=""
|
|
||||||
local nsname_on_server=""
|
|
||||||
local suffix_on_server=""
|
|
||||||
local nsname_from_file=""
|
|
||||||
local suffix_from_file=""
|
|
||||||
local found=0
|
|
||||||
local addon_path=""
|
|
||||||
|
|
||||||
# objects that were moved between namespaces will have different nsname
|
|
||||||
# because the namespace is included. So they will be treated
|
|
||||||
# like different objects and not updated but deleted and created again
|
|
||||||
# (in the current version update is also delete+create, so it does not matter)
|
|
||||||
for nsname_on_server in ${addon_nsnames_on_server}; do
|
|
||||||
basensname_on_server=$(get-basename ${nsname_on_server} ${separator})
|
|
||||||
suffix_on_server="$(get-suffix ${nsname_on_server} ${separator})"
|
|
||||||
|
|
||||||
log DB3 "Found existing addon ${nsname_on_server}, basename=${basensname_on_server}"
|
|
||||||
|
|
||||||
# check if the addon is present in the directory and decide
|
|
||||||
# what to do with it
|
|
||||||
# this is not optimal because we're reading the files over and over
|
|
||||||
# again. But for small number of addons it doesn't matter so much.
|
|
||||||
found=0
|
|
||||||
for addon_path in ${addon_paths_in_files}; do
|
|
||||||
nsname_from_file=$(get-object-nsname-from-file ${addon_path})
|
|
||||||
if [[ "${nsname_from_file}" == "ERROR" ]]; then
|
|
||||||
log INFO "Cannot read object name from ${addon_path}. Ignoring"
|
|
||||||
continue
|
|
||||||
else
|
|
||||||
log DB2 "Found object name '${nsname_from_file}' in file ${addon_path}"
|
|
||||||
fi
|
|
||||||
suffix_from_file="$(get-suffix ${nsname_from_file} ${separator})"
|
|
||||||
|
|
||||||
log DB3 "matching: ${basensname_on_server}${suffix_from_file} == ${nsname_from_file}"
|
|
||||||
if [[ "${basensname_on_server}${suffix_from_file}" == "${nsname_from_file}" ]]; then
|
|
||||||
log DB3 "matched existing ${obj_type} ${nsname_on_server} to file ${addon_path}; suffix_on_server=${suffix_on_server}, suffix_from_file=${suffix_from_file}"
|
|
||||||
found=1
|
|
||||||
matched_files="${matched_files} ${addon_path}"
|
|
||||||
if [[ "${suffix_on_server}" == "${suffix_from_file}" ]]; then
|
|
||||||
nsnames_for_ignore="${nsnames_for_ignore} ${nsname_from_file}"
|
|
||||||
else
|
|
||||||
for_update="${for_update} ${nsname_on_server};${addon_path}"
|
|
||||||
fi
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [[ ${found} -eq 0 ]]; then
|
|
||||||
log DB2 "No definition file found for replication controller ${nsname_on_server}. Scheduling for deletion"
|
|
||||||
nsnames_for_delete="${nsnames_for_delete} ${nsname_on_server}"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
log DB3 "matched_files=${matched_files}"
|
|
||||||
|
|
||||||
|
|
||||||
# note that if the addon file is invalid (or got removed after listing files
|
|
||||||
# but before we managed to match it) it will not be matched to any
|
|
||||||
# of the existing objects. So we will treat it as a new file
|
|
||||||
# and try to create its object.
|
|
||||||
for addon_path in ${addon_paths_in_files}; do
|
|
||||||
echo ${matched_files} | grep "${addon_path}" >/dev/null
|
|
||||||
if [[ $? -ne 0 ]]; then
|
|
||||||
new_files="${new_files} ${addon_path}"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
function reconcile-objects() {
|
|
||||||
local -r addon_path=$1
|
|
||||||
local -r obj_type=$2
|
|
||||||
local -r separator=$3 # name separator
|
|
||||||
match-objects ${addon_path} ${obj_type} ${separator}
|
|
||||||
|
|
||||||
log DBG "${obj_type}: nsnames_for_delete=${nsnames_for_delete}"
|
|
||||||
log DBG "${obj_type}: for_update=${for_update}"
|
|
||||||
log DBG "${obj_type}: nsnames_for_ignore=${nsnames_for_ignore}"
|
|
||||||
log DBG "${obj_type}: new_files=${new_files}"
|
|
||||||
|
|
||||||
delete-objects "${obj_type}" "${nsnames_for_delete}"
|
|
||||||
# wait for jobs below is a protection against changing the basename
|
|
||||||
# of a replication controllerm without changing the selector.
|
|
||||||
# If we don't wait, the new rc may be created before the old one is deleted
|
|
||||||
# In such case the old one will wait for all its pods to be gone, but the pods
|
|
||||||
# are created by the new replication controller.
|
|
||||||
# passing --cascade=false could solve the problem, but we want
|
|
||||||
# all orphan pods to be deleted.
|
|
||||||
wait-for-jobs
|
|
||||||
deleteResult=$?
|
|
||||||
|
|
||||||
create-objects "${obj_type}" "${new_files}"
|
|
||||||
update-objects "${obj_type}" "${for_update}"
|
|
||||||
|
|
||||||
local nsname
|
|
||||||
for nsname in ${nsnames_for_ignore}; do
|
|
||||||
log DB2 "The ${obj_type} ${nsname} is already up to date"
|
|
||||||
done
|
|
||||||
|
|
||||||
wait-for-jobs
|
|
||||||
createUpdateResult=$?
|
|
||||||
|
|
||||||
if [[ ${deleteResult} -eq 0 ]] && [[ ${createUpdateResult} -eq 0 ]]; then
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function update-addons() {
|
|
||||||
local -r addon_path=$1
|
|
||||||
# be careful, reconcile-objects uses global variables
|
|
||||||
reconcile-objects ${addon_path} ReplicationController "-" &
|
|
||||||
reconcile-objects ${addon_path} Deployment "-" &
|
|
||||||
reconcile-objects ${addon_path} DaemonSet "-" &
|
|
||||||
reconcile-objects ${addon_path} PetSet "-" &
|
|
||||||
|
|
||||||
# We don't expect names to be versioned for the following kinds, so
|
|
||||||
# we match the entire name, ignoring version suffix.
|
|
||||||
# That's why we pass an empty string as the version separator.
|
|
||||||
# If the description differs on disk, the object should be recreated.
|
|
||||||
# This is not implemented in this version.
|
|
||||||
reconcile-objects ${addon_path} Service "" &
|
|
||||||
reconcile-objects ${addon_path} PersistentVolume "" &
|
|
||||||
reconcile-objects ${addon_path} PersistentVolumeClaim "" &
|
|
||||||
reconcile-objects ${addon_path} ConfigMap "" &
|
|
||||||
|
|
||||||
wait-for-jobs
|
|
||||||
if [[ $? -eq 0 ]]; then
|
|
||||||
log INFO "== Kubernetes addon update completed successfully at $(date -Is) =="
|
|
||||||
else
|
|
||||||
log WRN "== Kubernetes addon update completed with errors at $(date -Is) =="
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# input parameters:
|
|
||||||
# $1 input directory
|
|
||||||
# $2 retry period in seconds - the script will retry api-server errors for approximately
|
|
||||||
# this amound of time (it is not very precise), at interval equal $DELAY_AFTER_ERROR_SEC.
|
|
||||||
#
|
|
||||||
|
|
||||||
if [[ $# -ne 2 ]]; then
|
|
||||||
echo "Illegal number of parameters. Usage $0 addon-dir [retry-period]" 1>&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
NUM_TRIES=$(($2 / ${DELAY_AFTER_ERROR_SEC}))
|
|
||||||
if [[ ${NUM_TRIES} -le 0 ]]; then
|
|
||||||
NUM_TRIES=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
addon_path=$1
|
|
||||||
update-addons ${addon_path}
|
|
||||||
@ -14,6 +14,18 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
# LIMITATIONS
|
||||||
|
# 1. Exit code is probably not always correct.
|
||||||
|
# 2. There are no unittests.
|
||||||
|
# 3. Will not work if the total length of paths to addons is greater than
|
||||||
|
# bash can handle. Probably it is not a problem: ARG_MAX=2097152 on GCE.
|
||||||
|
|
||||||
|
# cosmetic improvements to be done
|
||||||
|
# 1. Improve the log function; add timestamp, file name, etc.
|
||||||
|
# 2. Logging doesn't work from files that print things out.
|
||||||
|
# 3. Kubectl prints the output to stderr (the output should be captured and then
|
||||||
|
# logged)
|
||||||
|
|
||||||
# The business logic for whether a given object should be created
|
# The business logic for whether a given object should be created
|
||||||
# was already enforced by salt, and /etc/kubernetes/addons is the
|
# was already enforced by salt, and /etc/kubernetes/addons is the
|
||||||
# managed result is of that. Start everything below that directory.
|
# managed result is of that. Start everything below that directory.
|
||||||
@ -21,10 +33,66 @@ KUBECTL=${KUBECTL_BIN:-/usr/local/bin/kubectl}
|
|||||||
KUBECTL_OPTS=${KUBECTL_OPTS:-}
|
KUBECTL_OPTS=${KUBECTL_OPTS:-}
|
||||||
|
|
||||||
ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-60}
|
ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-60}
|
||||||
|
ADDON_PATH=${ADDON_PATH:-/etc/kubernetes/addons}
|
||||||
|
|
||||||
SYSTEM_NAMESPACE=kube-system
|
SYSTEM_NAMESPACE=kube-system
|
||||||
trusty_master=${TRUSTY_MASTER:-false}
|
trusty_master=${TRUSTY_MASTER:-false}
|
||||||
|
|
||||||
|
# Remember that you can't log from functions that print some output (because
|
||||||
|
# logs are also printed on stdout).
|
||||||
|
# $1 level
|
||||||
|
# $2 message
|
||||||
|
function log() {
|
||||||
|
# manage log levels manually here
|
||||||
|
|
||||||
|
# add the timestamp if you find it useful
|
||||||
|
case $1 in
|
||||||
|
DB3 )
|
||||||
|
# echo "$1: $2"
|
||||||
|
;;
|
||||||
|
DB2 )
|
||||||
|
# echo "$1: $2"
|
||||||
|
;;
|
||||||
|
DBG )
|
||||||
|
# echo "$1: $2"
|
||||||
|
;;
|
||||||
|
INFO )
|
||||||
|
echo "$1: $2"
|
||||||
|
;;
|
||||||
|
WRN )
|
||||||
|
echo "$1: $2"
|
||||||
|
;;
|
||||||
|
ERR )
|
||||||
|
echo "$1: $2"
|
||||||
|
;;
|
||||||
|
* )
|
||||||
|
echo "INVALID_LOG_LEVEL $1: $2"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# $1 command to execute.
|
||||||
|
# $2 count of tries to execute the command.
|
||||||
|
# $3 delay in seconds between two consecutive tries
|
||||||
|
function run_until_success() {
|
||||||
|
local -r command=$1
|
||||||
|
local tries=$2
|
||||||
|
local -r delay=$3
|
||||||
|
local -r command_name=$1
|
||||||
|
while [ ${tries} -gt 0 ]; do
|
||||||
|
log DBG "executing: '$command'"
|
||||||
|
# let's give the command as an argument to bash -c, so that we can use
|
||||||
|
# && and || inside the command itself
|
||||||
|
/bin/bash -c "${command}" && \
|
||||||
|
log DB3 "== Successfully executed ${command_name} at $(date -Is) ==" && \
|
||||||
|
return 0
|
||||||
|
let tries=tries-1
|
||||||
|
log WRN "== Failed to execute ${command_name} at $(date -Is). ${tries} tries remaining. =="
|
||||||
|
sleep ${delay}
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
# $1 filename of addon to start.
|
# $1 filename of addon to start.
|
||||||
# $2 count of tries to start the addon.
|
# $2 count of tries to start the addon.
|
||||||
# $3 delay in seconds between two consecutive tries
|
# $3 delay in seconds between two consecutive tries
|
||||||
@ -35,7 +103,7 @@ function start_addon() {
|
|||||||
local -r delay=$3;
|
local -r delay=$3;
|
||||||
local -r namespace=$4
|
local -r namespace=$4
|
||||||
|
|
||||||
create-resource-from-string "$(cat ${addon_filename})" "${tries}" "${delay}" "${addon_filename}" "${namespace}"
|
create_resource_from_string "$(cat ${addon_filename})" "${tries}" "${delay}" "${addon_filename}" "${namespace}"
|
||||||
}
|
}
|
||||||
|
|
||||||
# $1 string with json or yaml.
|
# $1 string with json or yaml.
|
||||||
@ -43,7 +111,7 @@ function start_addon() {
|
|||||||
# $3 delay in seconds between two consecutive tries
|
# $3 delay in seconds between two consecutive tries
|
||||||
# $4 name of this object to use when logging about it.
|
# $4 name of this object to use when logging about it.
|
||||||
# $5 namespace for this object
|
# $5 namespace for this object
|
||||||
function create-resource-from-string() {
|
function create_resource_from_string() {
|
||||||
local -r config_string=$1;
|
local -r config_string=$1;
|
||||||
local tries=$2;
|
local tries=$2;
|
||||||
local -r delay=$3;
|
local -r delay=$3;
|
||||||
@ -51,19 +119,51 @@ function create-resource-from-string() {
|
|||||||
local -r namespace=$5;
|
local -r namespace=$5;
|
||||||
while [ ${tries} -gt 0 ]; do
|
while [ ${tries} -gt 0 ]; do
|
||||||
echo "${config_string}" | ${KUBECTL} ${KUBECTL_OPTS} --namespace="${namespace}" apply -f - && \
|
echo "${config_string}" | ${KUBECTL} ${KUBECTL_OPTS} --namespace="${namespace}" apply -f - && \
|
||||||
echo "== Successfully started ${config_name} in namespace ${namespace} at $(date -Is)" && \
|
log INFO "== Successfully started ${config_name} in namespace ${namespace} at $(date -Is)" && \
|
||||||
return 0;
|
return 0;
|
||||||
let tries=tries-1;
|
let tries=tries-1;
|
||||||
echo "== Failed to start ${config_name} in namespace ${namespace} at $(date -Is). ${tries} tries remaining. =="
|
log WRN "== Failed to start ${config_name} in namespace ${namespace} at $(date -Is). ${tries} tries remaining. =="
|
||||||
sleep ${delay};
|
sleep ${delay};
|
||||||
done
|
done
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# $1 resource type.
|
||||||
|
function annotate_addons() {
|
||||||
|
local -r obj_type=$1;
|
||||||
|
|
||||||
|
# Annotate to objects already have this annotation should fail.
|
||||||
|
# Only try once for now.
|
||||||
|
${KUBECTL} ${KUBECTL_OPTS} annotate ${obj_type} --namespace=${SYSTEM_NAMESPACE} -l kubernetes.io/cluster-service=true \
|
||||||
|
kubectl.kubernetes.io/last-applied-configuration='' --overwrite=false
|
||||||
|
|
||||||
|
if [[ $? -eq 0 ]]; then
|
||||||
|
log INFO "== Annotate resources completed successfully at $(date -Is) =="
|
||||||
|
else
|
||||||
|
log WRN "== Annotate resources completed with errors at $(date -Is) =="
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# $1 enable --prune or not.
|
||||||
|
# $2 additional option for command.
|
||||||
|
function update_addons() {
|
||||||
|
local -r enable_prune=$1;
|
||||||
|
local -r additional_opt=$2;
|
||||||
|
|
||||||
|
run_until_success "${KUBECTL} ${KUBECTL_OPTS} apply --namespace=${SYSTEM_NAMESPACE} -f ${ADDON_PATH} \
|
||||||
|
--prune=${enable_prune} -l kubernetes.io/cluster-service=true --recursive ${additional_opt}" 3 5
|
||||||
|
|
||||||
|
if [[ $? -eq 0 ]]; then
|
||||||
|
log INFO "== Kubernetes addon update completed successfully at $(date -Is) =="
|
||||||
|
else
|
||||||
|
log WRN "== Kubernetes addon update completed with errors at $(date -Is) =="
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# The business logic for whether a given object should be created
|
# The business logic for whether a given object should be created
|
||||||
# was already enforced by salt, and /etc/kubernetes/addons is the
|
# was already enforced by salt, and /etc/kubernetes/addons is the
|
||||||
# managed result is of that. Start everything below that directory.
|
# managed result is of that. Start everything below that directory.
|
||||||
echo "== Kubernetes addon manager started at $(date -Is) with ADDON_CHECK_INTERVAL_SEC=${ADDON_CHECK_INTERVAL_SEC} =="
|
log INFO "== Kubernetes addon manager started at $(date -Is) with ADDON_CHECK_INTERVAL_SEC=${ADDON_CHECK_INTERVAL_SEC} =="
|
||||||
|
|
||||||
# Create the namespace that will be used to host the cluster-level add-ons.
|
# Create the namespace that will be used to host the cluster-level add-ons.
|
||||||
start_addon /opt/namespace.yaml 100 10 "" &
|
start_addon /opt/namespace.yaml 100 10 "" &
|
||||||
@ -75,22 +175,43 @@ while [ -z "${token_found}" ]; do
|
|||||||
token_found=$(${KUBECTL} ${KUBECTL_OPTS} get --namespace="${SYSTEM_NAMESPACE}" serviceaccount default -o go-template="{{with index .secrets 0}}{{.name}}{{end}}" || true)
|
token_found=$(${KUBECTL} ${KUBECTL_OPTS} get --namespace="${SYSTEM_NAMESPACE}" serviceaccount default -o go-template="{{with index .secrets 0}}{{.name}}{{end}}" || true)
|
||||||
done
|
done
|
||||||
|
|
||||||
echo "== default service account in the ${SYSTEM_NAMESPACE} namespace has token ${token_found} =="
|
log INFO "== Default service account in the ${SYSTEM_NAMESPACE} namespace has token ${token_found} =="
|
||||||
|
|
||||||
# Create admission_control objects if defined before any other addon services. If the limits
|
# Create admission_control objects if defined before any other addon services. If the limits
|
||||||
# are defined in a namespace other than default, we should still create the limits for the
|
# are defined in a namespace other than default, we should still create the limits for the
|
||||||
# default namespace.
|
# default namespace.
|
||||||
for obj in $(find /etc/kubernetes/admission-controls \( -name \*.yaml -o -name \*.json \)); do
|
for obj in $(find /etc/kubernetes/admission-controls \( -name \*.yaml -o -name \*.json \)); do
|
||||||
start_addon "${obj}" 100 10 default &
|
start_addon "${obj}" 100 10 default &
|
||||||
echo "++ obj ${obj} is created ++"
|
log INFO "++ obj ${obj} is created ++"
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# Fake the "kubectl.kubernetes.io/last-applied-configuration" annotation on old resources
|
||||||
|
# in order to clean them up by `kubectl apply --prune`.
|
||||||
|
# RCs have to be annotated for 1.4->1.5 upgrade, because we are migrating from RCs to deployments for all default addons.
|
||||||
|
# Other types resources will also need this fake annotation if their names are changed,
|
||||||
|
# otherwise they would be leaked during upgrade.
|
||||||
|
log INFO "== Annotating the old addon resources at $(date -Is) =="
|
||||||
|
annotate_addons ReplicationController
|
||||||
|
|
||||||
|
# Create new addon resources by apply (with --prune=false).
|
||||||
|
# The old RCs will not fight for pods created by new Deployments with the same label because the `controllerRef` feature.
|
||||||
|
# The new Deployments will not fight for pods created by old RCs with the same label because the additional `pod-template-hash` label.
|
||||||
|
# Apply will fail if some fields are modified but not are allowed, in that case should bump up addon version and name (e.g. handle externally).
|
||||||
|
log INFO "== Executing apply to spin up new addon resources at $(date -Is) =="
|
||||||
|
update_addons false
|
||||||
|
|
||||||
|
# Wait for new addons to be spinned up before delete old resources
|
||||||
|
log INFO "== Wait for addons to be spinned up at $(date -Is) =="
|
||||||
|
sleep ${ADDON_CHECK_INTERVAL_SEC}
|
||||||
|
|
||||||
|
# Start the apply loop.
|
||||||
# Check if the configuration has changed recently - in case the user
|
# Check if the configuration has changed recently - in case the user
|
||||||
# created/updated/deleted the files on the master.
|
# created/updated/deleted the files on the master.
|
||||||
|
log INFO "== Entering periodical apply loop at $(date -Is) =="
|
||||||
while true; do
|
while true; do
|
||||||
start_sec=$(date +"%s")
|
start_sec=$(date +"%s")
|
||||||
#kube-addon-update.sh must be deployed in the same directory as this file
|
# Only print stderr for the readability of logging
|
||||||
`dirname $0`/kube-addon-update.sh /etc/kubernetes/addons ${ADDON_CHECK_INTERVAL_SEC}
|
update_addons true ">/dev/null"
|
||||||
end_sec=$(date +"%s")
|
end_sec=$(date +"%s")
|
||||||
len_sec=$((${end_sec}-${start_sec}))
|
len_sec=$((${end_sec}-${start_sec}))
|
||||||
# subtract the time passed from the sleep time
|
# subtract the time passed from the sleep time
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user