cluster/coreos: update to gci based implementation

This update includes significant refactoring. It moves almost all of the
logic into bash scripts, modeled after the `gci` cluster scripts.

The primary differences between the two are the following:
1. Use of the `/opt/kubernetes` directory over `/home/kubernetes`
2. Support for rkt as a runtime
3. No use of logrotate
4. No use of `/etc/default/`
5. No logic related to noexec mounts or gci-specific firewall-stuff
This commit is contained in:
Euan Kemp 2016-11-01 14:29:04 -07:00 committed by Euan Kemp
parent e2644bb442
commit 13afe18ab4
11 changed files with 276 additions and 505 deletions

View File

@ -331,14 +331,12 @@ function kube::release::package_kube_manifests_tarball() {
cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh"
cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${dst_dir}/gci-mounter"
cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"
cp "${KUBE_ROOT}/cluster/gce/coreos/configure-helper.sh" "${dst_dir}/coreos-configure-helper.sh"
cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}"
local objects
objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${dst_dir}"
# This is for coreos only. ContainerVM, GCI, or Trusty does not use it.
cp -r "${KUBE_ROOT}/cluster/gce/coreos/kube-manifests"/* "${release_stage}/"
kube::release::clean_cruft
local package_name="${RELEASE_DIR}/kubernetes-manifests.tar.gz"

View File

@ -1,4 +1,3 @@
# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:

View File

@ -1,4 +1,3 @@
# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml
apiVersion: v1
kind: Service
metadata:

View File

@ -1,11 +1,8 @@
# Container-VM Image
# CoreOS image
[Container-VM Image](https://cloud.google.com/compute/docs/containers/vm-image/)
is a container-optimized OS image for the Google Cloud Platform (GCP). It is
primarily for running Google services on GCP. Unlike the open preview version
of container-vm, the new Container-VM Image is based on the open source
ChromiumOS project, allowing us greater control over the build management,
security compliance, and customizations for GCP.
The [CoreOS operating system](https://coreos.com/why/) is a Linux distribution optimized for running containers securely at scale.
CoreOS provides [an image](https://coreos.com/os/docs/latest/booting-on-google-compute-engine.html) for Google Cloud Platform (GCP).
This folder contains configuration and tooling to allow kube-up to create a Kubernetes cluster on Google Cloud Platform running on the official CoreOS image.
[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/gce/gci/README.md?pixel)]()
[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/gce/coreos/README.md?pixel)]()

322
cluster/gce/coreos/configure-helper.sh Normal file → Executable file
View File

@ -19,37 +19,12 @@
# TODO: this script duplicates templating logic from cluster/saltbase/salt
# using sed. It should use an actual template parser on the manifest
# files.
# files, or the manifest files should not be templated salt
set -o errexit
set -o nounset
set -o pipefail
function setup-os-params {
# Reset core_pattern. On GCI, the default core_pattern pipes the core dumps to
# /sbin/crash_reporter which is more restrictive in saving crash dumps. So for
# now, set a generic core_pattern that users can work with.
echo "core.%e.%p.%t" > /proc/sys/kernel/core_pattern
}
function config-ip-firewall {
echo "Configuring IP firewall rules"
# The GCI image has host firewall which drop most inbound/forwarded packets.
# We need to add rules to accept all TCP/UDP/ICMP packets.
if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then
echo "Add rules to accept all inbound TCP/UDP/ICMP packets"
iptables -A INPUT -w -p TCP -j ACCEPT
iptables -A INPUT -w -p UDP -j ACCEPT
iptables -A INPUT -w -p ICMP -j ACCEPT
fi
if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then
echo "Add rules to accept all forwarded TCP/UDP/ICMP packets"
iptables -A FORWARD -w -p TCP -j ACCEPT
iptables -A FORWARD -w -p UDP -j ACCEPT
iptables -A FORWARD -w -p ICMP -j ACCEPT
fi
}
function create-dirs {
echo "Creating required directories"
mkdir -p /var/lib/kubelet
@ -59,6 +34,13 @@ function create-dirs {
fi
}
# Create directories referenced in the kube-controller-manager manifest for
# bindmounts. This is used under the rkt runtime to work around
# https://github.com/kubernetes/kubernetes/issues/26816
function create-kube-controller-manager-dirs {
mkdir -p /etc/srv/kubernetes /var/ssl /etc/{ssl,openssl,pki}
}
# Formats the given device ($1) if needed and mounts it at given mount point
# ($2).
function safe-format-and-mount() {
@ -92,51 +74,6 @@ function ensure-local-ssds() {
done
}
# Installs logrotate configuration files
function setup-logrotate() {
mkdir -p /etc/logrotate.d/
cat >/etc/logrotate.d/docker-containers <<EOF
/var/lib/docker/containers/*/*-json.log {
rotate 5
copytruncate
missingok
notifempty
compress
maxsize 10M
daily
dateext
dateformat -%Y%m%d-%s
create 0644 root root
}
EOF
# Configure log rotation for all logs in /var/log, which is where k8s services
# are configured to write their log files. Whenever logrotate is ran, this
# config will:
# * rotate the log file if its size is > 100Mb OR if one day has elapsed
# * save rotated logs into a gzipped timestamped backup
# * log file timestamp (controlled by 'dateformat') includes seconds too. This
# ensures that logrotate can generate unique logfiles during each rotation
# (otherwise it skips rotation if 'maxsize' is reached multiple times in a
# day).
# * keep only 5 old (rotated) logs, and will discard older logs.
cat > /etc/logrotate.d/allvarlogs <<EOF
/var/log/*.log {
rotate 5
copytruncate
missingok
notifempty
compress
maxsize 100M
daily
dateext
dateformat -%Y%m%d-%s
create 0644 root root
}
EOF
}
# Finds the master PD device; returns it in MASTER_PD_DEVICE
function find-master-pd {
MASTER_PD_DEVICE=""
@ -387,8 +324,8 @@ function create-master-etcd-auth {
fi
}
function assemble-docker-flags {
echo "Assemble docker command line flags"
function configure-docker-daemon {
echo "Configuring the Docker daemon"
local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false"
if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
docker_opts+=" --log-level=debug"
@ -411,28 +348,17 @@ function assemble-docker-flags {
docker_opts+=" --registry-mirror=${DOCKER_REGISTRY_MIRROR_URL}"
fi
echo "DOCKER_OPTS=\"${docker_opts} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker
if [[ "${use_net_plugin}" == "true" ]]; then
# If using a network plugin, extend the docker configuration to always remove
# the network checkpoint to avoid corrupt checkpoints.
# (https://github.com/docker/docker/issues/18283).
echo "Extend the default docker.service configuration"
mkdir -p /etc/systemd/system/docker.service.d
cat <<EOF >/etc/systemd/system/docker.service.d/01network.conf
mkdir -p /etc/systemd/system/docker.service.d/
local kubernetes_conf_dropin="/etc/systemd/system/docker.service.d/00_kubelet.conf"
cat > "${kubernetes_conf_dropin}" <<EOF
[Service]
ExecStartPre=/bin/sh -x -c "rm -rf /var/lib/docker/network"
Environment="DOCKER_OPTS=${docker_opts} ${EXTRA_DOCKER_OPTS:-}"
EOF
systemctl daemon-reload
# If using a network plugin, we need to explicitly restart docker daemon, because
# kubelet will not do it.
echo "Docker command line is updated. Restart docker to pick it up"
systemctl restart docker
fi
# Always restart to get the cbr0 change
echo "Docker daemon options updated. Restarting docker..."
systemctl daemon-reload
systemctl restart docker
}
# A helper function for loading a docker image. It keeps trying up to 5 times.
#
# $1: Full path of the docker image
@ -443,15 +369,37 @@ function try-load-docker-image {
set +e
local -r max_attempts=5
local -i attempt_num=1
until timeout 30 docker load -i "${img}"; do
if [[ "${attempt_num}" == "${max_attempts}" ]]; then
echo "Fail to load docker image file ${img} after ${max_attempts} retries. Exit!!"
exit 1
else
attempt_num=$((attempt_num+1))
if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then
for attempt_num in $(seq 1 "${max_attempts}"); do
local aci_tmpdir="$(mktemp -t -d docker2aci.XXXXX)"
(cd "${aci_tmpdir}"; timeout 40 "${DOCKER2ACI_BIN}" "$1")
local aci_success=$?
timeout 40 "${RKT_BIN}" fetch --insecure-options=image "${aci_tmpdir}"/*.aci
local fetch_success=$?
rm -f "${aci_tmpdir}"/*.aci
rmdir "${aci_tmpdir}"
if [[ ${fetch_success} && ${aci_success} ]]; then
echo "rkt: Loaded ${img}"
break
fi
if [[ "${attempt}" == "${max_attempts}" ]]; then
echo "rkt: Failed to load image file ${img} after ${max_attempts} retries."
exit 1
fi
sleep 5
fi
done
done
else
until timeout 30 docker load -i "${img}"; do
if [[ "${attempt_num}" == "${max_attempts}" ]]; then
echo "Fail to load docker image file ${img} after ${max_attempts} retries."
exit 1
else
attempt_num=$((attempt_num+1))
sleep 5
fi
done
fi
# Re-enable errexit.
set -e
}
@ -476,19 +424,6 @@ function start-kubelet {
echo "Start kubelet"
local kubelet_bin="${KUBE_HOME}/bin/kubelet"
local -r version="$("${kubelet_bin}" --version=true | cut -f2 -d " ")"
local -r builtin_kubelet="/usr/bin/kubelet"
if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
# Determine which binary to use on test clusters. We use the built-in
# version only if the downloaded version is the same as the built-in
# version. This allows GCI to run some of the e2e tests to qualify the
# built-in kubelet.
if [[ -x "${builtin_kubelet}" ]]; then
local -r builtin_version="$("${builtin_kubelet}" --version=true | cut -f2 -d " ")"
if [[ "${builtin_version}" == "${version}" ]]; then
kubelet_bin="${builtin_kubelet}"
fi
fi
fi
echo "Using kubelet binary at ${kubelet_bin}"
local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}"
flags+=" --allow-privileged=true"
@ -498,7 +433,6 @@ function start-kubelet {
flags+=" --cluster-dns=${DNS_SERVER_IP}"
flags+=" --cluster-domain=${DNS_DOMAIN}"
flags+=" --config=/etc/kubernetes/manifests"
flags+=" --experimental-mounter-path=${KUBE_HOME}/bin/mounter"
flags+=" --experimental-check-node-capabilities-before-mount=true"
if [[ -n "${KUBELET_PORT:-}" ]]; then
@ -527,9 +461,9 @@ function start-kubelet {
# Network plugin
if [[ -n "${NETWORK_PROVIDER:-}" ]]; then
if [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then
flags+=" --cni-bin-dir=/home/kubernetes/bin"
flags+=" --cni-bin-dir=/opt/kubernetes/bin"
else
flags+=" --network-plugin-dir=/home/kubernetes/bin"
flags+=" --network-plugin-dir=/opt/kubernetes/bin"
fi
flags+=" --network-plugin=${NETWORK_PROVIDER}"
fi
@ -552,8 +486,13 @@ function start-kubelet {
if [[ -n "${FEATURE_GATES:-}" ]]; then
flags+=" --feature-gates=${FEATURE_GATES}"
fi
if [[ -n "${CONTAINER_RUNTIME:-}" ]]; then
flags+=" --container-runtime=${CONTAINER_RUNTIME}"
flags+=" --rkt-path=${KUBE_HOME}/bin/rkt"
flags+=" --rkt-stage1-image=${RKT_STAGE1_IMAGE}"
fi
local -r kubelet_env_file="/etc/default/kubelet"
local -r kubelet_env_file="/etc/kubelet-env"
echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}"
# Write the systemd service file for kubelet.
@ -600,7 +539,7 @@ function start-kube-proxy {
if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then
kube_docker_registry=${KUBE_DOCKER_REGISTRY}
fi
local -r kube_proxy_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-proxy.docker_tag)
local -r kube_proxy_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-proxy.docker_tag)
local api_servers="--master=https://${KUBERNETES_MASTER_NAME}"
local params="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=2"}"
if [[ -n "${FEATURE_GATES:-}" ]]; then
@ -618,6 +557,18 @@ function start-kube-proxy {
if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then
sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file}
fi
if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then
# Work arounds for https://github.com/coreos/rkt/issues/3245 and https://github.com/coreos/rkt/issues/3264
# This is an incredibly hacky workaround. It's fragile too. If the kube-proxy command changes too much, this breaks
# TODO, this could be done much better in many other places, such as an
# init script within the container, or even within kube-proxy's code.
local extra_workaround_cmd="ln -sf /proc/self/mounts /etc/mtab; \
mount -o remount,rw /proc; \
mount -o remount,rw /proc/sys; \
mount -o remount,rw /sys; "
sed -i -e "s@-\\s\\+kube-proxy@- ${extra_workaround_cmd} kube-proxy@g" "${src_file}"
fi
cp "${src_file}" /etc/kubernetes/manifests
}
@ -629,7 +580,7 @@ function start-kube-proxy {
# $4: value for variable 'cpulimit'
# $5: pod name, which should be either etcd or etcd-events
function prepare-etcd-manifest {
local host_name=$(hostname)
local host_name=$(hostname -s)
local etcd_cluster=""
local cluster_state="new"
local etcd_protocol="http"
@ -671,6 +622,7 @@ function prepare-etcd-manifest {
else
sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${temp_file}"
fi
sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}"
sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}"
if [[ -n "${ETCD_VERSION:-}" ]]; then
@ -862,7 +814,7 @@ function start-kube-apiserver {
src_file="${src_dir}/kube-apiserver.manifest"
remove-salt-config-comments "${src_file}"
# Evaluate variables.
local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
local -r kube_apiserver_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}"
@ -927,7 +879,7 @@ function start-kube-controller-manager {
if [[ -n "${FEATURE_GATES:-}" ]]; then
params+=" --feature-gates=${FEATURE_GATES}"
fi
local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
local -r kube_rc_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest"
remove-salt-config-comments "${src_file}"
@ -1182,63 +1134,74 @@ function start-rescheduler {
fi
}
# Setup working directory for kubelet.
function setup-kubelet-dir {
echo "Making /var/lib/kubelet executable for kubelet"
mount -B /var/lib/kubelet /var/lib/kubelet/
mount -B -o remount,exec,suid,dev /var/lib/kubelet
}
# Install and setup rkt
# TODO(euank): There should be a toggle to use the distro-provided rkt binary
# Sets the following variables:
# RKT_BIN: the path to the rkt binary
function setup-rkt {
local rkt_bin="${KUBE_HOME}/bin/rkt"
if [[ -x "${rkt_bin}" ]]; then
# idempotency, skip downloading this time
# TODO(euank): this might get in the way of updates, but 'file busy'
# because of rkt-api would too
RKT_BIN="${rkt_bin}"
return
fi
mkdir -p /etc/rkt "${KUBE_HOME}/download/"
local rkt_tar="${KUBE_HOME}/download/rkt.tar.gz"
local rkt_tmpdir=$(mktemp -d "${KUBE_HOME}/rkt_download.XXXXX")
curl --retry 5 --retry-delay 3 --fail --silent --show-error \
--location --create-dirs --output "${rkt_tar}" \
https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
tar --strip-components=1 -xf "${rkt_tar}" -C "${rkt_tmpdir}" --overwrite
mv "${rkt_tmpdir}/rkt" "${rkt_bin}"
if [[ ! -x "${rkt_bin}" ]]; then
echo "Could not download requested rkt binary"
exit 1
fi
RKT_BIN="${rkt_bin}"
# Cache rkt stage1 images for speed
"${RKT_BIN}" fetch --insecure-options=image "${rkt_tmpdir}"/*.aci
rm -rf "${rkt_tmpdir}"
function reset-motd {
# kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
local -r version="$("${KUBE_HOME}"/bin/kubelet --version=true | cut -f2 -d " ")"
# This logic grabs either a release tag (v1.2.1 or v1.2.1-alpha.1),
# or the git hash that's in the build info.
local gitref="$(echo "${version}" | sed -r "s/(v[0-9]+\.[0-9]+\.[0-9]+)(-[a-z]+\.[0-9]+)?.*/\1\2/g")"
local devel=""
if [[ "${gitref}" != "${version}" ]]; then
devel="
Note: This looks like a development version, which might not be present on GitHub.
If it isn't, the closest tag is at:
https://github.com/kubernetes/kubernetes/tree/${gitref}
"
gitref="${version//*+/}"
fi
cat > /etc/motd <<EOF
cat > /etc/systemd/system/rkt-api.service <<EOF
[Unit]
Description=rkt api service
Documentation=http://github.com/coreos/rkt
After=network.target
Welcome to Kubernetes ${version}!
You can find documentation for Kubernetes at:
http://docs.kubernetes.io/
The source for this release can be found at:
/home/kubernetes/kubernetes-src.tar.gz
Or you can download it at:
https://storage.googleapis.com/kubernetes-release/release/${version}/kubernetes-src.tar.gz
It is based on the Kubernetes source at:
https://github.com/kubernetes/kubernetes/tree/${gitref}
${devel}
For Kubernetes copyright and licensing information, see:
/home/kubernetes/LICENSES
[Service]
ExecStart=${RKT_BIN} api-service --listen=127.0.0.1:15441
[Install]
WantedBy=multi-user.target
EOF
systemctl enable rkt-api.service
systemctl start rkt-api.service
}
function override-kubectl {
echo "overriding kubectl"
echo "export PATH=${KUBE_HOME}/bin:\$PATH" > /etc/profile.d/kube_env.sh
}
function pre-warm-mounter {
echo "prewarming mounter"
${KUBE_HOME}/bin/mounter &> /dev/null
# Install docker2aci, needed to load server images if using rkt runtime
# This should be removed once rkt can fetch on-disk docker tarballs directly
# Sets the following variables:
# DOCKER2ACI_BIN: the path to the docker2aci binary
function install-docker2aci {
local tar_path="${KUBE_HOME}/download/docker2aci.tar.gz"
local tmp_path="${KUBE_HOME}/docker2aci"
mkdir -p "${KUBE_HOME}/download/" "${tmp_path}"
curl --retry 5 --retry-delay 3 --fail --silent --show-error \
--location --create-dirs --output "${tar_path}" \
https://github.com/appc/docker2aci/releases/download/v0.14.0/docker2aci-v0.14.0.tar.gz
tar --strip-components=1 -xf "${tar_path}" -C "${tmp_path}" --overwrite
DOCKER2ACI_BIN="${KUBE_HOME}/bin/docker2aci"
mv "${tmp_path}/docker2aci" "${DOCKER2ACI_BIN}"
}
########### Main Function ###########
echo "Start to configure instance for kubernetes"
KUBE_HOME="/home/kubernetes"
# Note: this name doesn't make as much sense here as in gci where it's actually
# /home/kubernetes, but for ease of diff-ing, retain the same variable name
KUBE_HOME="/opt/kubernetes"
if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
exit 1
@ -1253,12 +1216,13 @@ if [[ -n "${KUBE_USER:-}" ]]; then
fi
fi
setup-os-params
config-ip-firewall
# KUBERNETES_CONTAINER_RUNTIME is set by the `kube-env` file, but it's a bit of a mouthful
if [[ "${CONTAINER_RUNTIME:-}" == "" ]]; then
CONTAINER_RUNTIME="${KUBERNETES_CONTAINER_RUNTIME:-docker}"
fi
create-dirs
setup-kubelet-dir
ensure-local-ssds
setup-logrotate
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
mount-master-pd
create-master-auth
@ -1269,10 +1233,16 @@ else
create-kubeproxy-kubeconfig
fi
override-kubectl
# Run the containerized mounter once to pre-cache the container image.
pre-warm-mounter
assemble-docker-flags
if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then
systemctl stop docker
systemctl disable docker
setup-rkt
install-docker2aci
create-kube-controller-manager-dirs
else
configure-docker-daemon
fi
load-docker-images
start-kubelet
@ -1298,5 +1268,5 @@ else
start-image-puller
fi
fi
reset-motd
start-fluentd
echo "Done for the configuration for kubernetes"

63
cluster/gce/coreos/configure.sh Normal file → Executable file
View File

@ -14,30 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Due to the GCE custom metadata size limit, we split the entire script into two
# files configure.sh and configure-helper.sh. The functionality of downloading
# kubernetes configuration, manifests, docker images, and binary files are
# put in configure.sh, which is uploaded via GCE custom metadata.
set -o errexit
set -o nounset
set -o pipefail
function set-broken-motd {
cat > /etc/motd <<EOF
Broken (or in progress) Kubernetes node setup! Check the cluster initialization status
using the following commands.
Master instance:
- sudo systemctl status kube-master-installation
- sudo systemctl status kube-master-configuration
Node instance:
- sudo systemctl status kube-node-installation
- sudo systemctl status kube-node-configuration
EOF
}
function download-kube-env {
# Fetch kube-env from GCE metadata server.
local -r tmp_kube_env="/tmp/kube-env.yaml"
@ -46,11 +26,7 @@ function download-kube-env {
-o "${tmp_kube_env}" \
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
# Convert the yaml format file into a shell-style file.
eval $(python -c '''
import pipes,sys,yaml
for k,v in yaml.load(sys.stdin).iteritems():
print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
''' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env")
sed 's/: /=/' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env"
rm -f "${tmp_kube_env}"
}
@ -65,6 +41,7 @@ function validate-hash {
fi
}
# Retry a download until we get it. Takes a hash and a set of URLs.
#
# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
@ -98,24 +75,8 @@ function split-commas {
echo $1 | tr "," "\n"
}
function install-gci-mounter-tools {
local -r rkt_version="v1.18.0"
local -r gci_mounter_version="v2"
local -r rkt_binary_sha1="75fc8f29c79bc9e505f3e7f6e8fadf2425c21967"
local -r rkt_stage1_fly_sha1="474df5a1f934960ba669b360ab713d0a54283091"
local -r gci_mounter_sha1="851e841d8640d6a05e64e22c493f5ac3c4cba561"
download-or-bust "${rkt_binary_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/rkt"
download-or-bust "${rkt_stage1_fly_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/stage1-fly.aci"
download-or-bust "${gci_mounter_sha1}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-${gci_mounter_version}.aci"
local -r rkt_dst="${KUBE_HOME}/bin/"
mv "${KUBE_HOME}/rkt" "${rkt_dst}/rkt"
mv "${KUBE_HOME}/stage1-fly.aci" "${rkt_dst}/stage1-fly.aci"
mv "${KUBE_HOME}/gci-mounter-${gci_mounter_version}.aci" "${rkt_dst}/gci-mounter-${gci_mounter_version}.aci"
chmod a+x "${rkt_dst}/rkt"
}
# Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
# and places them into suitable directories. Files are placed in /home/kubernetes.
# and places them into suitable directories. Files are placed in /opt/kubernetes.
function install-kube-binary-config {
cd "${KUBE_HOME}"
local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") )
@ -186,14 +147,9 @@ function install-kube-binary-config {
find "${dst_dir}" -name \*.manifest -or -name \*.json | \
xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
fi
cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter"
cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh"
cp "${dst_dir}/kubernetes/gci-trusty/coreos-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
chmod -R 755 "${kube_bin}"
# Install gci mounter related artifacts to allow mounting storage volumes in GCI
install-gci-mounter-tools
# Clean up.
rm -rf "${KUBE_HOME}/kubernetes"
rm -f "${KUBE_HOME}/${server_binary_tar}"
@ -204,10 +160,17 @@ function install-kube-binary-config {
######### Main Function ##########
echo "Start to install kubernetes files"
set-broken-motd
KUBE_HOME="/home/kubernetes"
KUBE_HOME="/opt/kubernetes"
mkdir -p "${KUBE_HOME}"
download-kube-env
source "${KUBE_HOME}/kube-env"
install-kube-binary-config
echo "Done for installing kubernetes files"
# On CoreOS, the hosts is in /usr/share/baselayout/hosts
# So we need to manually populdate the hosts file here on gce.
echo "127.0.0.1 localhost" >> /etc/hosts
echo "::1 localhost" >> /etc/hosts
echo "Configuring hostname"
hostnamectl set-hostname $(hostname | cut -f1 -d.)

View File

@ -14,19 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# A library of helper functions and constant for GCI distro
# A library of helper functions and constants for the CoreOS distro
# Creates the GCI specific metadata files if they do not exit.
# Assumed var
# KUBE_TEMP
function ensure-gci-metadata-files {
if [[ ! -f "${KUBE_TEMP}/gci-update.txt" ]]; then
echo -n "update_disabled" > "${KUBE_TEMP}/gci-update.txt"
fi
if [[ ! -f "${KUBE_TEMP}/gci-ensure-gke-docker.txt" ]]; then
echo -n "true" > "${KUBE_TEMP}/gci-ensure-gke-docker.txt"
fi
if [[ ! -f "${KUBE_TEMP}/gci-docker-version.txt" ]]; then
echo -n "${GCI_DOCKER_VERSION:-}" > "${KUBE_TEMP}/gci-docker-version.txt"
fi
}
# This file intentionally left blank

View File

@ -14,8 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# A library of helper functions and constant for GCI distro
source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
# A library of helper functions and constant for coreos os distro
source "${KUBE_ROOT}/cluster/gce/coreos/helper.sh"
# create-master-instance creates the master instance. If called with
# an argument, the argument is used as the name to a reserved IP
@ -35,7 +35,6 @@ function create-master-instance {
[[ -n ${1:-} ]] && address_opt="--address ${1}"
write-master-env
ensure-gci-metadata-files
create-master-instance-internal "${MASTER_NAME}" "${address_opt}"
}
@ -60,9 +59,6 @@ function replicate-master-instance() {
echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml
get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt"
get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt"
get-metadata "${existing_master_zone}" "${existing_master_name}" gci-ensure-gke-docker > "${KUBE_TEMP}/gci-ensure-gke-docker.txt"
get-metadata "${existing_master_zone}" "${existing_master_name}" gci-docker-version > "${KUBE_TEMP}/gci-docker-version.txt"
create-master-instance-internal "${REPLICA_NAME}"
}
@ -89,9 +85,9 @@ function create-master-instance-internal() {
--scopes "storage-ro,compute-rw,monitoring,logging-write" \
--can-ip-forward \
--metadata-from-file \
"kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" \
"kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/coreos/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
--disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
--boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \
--boot-disk-size "${MASTER_ROOT_DISK_SIZE:-30}" \
${preemptible_master}
}

View File

@ -1,124 +1,56 @@
#cloud-config
write_files:
- path: /etc/systemd/system/kube-master-installation.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Download and install k8s binaries and configurations
After=network-online.target
coreos:
update:
reboot-strategy: off
units:
- name: locksmithd.service
mask: true
- name: kube-master-installation.service
command: start
content: |
[Unit]
Description=Download and install k8s binaries and configurations
After=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /home/kubernetes/bin
ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin
ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh
ExecStart=/home/kubernetes/bin/configure.sh
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /opt/kubernetes/bin
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /opt/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure.sh
ExecStart=/opt/kubernetes/bin/configure.sh
[Install]
WantedBy=kubernetes.target
[Install]
WantedBy=kubernetes.target
- name: kube-master-configuration.service
command: start
content: |
[Unit]
Description=Configure kubernetes master
After=kube-master-installation.service
- path: /etc/systemd/system/kube-master-configuration.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Configure kubernetes master
After=kube-master-installation.service
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure-helper.sh
ExecStart=/opt/kubernetes/bin/configure-helper.sh
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
ExecStart=/home/kubernetes/bin/configure-helper.sh
[Install]
WantedBy=kubernetes.target
- name: kubernetes.target
enable: true
command: start
content: |
[Unit]
Description=Kubernetes
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-docker-monitor.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes health monitoring for docker
After=kube-master-configuration.service
[Service]
Restart=always
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh docker
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kubelet-monitor.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes health monitoring for kubelet
After=kube-master-configuration.service
[Service]
Restart=always
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-logrotate.timer
permissions: 0644
owner: root
content: |
[Unit]
Description=Hourly kube-logrotate invocation
[Timer]
OnCalendar=hourly
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-logrotate.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes log rotation
After=kube-master-configuration.service
[Service]
Type=oneshot
ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kubernetes.target
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes
runcmd:
- systemctl daemon-reload
- systemctl enable kube-master-installation.service
- systemctl enable kube-master-configuration.service
- systemctl enable kube-docker-monitor.service
- systemctl enable kubelet-monitor.service
- systemctl enable kube-logrotate.timer
- systemctl enable kube-logrotate.service
- systemctl start kubernetes.target
[Install]
WantedBy=multi-user.target
- name: docker.service
drop-ins:
- name: "use-cgroupfs-driver.conf"
# This is required for setting cgroup parent in the current ~1.4 per-pod cgroup impl
content: |
[Service]
Environment="DOCKER_CGROUPS=--exec-opt native.cgroupdriver="

View File

@ -14,19 +14,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# A library of helper functions and constant for GCI distro
source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
# A library of helper functions and constant for the CoreOS distro
source "${KUBE_ROOT}/cluster/gce/coreos/helper.sh"
# $1: template name (required).
function create-node-instance-template {
local template_name="$1"
ensure-gci-metadata-files
create-node-template "$template_name" "${scope_flags[*]}" \
"kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
"user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml" \
"configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" \
"cluster-name=${KUBE_TEMP}/cluster-name.txt" \
"gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \
"gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" \
"gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt"
"user-data=${KUBE_ROOT}/cluster/gce/coreos/node.yaml" \
"configure-sh=${KUBE_ROOT}/cluster/gce/coreos/configure.sh" \
"cluster-name=${KUBE_TEMP}/cluster-name.txt"
# TODO(euank): We should include update-strategy here. We should also switch to ignition
}

View File

@ -1,124 +1,56 @@
#cloud-config
write_files:
- path: /etc/systemd/system/kube-node-installation.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Download and install k8s binaries and configurations
After=network-online.target
coreos:
update:
reboot-strategy: off
units:
- name: locksmithd.service
mask: true
- name: kube-node-installation.service
command: start
content: |
[Unit]
Description=Download and install k8s binaries and configurations
After=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /home/kubernetes/bin
ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin
ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh
ExecStart=/home/kubernetes/bin/configure.sh
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /opt/kubernetes/bin
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /opt/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure.sh
ExecStart=/opt/kubernetes/bin/configure.sh
[Install]
WantedBy=kubernetes.target
[Install]
WantedBy=kubernetes.target
- name: kube-node-configuration.service
command: start
content: |
[Unit]
Description=Configure kubernetes master
After=kube-node-installation.service
- path: /etc/systemd/system/kube-node-configuration.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Configure kubernetes node
After=kube-node-installation.service
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure-helper.sh
ExecStart=/opt/kubernetes/bin/configure-helper.sh
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
ExecStart=/home/kubernetes/bin/configure-helper.sh
[Install]
WantedBy=kubernetes.target
- name: kubernetes.target
enable: true
command: start
content: |
[Unit]
Description=Kubernetes
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-docker-monitor.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes health monitoring for docker
After=kube-node-configuration.service
[Service]
Restart=always
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh docker
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kubelet-monitor.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes health monitoring for kubelet
After=kube-node-configuration.service
[Service]
Restart=always
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-logrotate.timer
permissions: 0644
owner: root
content: |
[Unit]
Description=Hourly kube-logrotate invocation
[Timer]
OnCalendar=hourly
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-logrotate.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes log rotation
After=kube-node-configuration.service
[Service]
Type=oneshot
ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kubernetes.target
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes
runcmd:
- systemctl daemon-reload
- systemctl enable kube-node-installation.service
- systemctl enable kube-node-configuration.service
- systemctl enable kube-docker-monitor.service
- systemctl enable kubelet-monitor.service
- systemctl enable kube-logrotate.timer
- systemctl enable kube-logrotate.service
- systemctl start kubernetes.target
[Install]
WantedBy=multi-user.target
- name: docker.service
drop-ins:
- name: "use-cgroupfs-driver.conf"
# This is required for setting cgroup parent in the current ~1.4 per-pod cgroup impl
content: |
[Service]
Environment="DOCKER_CGROUPS=--exec-opt native.cgroupdriver="