mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Trusty: Avoid reaching GCE custom metadata size limit
This commit is contained in:
Andy Zheng
parent
d5316c21ef
commit
0a8e68fb49
@ -946,6 +946,7 @@ function kube::release::package_kube_manifests_tarball() {
|
|||||||
cp "${salt_dir}/kube-addons/namespace.yaml" "${release_stage}/trusty"
|
cp "${salt_dir}/kube-addons/namespace.yaml" "${release_stage}/trusty"
|
||||||
cp "${salt_dir}/kube-addons/kube-addons.sh" "${release_stage}/trusty"
|
cp "${salt_dir}/kube-addons/kube-addons.sh" "${release_stage}/trusty"
|
||||||
cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${release_stage}/trusty"
|
cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${release_stage}/trusty"
|
||||||
|
cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${release_stage}/trusty"
|
||||||
cp -r "${salt_dir}/kube-admission-controls/limit-range" "${release_stage}/trusty"
|
cp -r "${salt_dir}/kube-admission-controls/limit-range" "${release_stage}/trusty"
|
||||||
local objects
|
local objects
|
||||||
objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
|
objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
|
||||||
|
|||||||
671
cluster/gce/trusty/configure-helper.sh
Normal file
671
cluster/gce/trusty/configure-helper.sh
Normal file
@ -0,0 +1,671 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2015 The Kubernetes Authors All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# This script contains functions for configuring instances to run kubernetes
|
||||||
|
# master and nodes. It is uploaded in kube-manifests tarball, and used by
|
||||||
|
# upstart jobs in cluster/gce/trusty/<master, node>.yaml
|
||||||
|
|
||||||
|
|
||||||
|
config_hostname() {
|
||||||
|
# Set the hostname to the short version.
|
||||||
|
host_name=$(curl --silent "http://metadata.google.internal/computeMetadata/v1/instance/hostname" -H "Metadata-Flavor: Google")
|
||||||
|
short_hostname=$(echo "${host_name}" | cut -d. -f1)
|
||||||
|
hostname "${short_hostname}"
|
||||||
|
}
|
||||||
|
|
||||||
|
config_ip_firewall() {
|
||||||
|
# We have seen that GCE image may have strict host firewall rules which drop
|
||||||
|
# most inbound/forwarded packets. In such a case, add rules to accept all
|
||||||
|
# TCP/UDP packets.
|
||||||
|
if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then
|
||||||
|
echo "Add rules to accpet all inbound TCP/UDP packets"
|
||||||
|
iptables -A INPUT -w -p TCP -j ACCEPT
|
||||||
|
iptables -A INPUT -w -p UDP -j ACCEPT
|
||||||
|
fi
|
||||||
|
if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then
|
||||||
|
echo "Add rules to accpet all forwarded TCP/UDP packets"
|
||||||
|
iptables -A FORWARD -w -p TCP -j ACCEPT
|
||||||
|
iptables -A FORWARD -w -p UDP -j ACCEPT
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
create_dirs() {
|
||||||
|
# Create required directories.
|
||||||
|
mkdir -p /var/lib/kubelet
|
||||||
|
mkdir -p /etc/kubernetes/manifests
|
||||||
|
if [ "${KUBERNETES_MASTER:-}" = "false" ]; then
|
||||||
|
mkdir -p /var/lib/kube-proxy
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
create_kubelet_kubeconfig() {
|
||||||
|
# Create the kubelet kubeconfig file.
|
||||||
|
if [ -z "${KUBELET_CA_CERT:-}" ]; then
|
||||||
|
KUBELET_CA_CERT="${CA_CERT}"
|
||||||
|
fi
|
||||||
|
cat > /var/lib/kubelet/kubeconfig << EOF
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Config
|
||||||
|
users:
|
||||||
|
- name: kubelet
|
||||||
|
user:
|
||||||
|
client-certificate-data: "${KUBELET_CERT}"
|
||||||
|
client-key-data: "${KUBELET_KEY}"
|
||||||
|
clusters:
|
||||||
|
- name: local
|
||||||
|
cluster:
|
||||||
|
certificate-authority-data: "${KUBELET_CA_CERT}"
|
||||||
|
contexts:
|
||||||
|
- context:
|
||||||
|
cluster: local
|
||||||
|
user: kubelet
|
||||||
|
name: service-account-context
|
||||||
|
current-context: service-account-context
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
create_kubeproxy_kubeconfig() {
|
||||||
|
# Create the kube-proxy config file.
|
||||||
|
cat > /var/lib/kube-proxy/kubeconfig << EOF
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Config
|
||||||
|
users:
|
||||||
|
- name: kube-proxy
|
||||||
|
user:
|
||||||
|
token: "${KUBE_PROXY_TOKEN}"
|
||||||
|
clusters:
|
||||||
|
- name: local
|
||||||
|
cluster:
|
||||||
|
certificate-authority-data: "${CA_CERT}"
|
||||||
|
contexts:
|
||||||
|
- context:
|
||||||
|
cluster: local
|
||||||
|
user: kube-proxy
|
||||||
|
name: service-account-context
|
||||||
|
current-context: service-account-context
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
# Installs the critical packages that are required by spinning up a cluster.
|
||||||
|
install_critical_packages() {
|
||||||
|
apt-get update
|
||||||
|
# Install docker and brctl if they are not in the image.
|
||||||
|
if ! which docker > /dev/null; then
|
||||||
|
echo "Do not find docker. Install it."
|
||||||
|
curl -fsSL https://get.docker.com/ | sh
|
||||||
|
fi
|
||||||
|
if ! which brctl > /dev/null; then
|
||||||
|
echo "Do not find brctl. Install it."
|
||||||
|
apt-get install --yes bridge-utils
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Install the packages that are useful but not required by spinning up a cluster.
|
||||||
|
install_additional_packages() {
|
||||||
|
if ! which socat > /dev/null; then
|
||||||
|
echo "Do not find socat. Install it."
|
||||||
|
apt-get install --yes socat
|
||||||
|
fi
|
||||||
|
if ! which nsenter > /dev/null; then
|
||||||
|
echo "Do not find nsenter. Install it."
|
||||||
|
mkdir -p /tmp/nsenter-install
|
||||||
|
cd /tmp/nsenter-install
|
||||||
|
curl https://www.kernel.org/pub/linux/utils/util-linux/v2.24/util-linux-2.24.tar.gz | tar -zxf-
|
||||||
|
apt-get --yes install make
|
||||||
|
apt-get --yes install gcc
|
||||||
|
cd util-linux-2.24
|
||||||
|
./configure --without-ncurses
|
||||||
|
make nsenter
|
||||||
|
cp nsenter /usr/local/bin
|
||||||
|
rm -rf /tmp/nsenter-install
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Assembles kubelet command line flags.
|
||||||
|
# It should be called by master and nodes before running kubelet process. The caller
|
||||||
|
# needs to source the config file /etc/kube-env. This function sets the following
|
||||||
|
# variable that will be used in kubelet command line.
|
||||||
|
# KUBELET_CMD_FLAGS
|
||||||
|
assemble_kubelet_flags() {
|
||||||
|
KUBELET_CMD_FLAGS="--v=2"
|
||||||
|
if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_TEST_LOG_LEVEL}"
|
||||||
|
fi
|
||||||
|
if [ -n "${KUBELET_PORT:-}" ]; then
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --port=${KUBELET_PORT}"
|
||||||
|
fi
|
||||||
|
if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} ${KUBELET_TEST_ARGS}"
|
||||||
|
fi
|
||||||
|
if [ "${KUBERNETES_MASTER:-}" = "true" ] && \
|
||||||
|
[ ! -z "${KUBELET_APISERVER:-}" ] && \
|
||||||
|
[ ! -z "${KUBELET_CERT:-}" ] && \
|
||||||
|
[ ! -z "${KUBELET_KEY:-}" ]; then
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER}"
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --register-schedulable=false --reconcile-cidr=false"
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=10.123.45.0/30"
|
||||||
|
fi
|
||||||
|
if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}"
|
||||||
|
fi
|
||||||
|
if [ "${KUBERNETES_MASTER:-}" = "true" ]; then
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=none"
|
||||||
|
elif [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \
|
||||||
|
[ "${HAIRPIN_MODE:-}" = "hairpin-veth" ] || \
|
||||||
|
[ "${HAIRPIN_MODE:-}" = "none" ]; then
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=${HAIRPIN_MODE}"
|
||||||
|
fi
|
||||||
|
if [ -n "${ENABLE_CUSTOM_METRICS:-}" ]; then
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}"
|
||||||
|
fi
|
||||||
|
if [ -n "${NODE_LABELS:-}" ]; then
|
||||||
|
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --node-labels=${NODE_LABELS}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
restart_docker_daemon() {
|
||||||
|
readonly DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false"
|
||||||
|
echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker
|
||||||
|
# Make sure the network interface cbr0 is created before restarting docker daemon
|
||||||
|
while ! [ -L /sys/class/net/cbr0 ]; do
|
||||||
|
echo "Sleep 1 second to wait for cbr0"
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
# Remove docker0
|
||||||
|
ifconfig docker0 down
|
||||||
|
brctl delbr docker0
|
||||||
|
# Ensure docker daemon is really functional before exiting. Operations afterwards may
|
||||||
|
# assume it is running.
|
||||||
|
while ! docker version > /dev/null; do
|
||||||
|
echo "Sleep 1 second to wait for docker daemon"
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# Create the log file and set its properties.
|
||||||
|
#
|
||||||
|
# $1 is the file to create
|
||||||
|
prepare_log_file() {
|
||||||
|
touch $1
|
||||||
|
chmod 644 $1
|
||||||
|
chown root:root $1
|
||||||
|
}
|
||||||
|
|
||||||
|
# It monitors the health of several master and node components.
|
||||||
|
health_monitoring() {
|
||||||
|
sleep_seconds=10
|
||||||
|
max_seconds=10
|
||||||
|
# We simply kill the process when there is a failure. Another upstart job will automatically
|
||||||
|
# restart the process.
|
||||||
|
while [ 1 ]; do
|
||||||
|
if ! timeout 10 docker version > /dev/null; then
|
||||||
|
echo "Docker daemon failed!"
|
||||||
|
pkill docker
|
||||||
|
# Wait for a while, as we don't want to kill it again before it is really up.
|
||||||
|
sleep 30
|
||||||
|
fi
|
||||||
|
if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then
|
||||||
|
echo "Kubelet is unhealthy!"
|
||||||
|
pkill kubelet
|
||||||
|
# Wait for a while, as we don't want to kill it again before it is really up.
|
||||||
|
sleep 60
|
||||||
|
fi
|
||||||
|
sleep "${sleep_seconds}"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
########## The functions below are for master only ##########
|
||||||
|
|
||||||
|
# Mounts a persistent disk (formatting if needed) to store the persistent data
|
||||||
|
# on the master -- etcd's data, a few settings, and security certs/keys/tokens.
|
||||||
|
# safe_format_and_mount only formats an unformatted disk, and mkdir -p will
|
||||||
|
# leave a directory be if it already exists.
|
||||||
|
mount_master_pd() {
|
||||||
|
readonly pd_path="/dev/disk/by-id/google-master-pd"
|
||||||
|
readonly mount_point="/mnt/disks/master-pd"
|
||||||
|
|
||||||
|
# TODO(zmerlynn): GKE is still lagging in master-pd creation
|
||||||
|
if [ ! -e "${pd_path}" ]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
# Format and mount the disk, create directories on it for all of the master's
|
||||||
|
# persistent data, and link them to where they're used.
|
||||||
|
mkdir -p "${mount_point}"
|
||||||
|
/usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${pd_path}" "${mount_point}" >/var/log/master-pd-mount.log || \
|
||||||
|
{ echo "!!! master-pd mount failed, review /var/log/master-pd-mount.log !!!"; return 1; }
|
||||||
|
# Contains all the data stored in etcd
|
||||||
|
mkdir -m 700 -p "${mount_point}/var/etcd"
|
||||||
|
# Contains the dynamically generated apiserver auth certs and keys
|
||||||
|
mkdir -p "${mount_point}/etc/srv/kubernetes"
|
||||||
|
# Directory for kube-apiserver to store SSH key (if necessary)
|
||||||
|
mkdir -p "${mount_point}/etc/srv/sshproxy"
|
||||||
|
ln -s -f "${mount_point}/var/etcd" /var/etcd
|
||||||
|
mkdir -p /etc/srv
|
||||||
|
ln -s -f "${mount_point}/etc/srv/kubernetes" /etc/srv/kubernetes
|
||||||
|
ln -s -f "${mount_point}/etc/srv/sshproxy" /etc/srv/sshproxy
|
||||||
|
|
||||||
|
if ! id etcd &>/dev/null; then
|
||||||
|
useradd -s /sbin/nologin -d /var/etcd etcd
|
||||||
|
fi
|
||||||
|
chown -R etcd "${mount_point}/var/etcd"
|
||||||
|
chgrp -R etcd "${mount_point}/var/etcd"
|
||||||
|
}
|
||||||
|
|
||||||
|
# A helper function that adds an entry to a token file.
|
||||||
|
# $1: account information
|
||||||
|
# $2: token file
|
||||||
|
add_token_entry() {
|
||||||
|
current_token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||||
|
echo "${current_token},$1,$1" >> $2
|
||||||
|
}
|
||||||
|
|
||||||
|
# After the first boot and on upgrade, these files exists on the master-pd
|
||||||
|
# and should never be touched again (except perhaps an additional service
|
||||||
|
# account, see NB below.)
|
||||||
|
create_master_auth() {
|
||||||
|
readonly auth_dir="/etc/srv/kubernetes"
|
||||||
|
if [ ! -e "${auth_dir}/ca.crt" ]; then
|
||||||
|
if [ ! -z "${CA_CERT:-}" ] && [ ! -z "${MASTER_CERT:-}" ] && [ ! -z "${MASTER_KEY:-}" ]; then
|
||||||
|
echo "${CA_CERT}" | base64 -d > "${auth_dir}/ca.crt"
|
||||||
|
echo "${MASTER_CERT}" | base64 -d > "${auth_dir}/server.cert"
|
||||||
|
echo "${MASTER_KEY}" | base64 -d > "${auth_dir}/server.key"
|
||||||
|
# Kubecfg cert/key are optional and included for backwards compatibility.
|
||||||
|
# TODO(roberthbailey): Remove these two lines once GKE no longer requires
|
||||||
|
# fetching clients certs from the master VM.
|
||||||
|
echo "${KUBECFG_CERT:-}" | base64 -d > "${auth_dir}/kubecfg.crt"
|
||||||
|
echo "${KUBECFG_KEY:-}" | base64 -d > "${auth_dir}/kubecfg.key"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
readonly basic_auth_csv="${auth_dir}/basic_auth.csv"
|
||||||
|
if [ ! -e "${basic_auth_csv}" ]; then
|
||||||
|
echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${basic_auth_csv}"
|
||||||
|
fi
|
||||||
|
readonly known_tokens_csv="${auth_dir}/known_tokens.csv"
|
||||||
|
if [ ! -e "${known_tokens_csv}" ]; then
|
||||||
|
echo "${KUBE_BEARER_TOKEN},admin,admin" > "${known_tokens_csv}"
|
||||||
|
echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${known_tokens_csv}"
|
||||||
|
echo "${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy" >> "${known_tokens_csv}"
|
||||||
|
|
||||||
|
# Generate tokens for other "service accounts". Append to known_tokens.
|
||||||
|
#
|
||||||
|
# NB: If this list ever changes, this script actually has to
|
||||||
|
# change to detect the existence of this file, kill any deleted
|
||||||
|
# old tokens and add any new tokens (to handle the upgrade case).
|
||||||
|
add_token_entry "system:scheduler" "${known_tokens_csv}"
|
||||||
|
add_token_entry "system:controller_manager" "${known_tokens_csv}"
|
||||||
|
add_token_entry "system:logging" "${known_tokens_csv}"
|
||||||
|
add_token_entry "system:monitoring" "${known_tokens_csv}"
|
||||||
|
add_token_entry "system:dns" "${known_tokens_csv}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
|
||||||
|
cat <<EOF >/etc/gce.conf
|
||||||
|
[global]
|
||||||
|
token-url = "${TOKEN_URL}"
|
||||||
|
token-body = "${TOKEN_BODY}"
|
||||||
|
project-id = "${PROJECT_ID}"
|
||||||
|
network-name = "${NODE_NETWORK}"
|
||||||
|
EOF
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY
|
||||||
|
# to generate a kubeconfig file for the kubelet to securely connect to the apiserver.
|
||||||
|
create_master_kubelet_auth() {
|
||||||
|
# Only configure the kubelet on the master if the required variables are
|
||||||
|
# set in the environment.
|
||||||
|
if [ -n "${KUBELET_APISERVER:-}" ] && [ -n "${KUBELET_CERT:-}" ] && [ -n "${KUBELET_KEY:-}" ]; then
|
||||||
|
create_kubelet_kubeconfig
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Replaces the variables in the etcd manifest file with the real values, and then
|
||||||
|
# copy the file to the manifest dir
|
||||||
|
# $1: value for variable 'suffix'
|
||||||
|
# $2: value for variable 'port'
|
||||||
|
# $3: value for variable 'server_port'
|
||||||
|
# $4: value for variable 'cpulimit'
|
||||||
|
# $5: pod name, which should be either etcd or etcd-events
|
||||||
|
prepare_etcd_manifest() {
|
||||||
|
etcd_temp_file="/tmp/$5"
|
||||||
|
cp /run/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}"
|
||||||
|
sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}"
|
||||||
|
sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}"
|
||||||
|
sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}"
|
||||||
|
sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${etcd_temp_file}"
|
||||||
|
# Replace the volume host path
|
||||||
|
sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${etcd_temp_file}"
|
||||||
|
mv "${etcd_temp_file}" /etc/kubernetes/manifests
|
||||||
|
}
|
||||||
|
|
||||||
|
# Starts etcd server pod (and etcd-events pod if needed).
|
||||||
|
# More specifically, it prepares dirs and files, sets the variable value
|
||||||
|
# in the manifests, and copies them to /etc/kubernetes/manifests.
|
||||||
|
start_etcd_servers() {
|
||||||
|
if [ -d /etc/etcd ]; then
|
||||||
|
rm -rf /etc/etcd
|
||||||
|
fi
|
||||||
|
if [ -e /etc/default/etcd ]; then
|
||||||
|
rm -f /etc/default/etcd
|
||||||
|
fi
|
||||||
|
if [ -e /etc/systemd/system/etcd.service ]; then
|
||||||
|
rm -f /etc/systemd/system/etcd.service
|
||||||
|
fi
|
||||||
|
if [ -e /etc/init.d/etcd ]; then
|
||||||
|
rm -f /etc/init.d/etcd
|
||||||
|
fi
|
||||||
|
prepare_log_file /var/log/etcd.log
|
||||||
|
prepare_etcd_manifest "" "4001" "2380" "200m" "etcd.manifest"
|
||||||
|
|
||||||
|
prepare_log_file /var/log/etcd-events.log
|
||||||
|
prepare_etcd_manifest "-events" "4002" "2381" "100m" "etcd-events.manifest"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Calculates the following variables based on env variables, which will be used
|
||||||
|
# by the manifests of several kube-master components.
|
||||||
|
# CLOUD_CONFIG_VOLUME
|
||||||
|
# CLOUD_CONFIG_MOUNT
|
||||||
|
# DOCKER_REGISTRY
|
||||||
|
compute_master_manifest_variables() {
|
||||||
|
CLOUD_CONFIG_VOLUME=""
|
||||||
|
CLOUD_CONFIG_MOUNT=""
|
||||||
|
if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
|
||||||
|
CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\"}},"
|
||||||
|
CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
|
||||||
|
fi
|
||||||
|
DOCKER_REGISTRY="gcr.io/google_containers"
|
||||||
|
if [ -n "${KUBE_DOCKER_REGISTRY:-}" ]; then
|
||||||
|
DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# A helper function for removing salt configuration and comments from a file.
|
||||||
|
# This is mainly for preparing a manifest file.
|
||||||
|
# $1: Full path of the file to manipulate
|
||||||
|
remove_salt_config_comments() {
|
||||||
|
# Remove salt configuration
|
||||||
|
sed -i "/^[ |\t]*{[#|%]/d" $1
|
||||||
|
# Remove comments
|
||||||
|
sed -i "/^[ |\t]*#/d" $1
|
||||||
|
}
|
||||||
|
|
||||||
|
# Starts k8s apiserver.
|
||||||
|
# It prepares the log file, loads the docker image, calculates variables, sets them
|
||||||
|
# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
||||||
|
#
|
||||||
|
# Assumed vars (which are calculated in function compute_master_manifest_variables)
|
||||||
|
# CLOUD_CONFIG_VOLUME
|
||||||
|
# CLOUD_CONFIG_MOUNT
|
||||||
|
# DOCKER_REGISTRY
|
||||||
|
start_kube_apiserver() {
|
||||||
|
prepare_log_file /var/log/kube-apiserver.log
|
||||||
|
# Load the docker image from file.
|
||||||
|
echo "Try to load docker image file kube-apiserver.tar"
|
||||||
|
timeout 30 docker load -i /run/kube-docker-files/kube-apiserver.tar
|
||||||
|
|
||||||
|
# Calculate variables and assemble the command line.
|
||||||
|
params="--cloud-provider=gce --address=127.0.0.1 --etcd-servers=http://127.0.0.1:4001 --tls-cert-file=/etc/srv/kubernetes/server.cert --tls-private-key-file=/etc/srv/kubernetes/server.key --secure-port=443 --client-ca-file=/etc/srv/kubernetes/ca.crt --token-auth-file=/etc/srv/kubernetes/known_tokens.csv --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv --allow-privileged=true"
|
||||||
|
params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002"
|
||||||
|
if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then
|
||||||
|
params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
|
||||||
|
fi
|
||||||
|
if [ -n "${ADMISSION_CONTROL:-}" ]; then
|
||||||
|
params="${params} --admission-control=${ADMISSION_CONTROL}"
|
||||||
|
fi
|
||||||
|
if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
|
||||||
|
params="${params} --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}"
|
||||||
|
fi
|
||||||
|
if [ -n "${RUNTIME_CONFIG:-}" ]; then
|
||||||
|
params="${params} --runtime-config=${RUNTIME_CONFIG}"
|
||||||
|
fi
|
||||||
|
if [ -n "${APISERVER_TEST_ARGS:-}" ]; then
|
||||||
|
params="${params} ${APISERVER_TEST_ARGS}"
|
||||||
|
fi
|
||||||
|
log_level="--v=2"
|
||||||
|
if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then
|
||||||
|
log_level="${API_SERVER_TEST_LOG_LEVEL}"
|
||||||
|
fi
|
||||||
|
params="${params} ${log_level}"
|
||||||
|
|
||||||
|
if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
|
||||||
|
readonly vm_external_ip=$(curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip")
|
||||||
|
params="${params} --cloud-config=/etc/gce.conf --advertise-address=${vm_external_ip} --ssh-user=${PROXY_SSH_USER} --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile"
|
||||||
|
fi
|
||||||
|
readonly kube_apiserver_docker_tag=$(cat /run/kube-docker-files/kube-apiserver.docker_tag)
|
||||||
|
|
||||||
|
src_file="/run/kube-manifests/kubernetes/trusty/kube-apiserver.manifest"
|
||||||
|
remove_salt_config_comments "${src_file}"
|
||||||
|
# Evaluate variables
|
||||||
|
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{secure_port}}@443@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{secure_port}}@8080@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
|
||||||
|
cp "${src_file}" /etc/kubernetes/manifests
|
||||||
|
}
|
||||||
|
|
||||||
|
# Starts k8s controller manager.
|
||||||
|
# It prepares the log file, loads the docker image, calculates variables, sets them
|
||||||
|
# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
||||||
|
#
|
||||||
|
# Assumed vars (which are calculated in function compute_master_manifest_variables)
|
||||||
|
# CLOUD_CONFIG_VOLUME
|
||||||
|
# CLOUD_CONFIG_MOUNT
|
||||||
|
# DOCKER_REGISTRY
|
||||||
|
start_kube_controller_manager() {
|
||||||
|
prepare_log_file /var/log/kube-controller-manager.log
|
||||||
|
# Load the docker image from file.
|
||||||
|
echo "Try to load docker image file kube-controller-manager.tar"
|
||||||
|
timeout 30 docker load -i /run/kube-docker-files/kube-controller-manager.tar
|
||||||
|
|
||||||
|
# Calculate variables and assemble the command line.
|
||||||
|
params="--master=127.0.0.1:8080 --cloud-provider=gce --root-ca-file=/etc/srv/kubernetes/ca.crt --service-account-private-key-file=/etc/srv/kubernetes/server.key"
|
||||||
|
if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
|
||||||
|
params="${params} --cloud-config=/etc/gce.conf"
|
||||||
|
fi
|
||||||
|
if [ -n "${INSTANCE_PREFIX:-}" ]; then
|
||||||
|
params="${params} --cluster-name=${INSTANCE_PREFIX}"
|
||||||
|
fi
|
||||||
|
if [ -n "${CLUSTER_IP_RANGE:-}" ]; then
|
||||||
|
params="${params} --cluster-cidr=${CLUSTER_IP_RANGE}"
|
||||||
|
fi
|
||||||
|
if [ "${ALLOCATE_NODE_CIDRS:-}" = "true" ]; then
|
||||||
|
params="${params} --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}"
|
||||||
|
fi
|
||||||
|
if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then
|
||||||
|
params="${params} --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
|
||||||
|
fi
|
||||||
|
log_level="--v=2"
|
||||||
|
if [ -n "${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-}" ]; then
|
||||||
|
log_level="${CONTROLLER_MANAGER_TEST_LOG_LEVEL}"
|
||||||
|
fi
|
||||||
|
params="${params} ${log_level}"
|
||||||
|
if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then
|
||||||
|
params="${params} ${CONTROLLER_MANAGER_TEST_ARGS}"
|
||||||
|
fi
|
||||||
|
readonly kube_rc_docker_tag=$(cat /run/kube-docker-files/kube-controller-manager.docker_tag)
|
||||||
|
|
||||||
|
src_file="/run/kube-manifests/kubernetes/trusty/kube-controller-manager.manifest"
|
||||||
|
remove_salt_config_comments "${src_file}"
|
||||||
|
# Evaluate variables
|
||||||
|
sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{pillar\['kube-controller-manager_docker_tag'\]}}@${kube_rc_docker_tag}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
|
||||||
|
cp "${src_file}" /etc/kubernetes/manifests
|
||||||
|
}
|
||||||
|
|
||||||
|
# Starts k8s scheduler.
|
||||||
|
# It prepares the log file, loads the docker image, calculates variables, sets them
|
||||||
|
# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
||||||
|
#
|
||||||
|
# Assumed vars (which are calculated in compute_master_manifest_variables())
|
||||||
|
# DOCKER_REGISTRY
|
||||||
|
start_kube_scheduler() {
|
||||||
|
prepare_log_file /var/log/kube-scheduler.log
|
||||||
|
# Load the docker image from file.
|
||||||
|
echo "Try to load docker image file kube-scheduler.tar"
|
||||||
|
timeout 30 docker load -i /run/kube-docker-files/kube-scheduler.tar
|
||||||
|
|
||||||
|
# Calculate variables and set them in the manifest.
|
||||||
|
params=""
|
||||||
|
log_level="--v=2"
|
||||||
|
if [ -n "${SCHEDULER_TEST_LOG_LEVEL:-}" ]; then
|
||||||
|
log_level="${SCHEDULER_TEST_LOG_LEVEL}"
|
||||||
|
fi
|
||||||
|
params="${params} ${log_level}"
|
||||||
|
if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then
|
||||||
|
params="${params} ${SCHEDULER_TEST_ARGS}"
|
||||||
|
fi
|
||||||
|
readonly kube_scheduler_docker_tag=$(cat /run/kube-docker-files/kube-scheduler.docker_tag)
|
||||||
|
|
||||||
|
# Remove salt comments and replace variables with values
|
||||||
|
src_file="/run/kube-manifests/kubernetes/trusty/kube-scheduler.manifest"
|
||||||
|
remove_salt_config_comments "${src_file}"
|
||||||
|
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
||||||
|
sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" "${src_file}"
|
||||||
|
cp "${src_file}" /etc/kubernetes/manifests
|
||||||
|
}
|
||||||
|
|
||||||
|
# Starts a fluentd static pod for logging.
|
||||||
|
start_fluentd() {
|
||||||
|
if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then
|
||||||
|
if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then
|
||||||
|
cp /run/kube-manifests/kubernetes/fluentd-gcp.yaml /etc/kubernetes/manifests/
|
||||||
|
elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then
|
||||||
|
cp /run/kube-manifests/kubernetes/fluentd-es.yaml /etc/kubernetes/manifests/
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# A helper function for copying addon manifests and set dir/files
|
||||||
|
# permissions.
|
||||||
|
# $1: addon category under /etc/kubernetes
|
||||||
|
# $2: manifest source dir
|
||||||
|
setup_addon_manifests() {
|
||||||
|
src_dir="/run/kube-manifests/kubernetes/trusty/$2"
|
||||||
|
dst_dir="/etc/kubernetes/$1/$2"
|
||||||
|
if [ ! -d "${dst_dir}" ]; then
|
||||||
|
mkdir -p "${dst_dir}"
|
||||||
|
fi
|
||||||
|
files=$(find "${src_dir}" -name "*.yaml")
|
||||||
|
if [ -n "${files}" ]; then
|
||||||
|
cp "${src_dir}/"*.yaml "${dst_dir}"
|
||||||
|
fi
|
||||||
|
files=$(find "${src_dir}" -name "*.json")
|
||||||
|
if [ -n "${files}" ]; then
|
||||||
|
cp "${src_dir}/"*.json "${dst_dir}"
|
||||||
|
fi
|
||||||
|
files=$(find "${src_dir}" -name "*.yaml.in")
|
||||||
|
if [ -n "${files}" ]; then
|
||||||
|
cp "${src_dir}/"*.yaml.in "${dst_dir}"
|
||||||
|
fi
|
||||||
|
chown -R root:root "${dst_dir}"
|
||||||
|
chmod 755 "${dst_dir}"
|
||||||
|
chmod 644 "${dst_dir}"/*
|
||||||
|
}
|
||||||
|
|
||||||
|
# Prepares the manifests of k8s addons static pods.
|
||||||
|
prepare_kube_addons() {
|
||||||
|
addon_src_dir="/run/kube-manifests/kubernetes/trusty"
|
||||||
|
addon_dst_dir="/etc/kubernetes/addons"
|
||||||
|
# Set up manifests of other addons.
|
||||||
|
if [ "${ENABLE_CLUSTER_MONITORING:-}" = "influxdb" ] || \
|
||||||
|
[ "${ENABLE_CLUSTER_MONITORING:-}" = "google" ] || \
|
||||||
|
[ "${ENABLE_CLUSTER_MONITORING:-}" = "standalone" ] || \
|
||||||
|
[ "${ENABLE_CLUSTER_MONITORING:-}" = "googleinfluxdb" ]; then
|
||||||
|
file_dir="cluster-monitoring/${ENABLE_CLUSTER_MONITORING}"
|
||||||
|
setup_addon_manifests "addons" "${file_dir}"
|
||||||
|
# Replace the salt configurations with variable values.
|
||||||
|
metrics_memory="200Mi"
|
||||||
|
eventer_memory="200Mi"
|
||||||
|
if [ -n "${NUM_NODES:-}" ] && [ "${NUM_NODES}" -ge 1 ]; then
|
||||||
|
num_kube_nodes="$((${NUM_NODES}-1))"
|
||||||
|
metrics_memory="$((${num_kube_nodes} * 4 + 200))Mi"
|
||||||
|
eventer_memory="$((${num_kube_nodes} * 500 + 200 * 1024))Ki"
|
||||||
|
fi
|
||||||
|
controller_yaml="${addon_dst_dir}/${file_dir}"
|
||||||
|
if [ "${ENABLE_CLUSTER_MONITORING:-}" = "googleinfluxdb" ]; then
|
||||||
|
controller_yaml="${controller_yaml}/heapster-controller-combined.yaml"
|
||||||
|
else
|
||||||
|
controller_yaml="${controller_yaml}/heapster-controller.yaml"
|
||||||
|
fi
|
||||||
|
remove_salt_config_comments "${controller_yaml}"
|
||||||
|
sed -i -e "s@{{ *metrics_memory *}}@${metrics_memory}@g" "${controller_yaml}"
|
||||||
|
sed -i -e "s@{{ *eventer_memory *}}@${eventer_memory}@g" "${controller_yaml}"
|
||||||
|
fi
|
||||||
|
cp "${addon_src_dir}/namespace.yaml" "${addon_dst_dir}"
|
||||||
|
if [ "${ENABLE_L7_LOADBALANCING:-}" = "glbc" ]; then
|
||||||
|
setup_addon_manifests "addons" "cluster-loadbalancing/glbc"
|
||||||
|
fi
|
||||||
|
if [ "${ENABLE_CLUSTER_DNS:-}" = "true" ]; then
|
||||||
|
setup_addon_manifests "addons" "dns"
|
||||||
|
dns_rc_file="${addon_dst_dir}/dns/skydns-rc.yaml"
|
||||||
|
dns_svc_file="${addon_dst_dir}/dns/skydns-svc.yaml"
|
||||||
|
mv "${addon_dst_dir}/dns/skydns-rc.yaml.in" "${dns_rc_file}"
|
||||||
|
mv "${addon_dst_dir}/dns/skydns-svc.yaml.in" "${dns_svc_file}"
|
||||||
|
# Replace the salt configurations with variable values.
|
||||||
|
sed -i -e "s@{{ *pillar\['dns_replicas'\] *}}@${DNS_REPLICAS}@g" "${dns_rc_file}"
|
||||||
|
sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${dns_rc_file}"
|
||||||
|
sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${dns_svc_file}"
|
||||||
|
fi
|
||||||
|
if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then
|
||||||
|
setup_addon_manifests "addons" "registry"
|
||||||
|
registry_pv_file="${addon_dst_dir}/registry/registry-pv.yaml"
|
||||||
|
registry_pvc_file="${addon_dst_dir}/registry/registry-pvc.yaml"
|
||||||
|
mv "${addon_dst_dir}/registry/registry-pv.yaml.in" "${registry_pv_file}"
|
||||||
|
mv "${addon_dst_dir}/registry/registry-pvc.yaml.in" "${registry_pvc_file}"
|
||||||
|
# Replace the salt configurations with variable values.
|
||||||
|
remove_salt_config_comments "${controller_yaml}"
|
||||||
|
sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pv_file}"
|
||||||
|
sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pvc_file}"
|
||||||
|
sed -i -e "s@{{ *pillar\['cluster_registry_disk_name'\] *}}@${CLUSTER_REGISTRY_DISK}@g" "${registry_pvc_file}"
|
||||||
|
fi
|
||||||
|
if [ "${ENABLE_NODE_LOGGING:-}" = "true" ] && \
|
||||||
|
[ "${LOGGING_DESTINATION:-}" = "elasticsearch" ] && \
|
||||||
|
[ "${ENABLE_CLUSTER_LOGGING:-}" = "true" ]; then
|
||||||
|
setup_addon_manifests "addons" "fluentd-elasticsearch"
|
||||||
|
fi
|
||||||
|
if [ "${ENABLE_CLUSTER_UI:-}" = "true" ]; then
|
||||||
|
setup_addon_manifests "addons" "dashboard"
|
||||||
|
fi
|
||||||
|
if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then
|
||||||
|
setup_addon_manifests "admission-controls" "limit-range"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Prepare the scripts for running addons.
|
||||||
|
addon_script_dir="/var/lib/cloud/scripts/kubernetes"
|
||||||
|
mkdir -p "${addon_script_dir}"
|
||||||
|
cp "${addon_src_dir}/kube-addons.sh" "${addon_script_dir}"
|
||||||
|
cp "${addon_src_dir}/kube-addon-update.sh" "${addon_script_dir}"
|
||||||
|
chmod 544 "${addon_script_dir}/"*.sh
|
||||||
|
# In case that some GCE customized trusty may have a read-only /root.
|
||||||
|
mount -t tmpfs tmpfs /root
|
||||||
|
mount --bind -o remount,rw,noexec /root
|
||||||
|
}
|
||||||
@ -20,38 +20,6 @@
|
|||||||
# of needed functions. The script itself is not supposed to be executed in
|
# of needed functions. The script itself is not supposed to be executed in
|
||||||
# other manners.
|
# other manners.
|
||||||
|
|
||||||
config_hostname() {
|
|
||||||
# Set the hostname to the short version.
|
|
||||||
host_name=$(curl --silent "http://metadata.google.internal/computeMetadata/v1/instance/hostname" -H "Metadata-Flavor: Google")
|
|
||||||
short_hostname=$(echo "${host_name}" | cut -d. -f1)
|
|
||||||
hostname "${short_hostname}"
|
|
||||||
}
|
|
||||||
|
|
||||||
config_ip_firewall() {
|
|
||||||
# We have seen that GCE image may have strict host firewall rules which drop
|
|
||||||
# most inbound/forwarded packets. In such a case, add rules to accept all
|
|
||||||
# TCP/UDP packets.
|
|
||||||
if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then
|
|
||||||
echo "Add rules to accpet all inbound TCP/UDP packets"
|
|
||||||
iptables -A INPUT -w -p TCP -j ACCEPT
|
|
||||||
iptables -A INPUT -w -p UDP -j ACCEPT
|
|
||||||
fi
|
|
||||||
if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then
|
|
||||||
echo "Add rules to accpet all forwarded TCP/UDP packets"
|
|
||||||
iptables -A FORWARD -w -p TCP -j ACCEPT
|
|
||||||
iptables -A FORWARD -w -p UDP -j ACCEPT
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
create_dirs() {
|
|
||||||
# Create required directories.
|
|
||||||
mkdir -p /var/lib/kubelet
|
|
||||||
mkdir -p /etc/kubernetes/manifests
|
|
||||||
if [ "${KUBERNETES_MASTER:-}" = "false" ]; then
|
|
||||||
mkdir -p /var/lib/kube-proxy
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
download_kube_env() {
|
download_kube_env() {
|
||||||
# Fetch kube-env from GCE metadata server.
|
# Fetch kube-env from GCE metadata server.
|
||||||
readonly tmp_install_dir="/var/cache/kubernetes-install"
|
readonly tmp_install_dir="/var/cache/kubernetes-install"
|
||||||
@ -68,89 +36,6 @@ for k,v in yaml.load(sys.stdin).iteritems():
|
|||||||
''' < "${tmp_install_dir}/kube_env.yaml" > /etc/kube-env)
|
''' < "${tmp_install_dir}/kube_env.yaml" > /etc/kube-env)
|
||||||
}
|
}
|
||||||
|
|
||||||
create_kubelet_kubeconfig() {
|
|
||||||
# Create the kubelet kubeconfig file.
|
|
||||||
if [ -z "${KUBELET_CA_CERT:-}" ]; then
|
|
||||||
KUBELET_CA_CERT="${CA_CERT}"
|
|
||||||
fi
|
|
||||||
cat > /var/lib/kubelet/kubeconfig << EOF
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Config
|
|
||||||
users:
|
|
||||||
- name: kubelet
|
|
||||||
user:
|
|
||||||
client-certificate-data: "${KUBELET_CERT}"
|
|
||||||
client-key-data: "${KUBELET_KEY}"
|
|
||||||
clusters:
|
|
||||||
- name: local
|
|
||||||
cluster:
|
|
||||||
certificate-authority-data: "${KUBELET_CA_CERT}"
|
|
||||||
contexts:
|
|
||||||
- context:
|
|
||||||
cluster: local
|
|
||||||
user: kubelet
|
|
||||||
name: service-account-context
|
|
||||||
current-context: service-account-context
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
create_kubeproxy_kubeconfig() {
|
|
||||||
# Create the kube-proxy config file.
|
|
||||||
cat > /var/lib/kube-proxy/kubeconfig << EOF
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Config
|
|
||||||
users:
|
|
||||||
- name: kube-proxy
|
|
||||||
user:
|
|
||||||
token: "${KUBE_PROXY_TOKEN}"
|
|
||||||
clusters:
|
|
||||||
- name: local
|
|
||||||
cluster:
|
|
||||||
certificate-authority-data: "${CA_CERT}"
|
|
||||||
contexts:
|
|
||||||
- context:
|
|
||||||
cluster: local
|
|
||||||
user: kube-proxy
|
|
||||||
name: service-account-context
|
|
||||||
current-context: service-account-context
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
# Installs the critical packages that are required by spinning up a cluster.
|
|
||||||
install_critical_packages() {
|
|
||||||
apt-get update
|
|
||||||
# Install docker and brctl if they are not in the image.
|
|
||||||
if ! which docker > /dev/null; then
|
|
||||||
echo "Do not find docker. Install it."
|
|
||||||
curl -fsSL https://get.docker.com/ | sh
|
|
||||||
fi
|
|
||||||
if ! which brctl > /dev/null; then
|
|
||||||
echo "Do not find brctl. Install it."
|
|
||||||
apt-get install --yes bridge-utils
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Install the packages that are useful but not required by spinning up a cluster.
|
|
||||||
install_additional_packages() {
|
|
||||||
if ! which socat > /dev/null; then
|
|
||||||
echo "Do not find socat. Install it."
|
|
||||||
apt-get install --yes socat
|
|
||||||
fi
|
|
||||||
if ! which nsenter > /dev/null; then
|
|
||||||
echo "Do not find nsenter. Install it."
|
|
||||||
mkdir -p /tmp/nsenter-install
|
|
||||||
cd /tmp/nsenter-install
|
|
||||||
curl https://www.kernel.org/pub/linux/utils/util-linux/v2.24/util-linux-2.24.tar.gz | tar -zxf-
|
|
||||||
apt-get --yes install make
|
|
||||||
apt-get --yes install gcc
|
|
||||||
cd util-linux-2.24
|
|
||||||
./configure --without-ncurses
|
|
||||||
make nsenter
|
|
||||||
cp nsenter /usr/local/bin
|
|
||||||
rm -rf /tmp/nsenter-install
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Retry a download until we get it.
|
# Retry a download until we get it.
|
||||||
#
|
#
|
||||||
# $1 is the file to create
|
# $1 is the file to create
|
||||||
@ -237,542 +122,7 @@ install_kube_binary_config() {
|
|||||||
find /run/kube-manifests -name \*.manifest -or -name \*.json | \
|
find /run/kube-manifests -name \*.manifest -or -name \*.json | \
|
||||||
xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
|
xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
|
||||||
fi
|
fi
|
||||||
|
cp /run/kube-manifests/kubernetes/trusty/configure-helper.sh /etc/kube-configure-helper.sh
|
||||||
rm "/run/kube-manifests/${manifests_sha1}"
|
rm "/run/kube-manifests/${manifests_sha1}"
|
||||||
rm "/run/kube-manifests/${manifests_tar}"
|
rm "/run/kube-manifests/${manifests_tar}"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Assembles kubelet command line flags.
|
|
||||||
# It should be called by master and nodes before running kubelet process. The caller
|
|
||||||
# needs to source the config file /etc/kube-env. This function sets the following
|
|
||||||
# variable that will be used in kubelet command line.
|
|
||||||
# KUBELET_CMD_FLAGS
|
|
||||||
assemble_kubelet_flags() {
|
|
||||||
KUBELET_CMD_FLAGS="--v=2"
|
|
||||||
if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_TEST_LOG_LEVEL}"
|
|
||||||
fi
|
|
||||||
if [ -n "${KUBELET_PORT:-}" ]; then
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --port=${KUBELET_PORT}"
|
|
||||||
fi
|
|
||||||
if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} ${KUBELET_TEST_ARGS}"
|
|
||||||
fi
|
|
||||||
if [ "${KUBERNETES_MASTER:-}" = "true" ] && \
|
|
||||||
[ ! -z "${KUBELET_APISERVER:-}" ] && \
|
|
||||||
[ ! -z "${KUBELET_CERT:-}" ] && \
|
|
||||||
[ ! -z "${KUBELET_KEY:-}" ]; then
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER}"
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --register-schedulable=false --reconcile-cidr=false"
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=10.123.45.0/30"
|
|
||||||
fi
|
|
||||||
if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}"
|
|
||||||
fi
|
|
||||||
if [ "${KUBERNETES_MASTER:-}" = "true" ]; then
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=none"
|
|
||||||
elif [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \
|
|
||||||
[ "${HAIRPIN_MODE:-}" = "hairpin-veth" ] || \
|
|
||||||
[ "${HAIRPIN_MODE:-}" = "none" ]; then
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=${HAIRPIN_MODE}"
|
|
||||||
fi
|
|
||||||
if [ -n "${ENABLE_CUSTOM_METRICS:-}" ]; then
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}"
|
|
||||||
fi
|
|
||||||
if [ -n "${NODE_LABELS:-}" ]; then
|
|
||||||
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --node-labels=${NODE_LABELS}"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
restart_docker_daemon() {
|
|
||||||
readonly DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false"
|
|
||||||
echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker
|
|
||||||
# Make sure the network interface cbr0 is created before restarting docker daemon
|
|
||||||
while ! [ -L /sys/class/net/cbr0 ]; do
|
|
||||||
echo "Sleep 1 second to wait for cbr0"
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
# Remove docker0
|
|
||||||
ifconfig docker0 down
|
|
||||||
brctl delbr docker0
|
|
||||||
# Ensure docker daemon is really functional before exiting. Operations afterwards may
|
|
||||||
# assume it is running.
|
|
||||||
while ! docker version > /dev/null; do
|
|
||||||
echo "Sleep 1 second to wait for docker daemon"
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# Create the log file and set its properties.
|
|
||||||
#
|
|
||||||
# $1 is the file to create
|
|
||||||
prepare_log_file() {
|
|
||||||
touch $1
|
|
||||||
chmod 644 $1
|
|
||||||
chown root:root $1
|
|
||||||
}
|
|
||||||
|
|
||||||
# It monitors the health of several master and node components.
|
|
||||||
health_monitoring() {
|
|
||||||
sleep_seconds=10
|
|
||||||
max_seconds=10
|
|
||||||
# We simply kill the process when there is a failure. Another upstart job will automatically
|
|
||||||
# restart the process.
|
|
||||||
while [ 1 ]; do
|
|
||||||
if ! timeout 10 docker version > /dev/null; then
|
|
||||||
echo "Docker daemon failed!"
|
|
||||||
pkill docker
|
|
||||||
# Wait for a while, as we don't want to kill it again before it is really up.
|
|
||||||
sleep 30
|
|
||||||
fi
|
|
||||||
if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then
|
|
||||||
echo "Kubelet is unhealthy!"
|
|
||||||
pkill kubelet
|
|
||||||
# Wait for a while, as we don't want to kill it again before it is really up.
|
|
||||||
sleep 60
|
|
||||||
fi
|
|
||||||
sleep "${sleep_seconds}"
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
########## The functions below are for master only ##########
|
|
||||||
|
|
||||||
# Mounts a persistent disk (formatting if needed) to store the persistent data
|
|
||||||
# on the master -- etcd's data, a few settings, and security certs/keys/tokens.
|
|
||||||
# safe_format_and_mount only formats an unformatted disk, and mkdir -p will
|
|
||||||
# leave a directory be if it already exists.
|
|
||||||
mount_master_pd() {
|
|
||||||
readonly pd_path="/dev/disk/by-id/google-master-pd"
|
|
||||||
readonly mount_point="/mnt/disks/master-pd"
|
|
||||||
|
|
||||||
# TODO(zmerlynn): GKE is still lagging in master-pd creation
|
|
||||||
if [ ! -e "${pd_path}" ]; then
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
# Format and mount the disk, create directories on it for all of the master's
|
|
||||||
# persistent data, and link them to where they're used.
|
|
||||||
mkdir -p "${mount_point}"
|
|
||||||
/usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${pd_path}" "${mount_point}" >/var/log/master-pd-mount.log || \
|
|
||||||
{ echo "!!! master-pd mount failed, review /var/log/master-pd-mount.log !!!"; return 1; }
|
|
||||||
# Contains all the data stored in etcd
|
|
||||||
mkdir -m 700 -p "${mount_point}/var/etcd"
|
|
||||||
# Contains the dynamically generated apiserver auth certs and keys
|
|
||||||
mkdir -p "${mount_point}/etc/srv/kubernetes"
|
|
||||||
# Directory for kube-apiserver to store SSH key (if necessary)
|
|
||||||
mkdir -p "${mount_point}/etc/srv/sshproxy"
|
|
||||||
ln -s -f "${mount_point}/var/etcd" /var/etcd
|
|
||||||
mkdir -p /etc/srv
|
|
||||||
ln -s -f "${mount_point}/etc/srv/kubernetes" /etc/srv/kubernetes
|
|
||||||
ln -s -f "${mount_point}/etc/srv/sshproxy" /etc/srv/sshproxy
|
|
||||||
|
|
||||||
if ! id etcd &>/dev/null; then
|
|
||||||
useradd -s /sbin/nologin -d /var/etcd etcd
|
|
||||||
fi
|
|
||||||
chown -R etcd "${mount_point}/var/etcd"
|
|
||||||
chgrp -R etcd "${mount_point}/var/etcd"
|
|
||||||
}
|
|
||||||
|
|
||||||
# A helper function that adds an entry to a token file.
|
|
||||||
# $1: account information
|
|
||||||
# $2: token file
|
|
||||||
add_token_entry() {
|
|
||||||
current_token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
|
||||||
echo "${current_token},$1,$1" >> $2
|
|
||||||
}
|
|
||||||
|
|
||||||
# After the first boot and on upgrade, these files exists on the master-pd
|
|
||||||
# and should never be touched again (except perhaps an additional service
|
|
||||||
# account, see NB below.)
|
|
||||||
create_master_auth() {
|
|
||||||
readonly auth_dir="/etc/srv/kubernetes"
|
|
||||||
if [ ! -e "${auth_dir}/ca.crt" ]; then
|
|
||||||
if [ ! -z "${CA_CERT:-}" ] && [ ! -z "${MASTER_CERT:-}" ] && [ ! -z "${MASTER_KEY:-}" ]; then
|
|
||||||
echo "${CA_CERT}" | base64 -d > "${auth_dir}/ca.crt"
|
|
||||||
echo "${MASTER_CERT}" | base64 -d > "${auth_dir}/server.cert"
|
|
||||||
echo "${MASTER_KEY}" | base64 -d > "${auth_dir}/server.key"
|
|
||||||
# Kubecfg cert/key are optional and included for backwards compatibility.
|
|
||||||
# TODO(roberthbailey): Remove these two lines once GKE no longer requires
|
|
||||||
# fetching clients certs from the master VM.
|
|
||||||
echo "${KUBECFG_CERT:-}" | base64 -d > "${auth_dir}/kubecfg.crt"
|
|
||||||
echo "${KUBECFG_KEY:-}" | base64 -d > "${auth_dir}/kubecfg.key"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
readonly basic_auth_csv="${auth_dir}/basic_auth.csv"
|
|
||||||
if [ ! -e "${basic_auth_csv}" ]; then
|
|
||||||
echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${basic_auth_csv}"
|
|
||||||
fi
|
|
||||||
readonly known_tokens_csv="${auth_dir}/known_tokens.csv"
|
|
||||||
if [ ! -e "${known_tokens_csv}" ]; then
|
|
||||||
echo "${KUBE_BEARER_TOKEN},admin,admin" > "${known_tokens_csv}"
|
|
||||||
echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${known_tokens_csv}"
|
|
||||||
echo "${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy" >> "${known_tokens_csv}"
|
|
||||||
|
|
||||||
# Generate tokens for other "service accounts". Append to known_tokens.
|
|
||||||
#
|
|
||||||
# NB: If this list ever changes, this script actually has to
|
|
||||||
# change to detect the existence of this file, kill any deleted
|
|
||||||
# old tokens and add any new tokens (to handle the upgrade case).
|
|
||||||
add_token_entry "system:scheduler" "${known_tokens_csv}"
|
|
||||||
add_token_entry "system:controller_manager" "${known_tokens_csv}"
|
|
||||||
add_token_entry "system:logging" "${known_tokens_csv}"
|
|
||||||
add_token_entry "system:monitoring" "${known_tokens_csv}"
|
|
||||||
add_token_entry "system:dns" "${known_tokens_csv}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
|
|
||||||
cat <<EOF >/etc/gce.conf
|
|
||||||
[global]
|
|
||||||
token-url = "${TOKEN_URL}"
|
|
||||||
token-body = "${TOKEN_BODY}"
|
|
||||||
project-id = "${PROJECT_ID}"
|
|
||||||
network-name = "${NODE_NETWORK}"
|
|
||||||
EOF
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY
|
|
||||||
# to generate a kubeconfig file for the kubelet to securely connect to the apiserver.
|
|
||||||
create_master_kubelet_auth() {
|
|
||||||
# Only configure the kubelet on the master if the required variables are
|
|
||||||
# set in the environment.
|
|
||||||
if [ -n "${KUBELET_APISERVER:-}" ] && [ -n "${KUBELET_CERT:-}" ] && [ -n "${KUBELET_KEY:-}" ]; then
|
|
||||||
create_kubelet_kubeconfig
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Replaces the variables in the etcd manifest file with the real values, and then
|
|
||||||
# copy the file to the manifest dir
|
|
||||||
# $1: value for variable 'suffix'
|
|
||||||
# $2: value for variable 'port'
|
|
||||||
# $3: value for variable 'server_port'
|
|
||||||
# $4: value for variable 'cpulimit'
|
|
||||||
# $5: pod name, which should be either etcd or etcd-events
|
|
||||||
prepare_etcd_manifest() {
|
|
||||||
etcd_temp_file="/tmp/$5"
|
|
||||||
cp /run/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}"
|
|
||||||
sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}"
|
|
||||||
sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}"
|
|
||||||
sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}"
|
|
||||||
sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${etcd_temp_file}"
|
|
||||||
# Replace the volume host path
|
|
||||||
sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${etcd_temp_file}"
|
|
||||||
mv "${etcd_temp_file}" /etc/kubernetes/manifests
|
|
||||||
}
|
|
||||||
|
|
||||||
# Starts etcd server pod (and etcd-events pod if needed).
|
|
||||||
# More specifically, it prepares dirs and files, sets the variable value
|
|
||||||
# in the manifests, and copies them to /etc/kubernetes/manifests.
|
|
||||||
start_etcd_servers() {
|
|
||||||
if [ -d /etc/etcd ]; then
|
|
||||||
rm -rf /etc/etcd
|
|
||||||
fi
|
|
||||||
if [ -e /etc/default/etcd ]; then
|
|
||||||
rm -f /etc/default/etcd
|
|
||||||
fi
|
|
||||||
if [ -e /etc/systemd/system/etcd.service ]; then
|
|
||||||
rm -f /etc/systemd/system/etcd.service
|
|
||||||
fi
|
|
||||||
if [ -e /etc/init.d/etcd ]; then
|
|
||||||
rm -f /etc/init.d/etcd
|
|
||||||
fi
|
|
||||||
prepare_log_file /var/log/etcd.log
|
|
||||||
prepare_etcd_manifest "" "4001" "2380" "200m" "etcd.manifest"
|
|
||||||
|
|
||||||
prepare_log_file /var/log/etcd-events.log
|
|
||||||
prepare_etcd_manifest "-events" "4002" "2381" "100m" "etcd-events.manifest"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Calculates the following variables based on env variables, which will be used
|
|
||||||
# by the manifests of several kube-master components.
|
|
||||||
# CLOUD_CONFIG_VOLUME
|
|
||||||
# CLOUD_CONFIG_MOUNT
|
|
||||||
# DOCKER_REGISTRY
|
|
||||||
compute_master_manifest_variables() {
|
|
||||||
CLOUD_CONFIG_VOLUME=""
|
|
||||||
CLOUD_CONFIG_MOUNT=""
|
|
||||||
if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
|
|
||||||
CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\"}},"
|
|
||||||
CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
|
|
||||||
fi
|
|
||||||
DOCKER_REGISTRY="gcr.io/google_containers"
|
|
||||||
if [ -n "${KUBE_DOCKER_REGISTRY:-}" ]; then
|
|
||||||
DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# A helper function for removing salt configuration and comments from a file.
|
|
||||||
# This is mainly for preparing a manifest file.
|
|
||||||
# $1: Full path of the file to manipulate
|
|
||||||
remove_salt_config_comments() {
|
|
||||||
# Remove salt configuration
|
|
||||||
sed -i "/^[ |\t]*{[#|%]/d" $1
|
|
||||||
# Remove comments
|
|
||||||
sed -i "/^[ |\t]*#/d" $1
|
|
||||||
}
|
|
||||||
|
|
||||||
# Starts k8s apiserver.
|
|
||||||
# It prepares the log file, loads the docker image, calculates variables, sets them
|
|
||||||
# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
|
||||||
#
|
|
||||||
# Assumed vars (which are calculated in function compute_master_manifest_variables)
|
|
||||||
# CLOUD_CONFIG_VOLUME
|
|
||||||
# CLOUD_CONFIG_MOUNT
|
|
||||||
# DOCKER_REGISTRY
|
|
||||||
start_kube_apiserver() {
|
|
||||||
prepare_log_file /var/log/kube-apiserver.log
|
|
||||||
# Load the docker image from file.
|
|
||||||
echo "Try to load docker image file kube-apiserver.tar"
|
|
||||||
timeout 30 docker load -i /run/kube-docker-files/kube-apiserver.tar
|
|
||||||
|
|
||||||
# Calculate variables and assemble the command line.
|
|
||||||
params="--cloud-provider=gce --address=127.0.0.1 --etcd-servers=http://127.0.0.1:4001 --tls-cert-file=/etc/srv/kubernetes/server.cert --tls-private-key-file=/etc/srv/kubernetes/server.key --secure-port=443 --client-ca-file=/etc/srv/kubernetes/ca.crt --token-auth-file=/etc/srv/kubernetes/known_tokens.csv --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv --allow-privileged=true"
|
|
||||||
params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002"
|
|
||||||
if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then
|
|
||||||
params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
|
|
||||||
fi
|
|
||||||
if [ -n "${ADMISSION_CONTROL:-}" ]; then
|
|
||||||
params="${params} --admission-control=${ADMISSION_CONTROL}"
|
|
||||||
fi
|
|
||||||
if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
|
|
||||||
params="${params} --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}"
|
|
||||||
fi
|
|
||||||
if [ -n "${RUNTIME_CONFIG:-}" ]; then
|
|
||||||
params="${params} --runtime-config=${RUNTIME_CONFIG}"
|
|
||||||
fi
|
|
||||||
if [ -n "${APISERVER_TEST_ARGS:-}" ]; then
|
|
||||||
params="${params} ${APISERVER_TEST_ARGS}"
|
|
||||||
fi
|
|
||||||
log_level="--v=2"
|
|
||||||
if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then
|
|
||||||
log_level="${API_SERVER_TEST_LOG_LEVEL}"
|
|
||||||
fi
|
|
||||||
params="${params} ${log_level}"
|
|
||||||
|
|
||||||
if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
|
|
||||||
readonly vm_external_ip=$(curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip")
|
|
||||||
params="${params} --cloud-config=/etc/gce.conf --advertise-address=${vm_external_ip} --ssh-user=${PROXY_SSH_USER} --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile"
|
|
||||||
fi
|
|
||||||
readonly kube_apiserver_docker_tag=$(cat /run/kube-docker-files/kube-apiserver.docker_tag)
|
|
||||||
|
|
||||||
src_file="/run/kube-manifests/kubernetes/trusty/kube-apiserver.manifest"
|
|
||||||
remove_salt_config_comments "${src_file}"
|
|
||||||
# Evaluate variables
|
|
||||||
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{secure_port}}@443@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{secure_port}}@8080@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
|
|
||||||
cp "${src_file}" /etc/kubernetes/manifests
|
|
||||||
}
|
|
||||||
|
|
||||||
# Starts k8s controller manager.
|
|
||||||
# It prepares the log file, loads the docker image, calculates variables, sets them
|
|
||||||
# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
|
||||||
#
|
|
||||||
# Assumed vars (which are calculated in function compute_master_manifest_variables)
|
|
||||||
# CLOUD_CONFIG_VOLUME
|
|
||||||
# CLOUD_CONFIG_MOUNT
|
|
||||||
# DOCKER_REGISTRY
|
|
||||||
start_kube_controller_manager() {
|
|
||||||
prepare_log_file /var/log/kube-controller-manager.log
|
|
||||||
# Load the docker image from file.
|
|
||||||
echo "Try to load docker image file kube-controller-manager.tar"
|
|
||||||
timeout 30 docker load -i /run/kube-docker-files/kube-controller-manager.tar
|
|
||||||
|
|
||||||
# Calculate variables and assemble the command line.
|
|
||||||
params="--master=127.0.0.1:8080 --cloud-provider=gce --root-ca-file=/etc/srv/kubernetes/ca.crt --service-account-private-key-file=/etc/srv/kubernetes/server.key"
|
|
||||||
if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
|
|
||||||
params="${params} --cloud-config=/etc/gce.conf"
|
|
||||||
fi
|
|
||||||
if [ -n "${INSTANCE_PREFIX:-}" ]; then
|
|
||||||
params="${params} --cluster-name=${INSTANCE_PREFIX}"
|
|
||||||
fi
|
|
||||||
if [ -n "${CLUSTER_IP_RANGE:-}" ]; then
|
|
||||||
params="${params} --cluster-cidr=${CLUSTER_IP_RANGE}"
|
|
||||||
fi
|
|
||||||
if [ "${ALLOCATE_NODE_CIDRS:-}" = "true" ]; then
|
|
||||||
params="${params} --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}"
|
|
||||||
fi
|
|
||||||
if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then
|
|
||||||
params="${params} --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
|
|
||||||
fi
|
|
||||||
log_level="--v=2"
|
|
||||||
if [ -n "${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-}" ]; then
|
|
||||||
log_level="${CONTROLLER_MANAGER_TEST_LOG_LEVEL}"
|
|
||||||
fi
|
|
||||||
params="${params} ${log_level}"
|
|
||||||
if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then
|
|
||||||
params="${params} ${CONTROLLER_MANAGER_TEST_ARGS}"
|
|
||||||
fi
|
|
||||||
readonly kube_rc_docker_tag=$(cat /run/kube-docker-files/kube-controller-manager.docker_tag)
|
|
||||||
|
|
||||||
src_file="/run/kube-manifests/kubernetes/trusty/kube-controller-manager.manifest"
|
|
||||||
remove_salt_config_comments "${src_file}"
|
|
||||||
# Evaluate variables
|
|
||||||
sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{pillar\['kube-controller-manager_docker_tag'\]}}@${kube_rc_docker_tag}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
|
|
||||||
cp "${src_file}" /etc/kubernetes/manifests
|
|
||||||
}
|
|
||||||
|
|
||||||
# Starts k8s scheduler.
|
|
||||||
# It prepares the log file, loads the docker image, calculates variables, sets them
|
|
||||||
# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
|
||||||
#
|
|
||||||
# Assumed vars (which are calculated in compute_master_manifest_variables())
|
|
||||||
# DOCKER_REGISTRY
|
|
||||||
start_kube_scheduler() {
|
|
||||||
prepare_log_file /var/log/kube-scheduler.log
|
|
||||||
# Load the docker image from file.
|
|
||||||
echo "Try to load docker image file kube-scheduler.tar"
|
|
||||||
timeout 30 docker load -i /run/kube-docker-files/kube-scheduler.tar
|
|
||||||
|
|
||||||
# Calculate variables and set them in the manifest.
|
|
||||||
params=""
|
|
||||||
log_level="--v=2"
|
|
||||||
if [ -n "${SCHEDULER_TEST_LOG_LEVEL:-}" ]; then
|
|
||||||
log_level="${SCHEDULER_TEST_LOG_LEVEL}"
|
|
||||||
fi
|
|
||||||
params="${params} ${log_level}"
|
|
||||||
if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then
|
|
||||||
params="${params} ${SCHEDULER_TEST_ARGS}"
|
|
||||||
fi
|
|
||||||
readonly kube_scheduler_docker_tag=$(cat /run/kube-docker-files/kube-scheduler.docker_tag)
|
|
||||||
|
|
||||||
# Remove salt comments and replace variables with values
|
|
||||||
src_file="/run/kube-manifests/kubernetes/trusty/kube-scheduler.manifest"
|
|
||||||
remove_salt_config_comments "${src_file}"
|
|
||||||
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
|
||||||
sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" "${src_file}"
|
|
||||||
cp "${src_file}" /etc/kubernetes/manifests
|
|
||||||
}
|
|
||||||
|
|
||||||
# Starts a fluentd static pod for logging.
|
|
||||||
start_fluentd() {
|
|
||||||
if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then
|
|
||||||
if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then
|
|
||||||
cp /run/kube-manifests/kubernetes/fluentd-gcp.yaml /etc/kubernetes/manifests/
|
|
||||||
elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then
|
|
||||||
cp /run/kube-manifests/kubernetes/fluentd-es.yaml /etc/kubernetes/manifests/
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# A helper function for copying addon manifests and set dir/files
|
|
||||||
# permissions.
|
|
||||||
# $1: addon category under /etc/kubernetes
|
|
||||||
# $2: manifest source dir
|
|
||||||
setup_addon_manifests() {
|
|
||||||
src_dir="/run/kube-manifests/kubernetes/trusty/$2"
|
|
||||||
dst_dir="/etc/kubernetes/$1/$2"
|
|
||||||
if [ ! -d "${dst_dir}" ]; then
|
|
||||||
mkdir -p "${dst_dir}"
|
|
||||||
fi
|
|
||||||
files=$(find "${src_dir}" -name "*.yaml")
|
|
||||||
if [ -n "${files}" ]; then
|
|
||||||
cp "${src_dir}/"*.yaml "${dst_dir}"
|
|
||||||
fi
|
|
||||||
files=$(find "${src_dir}" -name "*.json")
|
|
||||||
if [ -n "${files}" ]; then
|
|
||||||
cp "${src_dir}/"*.json "${dst_dir}"
|
|
||||||
fi
|
|
||||||
files=$(find "${src_dir}" -name "*.yaml.in")
|
|
||||||
if [ -n "${files}" ]; then
|
|
||||||
cp "${src_dir}/"*.yaml.in "${dst_dir}"
|
|
||||||
fi
|
|
||||||
chown -R root:root "${dst_dir}"
|
|
||||||
chmod 755 "${dst_dir}"
|
|
||||||
chmod 644 "${dst_dir}"/*
|
|
||||||
}
|
|
||||||
|
|
||||||
# Prepares the manifests of k8s addons static pods.
|
|
||||||
prepare_kube_addons() {
|
|
||||||
addon_src_dir="/run/kube-manifests/kubernetes/trusty"
|
|
||||||
addon_dst_dir="/etc/kubernetes/addons"
|
|
||||||
# Set up manifests of other addons.
|
|
||||||
if [ "${ENABLE_CLUSTER_MONITORING:-}" = "influxdb" ] || \
|
|
||||||
[ "${ENABLE_CLUSTER_MONITORING:-}" = "google" ] || \
|
|
||||||
[ "${ENABLE_CLUSTER_MONITORING:-}" = "standalone" ] || \
|
|
||||||
[ "${ENABLE_CLUSTER_MONITORING:-}" = "googleinfluxdb" ]; then
|
|
||||||
file_dir="cluster-monitoring/${ENABLE_CLUSTER_MONITORING}"
|
|
||||||
setup_addon_manifests "addons" "${file_dir}"
|
|
||||||
# Replace the salt configurations with variable values.
|
|
||||||
metrics_memory="200Mi"
|
|
||||||
eventer_memory="200Mi"
|
|
||||||
if [ -n "${NUM_NODES:-}" ] && [ "${NUM_NODES}" -ge 1 ]; then
|
|
||||||
num_kube_nodes="$((${NUM_NODES}-1))"
|
|
||||||
metrics_memory="$((${num_kube_nodes} * 4 + 200))Mi"
|
|
||||||
eventer_memory="$((${num_kube_nodes} * 500 + 200 * 1024))Ki"
|
|
||||||
fi
|
|
||||||
controller_yaml="${addon_dst_dir}/${file_dir}"
|
|
||||||
if [ "${ENABLE_CLUSTER_MONITORING:-}" = "googleinfluxdb" ]; then
|
|
||||||
controller_yaml="${controller_yaml}/heapster-controller-combined.yaml"
|
|
||||||
else
|
|
||||||
controller_yaml="${controller_yaml}/heapster-controller.yaml"
|
|
||||||
fi
|
|
||||||
remove_salt_config_comments "${controller_yaml}"
|
|
||||||
sed -i -e "s@{{ *metrics_memory *}}@${metrics_memory}@g" "${controller_yaml}"
|
|
||||||
sed -i -e "s@{{ *eventer_memory *}}@${eventer_memory}@g" "${controller_yaml}"
|
|
||||||
fi
|
|
||||||
cp "${addon_src_dir}/namespace.yaml" "${addon_dst_dir}"
|
|
||||||
if [ "${ENABLE_L7_LOADBALANCING:-}" = "glbc" ]; then
|
|
||||||
setup_addon_manifests "addons" "cluster-loadbalancing/glbc"
|
|
||||||
fi
|
|
||||||
if [ "${ENABLE_CLUSTER_DNS:-}" = "true" ]; then
|
|
||||||
setup_addon_manifests "addons" "dns"
|
|
||||||
dns_rc_file="${addon_dst_dir}/dns/skydns-rc.yaml"
|
|
||||||
dns_svc_file="${addon_dst_dir}/dns/skydns-svc.yaml"
|
|
||||||
mv "${addon_dst_dir}/dns/skydns-rc.yaml.in" "${dns_rc_file}"
|
|
||||||
mv "${addon_dst_dir}/dns/skydns-svc.yaml.in" "${dns_svc_file}"
|
|
||||||
# Replace the salt configurations with variable values.
|
|
||||||
sed -i -e "s@{{ *pillar\['dns_replicas'\] *}}@${DNS_REPLICAS}@g" "${dns_rc_file}"
|
|
||||||
sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${dns_rc_file}"
|
|
||||||
sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${dns_svc_file}"
|
|
||||||
fi
|
|
||||||
if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then
|
|
||||||
setup_addon_manifests "addons" "registry"
|
|
||||||
registry_pv_file="${addon_dst_dir}/registry/registry-pv.yaml"
|
|
||||||
registry_pvc_file="${addon_dst_dir}/registry/registry-pvc.yaml"
|
|
||||||
mv "${addon_dst_dir}/registry/registry-pv.yaml.in" "${registry_pv_file}"
|
|
||||||
mv "${addon_dst_dir}/registry/registry-pvc.yaml.in" "${registry_pvc_file}"
|
|
||||||
# Replace the salt configurations with variable values.
|
|
||||||
remove_salt_config_comments "${controller_yaml}"
|
|
||||||
sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pv_file}"
|
|
||||||
sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pvc_file}"
|
|
||||||
sed -i -e "s@{{ *pillar\['cluster_registry_disk_name'\] *}}@${CLUSTER_REGISTRY_DISK}@g" "${registry_pvc_file}"
|
|
||||||
fi
|
|
||||||
if [ "${ENABLE_NODE_LOGGING:-}" = "true" ] && \
|
|
||||||
[ "${LOGGING_DESTINATION:-}" = "elasticsearch" ] && \
|
|
||||||
[ "${ENABLE_CLUSTER_LOGGING:-}" = "true" ]; then
|
|
||||||
setup_addon_manifests "addons" "fluentd-elasticsearch"
|
|
||||||
fi
|
|
||||||
if [ "${ENABLE_CLUSTER_UI:-}" = "true" ]; then
|
|
||||||
setup_addon_manifests "addons" "dashboard"
|
|
||||||
fi
|
|
||||||
if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then
|
|
||||||
setup_addon_manifests "admission-controls" "limit-range"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Prepare the scripts for running addons.
|
|
||||||
addon_script_dir="/var/lib/cloud/scripts/kubernetes"
|
|
||||||
mkdir -p "${addon_script_dir}"
|
|
||||||
cp "${addon_src_dir}/kube-addons.sh" "${addon_script_dir}"
|
|
||||||
cp "${addon_src_dir}/kube-addon-update.sh" "${addon_script_dir}"
|
|
||||||
chmod 544 "${addon_script_dir}/"*.sh
|
|
||||||
# In case that some GCE customized trusty may have a read-only /root.
|
|
||||||
mount -t tmpfs tmpfs /root
|
|
||||||
mount --bind -o remount,rw,noexec /root
|
|
||||||
}
|
|
||||||
|
|||||||
@ -1,7 +1,36 @@
|
|||||||
From nobody Thu Mar 3 15:33:00 2016
|
From nobody Thu Mar 10 10:33:00 2016
|
||||||
Content-Type: multipart/mixed; boundary="===================================="
|
Content-Type: multipart/mixed; boundary="===================================="
|
||||||
MIME-Version: 1.0
|
MIME-Version: 1.0
|
||||||
|
|
||||||
|
--====================================
|
||||||
|
MIME-Version: 1.0
|
||||||
|
Content-Type: text/upstart-job; charset="us-ascii"
|
||||||
|
Content-Transfer-Encoding: 7bit
|
||||||
|
Content-Disposition: attachment; filename="kube-install-master.conf"
|
||||||
|
|
||||||
|
#upstart-job
|
||||||
|
|
||||||
|
description "Download and install k8s binaries and configurations"
|
||||||
|
|
||||||
|
start on cloud-config
|
||||||
|
|
||||||
|
script
|
||||||
|
set -o errexit
|
||||||
|
set -o nounset
|
||||||
|
|
||||||
|
# Fetch the script for installing master binary and configuration files.
|
||||||
|
curl --fail --silent --show-error \
|
||||||
|
-H "X-Google-Metadata-Request: True" \
|
||||||
|
-o /etc/kube-configure.sh \
|
||||||
|
http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
|
||||||
|
. /etc/kube-configure.sh
|
||||||
|
echo "Downloading kube-env file"
|
||||||
|
download_kube_env
|
||||||
|
. /etc/kube-env
|
||||||
|
echo "Install kube master binary and configuration files"
|
||||||
|
install_kube_binary_config
|
||||||
|
end script
|
||||||
|
|
||||||
--====================================
|
--====================================
|
||||||
MIME-Version: 1.0
|
MIME-Version: 1.0
|
||||||
Content-Type: text/upstart-job; charset="us-ascii"
|
Content-Type: text/upstart-job; charset="us-ascii"
|
||||||
@ -12,26 +41,18 @@ Content-Disposition: attachment; filename="kube-env.conf"
|
|||||||
|
|
||||||
description "Prepare kube master environment"
|
description "Prepare kube master environment"
|
||||||
|
|
||||||
start on cloud-config
|
start on stopped kube-install-master
|
||||||
|
|
||||||
script
|
script
|
||||||
set -o errexit
|
set -o errexit
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
# Fetch the script for configuring the instance.
|
. /etc/kube-configure-helper.sh
|
||||||
curl --fail --silent --show-error \
|
. /etc/kube-env
|
||||||
-H "X-Google-Metadata-Request: True" \
|
|
||||||
-o /etc/kube-configure.sh \
|
|
||||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
|
|
||||||
. /etc/kube-configure.sh
|
|
||||||
|
|
||||||
echo "Configuring hostname"
|
echo "Configuring hostname"
|
||||||
config_hostname
|
config_hostname
|
||||||
echo "Configuring IP firewall rules"
|
echo "Configuring IP firewall rules"
|
||||||
config_ip_firewall
|
config_ip_firewall
|
||||||
echo "Downloading kube-env file"
|
|
||||||
download_kube_env
|
|
||||||
. /etc/kube-env
|
|
||||||
echo "Creating required directories"
|
echo "Creating required directories"
|
||||||
create_dirs
|
create_dirs
|
||||||
echo "Mount master PD"
|
echo "Mount master PD"
|
||||||
@ -52,13 +73,13 @@ Content-Disposition: attachment; filename="kube-install-packages.conf"
|
|||||||
|
|
||||||
description "Install packages needed to run kubernetes"
|
description "Install packages needed to run kubernetes"
|
||||||
|
|
||||||
start on stopped kube-env
|
start on stopped kube-install-master
|
||||||
|
|
||||||
script
|
script
|
||||||
set -o errexit
|
set -o errexit
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
install_critical_packages
|
install_critical_packages
|
||||||
end script
|
end script
|
||||||
|
|
||||||
@ -78,31 +99,10 @@ script
|
|||||||
set -o errexit
|
set -o errexit
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
install_additional_packages
|
install_additional_packages
|
||||||
end script
|
end script
|
||||||
|
|
||||||
--====================================
|
|
||||||
MIME-Version: 1.0
|
|
||||||
Content-Type: text/upstart-job; charset="us-ascii"
|
|
||||||
Content-Transfer-Encoding: 7bit
|
|
||||||
Content-Disposition: attachment; filename="kube-install-master.conf"
|
|
||||||
|
|
||||||
#upstart-job
|
|
||||||
|
|
||||||
description "Download and install k8s binaries and configurations"
|
|
||||||
|
|
||||||
start on stopped kube-env
|
|
||||||
|
|
||||||
script
|
|
||||||
set -o errexit
|
|
||||||
set -o nounset
|
|
||||||
|
|
||||||
. /etc/kube-configure.sh
|
|
||||||
. /etc/kube-env
|
|
||||||
install_kube_binary_config
|
|
||||||
end script
|
|
||||||
|
|
||||||
--====================================
|
--====================================
|
||||||
MIME-Version: 1.0
|
MIME-Version: 1.0
|
||||||
Content-Type: text/upstart-job; charset="us-ascii"
|
Content-Type: text/upstart-job; charset="us-ascii"
|
||||||
@ -113,7 +113,7 @@ Content-Disposition: attachment; filename="kubelet.conf"
|
|||||||
|
|
||||||
description "Run kubelet service"
|
description "Run kubelet service"
|
||||||
|
|
||||||
start on stopped kube-install-packages and stopped kube-install-master
|
start on stopped kube-install-packages and stopped kube-env
|
||||||
|
|
||||||
respawn
|
respawn
|
||||||
|
|
||||||
@ -122,7 +122,7 @@ script
|
|||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
echo "Start kubelet upstart job"
|
echo "Start kubelet upstart job"
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
. /etc/kube-env
|
. /etc/kube-env
|
||||||
# Assemble command line flags based on env variables, which will put the string
|
# Assemble command line flags based on env variables, which will put the string
|
||||||
# of flags in variable KUBELET_CMD_FLAGS
|
# of flags in variable KUBELET_CMD_FLAGS
|
||||||
@ -163,7 +163,7 @@ script
|
|||||||
set -o errexit
|
set -o errexit
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
. /etc/kube-env
|
. /etc/kube-env
|
||||||
restart_docker_daemon
|
restart_docker_daemon
|
||||||
end script
|
end script
|
||||||
@ -184,7 +184,7 @@ script
|
|||||||
set -o errexit
|
set -o errexit
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
. /etc/kube-env
|
. /etc/kube-env
|
||||||
start_etcd_servers
|
start_etcd_servers
|
||||||
start_fluentd
|
start_fluentd
|
||||||
@ -247,7 +247,7 @@ script
|
|||||||
# Wait for a minute to let docker and kubelet processes finish initialization.
|
# Wait for a minute to let docker and kubelet processes finish initialization.
|
||||||
# TODO(andyzheng0831): replace it with a more reliable method if possible.
|
# TODO(andyzheng0831): replace it with a more reliable method if possible.
|
||||||
sleep 60
|
sleep 60
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
. /etc/kube-env
|
. /etc/kube-env
|
||||||
health_monitoring
|
health_monitoring
|
||||||
end script
|
end script
|
||||||
|
|||||||
@ -1,7 +1,36 @@
|
|||||||
From nobody Thu Mar 3 15:33:00 2016
|
From nobody Thu Mar 10 10:33:00 2016
|
||||||
Content-Type: multipart/mixed; boundary="===================================="
|
Content-Type: multipart/mixed; boundary="===================================="
|
||||||
MIME-Version: 1.0
|
MIME-Version: 1.0
|
||||||
|
|
||||||
|
--====================================
|
||||||
|
MIME-Version: 1.0
|
||||||
|
Content-Type: text/upstart-job; charset="us-ascii"
|
||||||
|
Content-Transfer-Encoding: 7bit
|
||||||
|
Content-Disposition: attachment; filename="kube-install-minion.conf"
|
||||||
|
|
||||||
|
#upstart-job
|
||||||
|
|
||||||
|
description "Download and install k8s binaries and configurations"
|
||||||
|
|
||||||
|
start on cloud-config
|
||||||
|
|
||||||
|
script
|
||||||
|
set -o errexit
|
||||||
|
set -o nounset
|
||||||
|
|
||||||
|
# Fetch the script for installing nodes binary and configuration files.
|
||||||
|
curl --fail --silent --show-error \
|
||||||
|
-H "X-Google-Metadata-Request: True" \
|
||||||
|
-o /etc/kube-configure.sh \
|
||||||
|
http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
|
||||||
|
. /etc/kube-configure.sh
|
||||||
|
echo "Downloading kube-env file"
|
||||||
|
download_kube_env
|
||||||
|
. /etc/kube-env
|
||||||
|
echo "Install kube nodes binary and configuration files"
|
||||||
|
install_kube_binary_config
|
||||||
|
end script
|
||||||
|
|
||||||
--====================================
|
--====================================
|
||||||
MIME-Version: 1.0
|
MIME-Version: 1.0
|
||||||
Content-Type: text/upstart-job; charset="us-ascii"
|
Content-Type: text/upstart-job; charset="us-ascii"
|
||||||
@ -12,27 +41,19 @@ Content-Disposition: attachment; filename="kube-env.conf"
|
|||||||
|
|
||||||
description "Prepare kube node environment"
|
description "Prepare kube node environment"
|
||||||
|
|
||||||
start on cloud-config
|
start on stopped kube-install-minion
|
||||||
|
|
||||||
script
|
script
|
||||||
set -o errexit
|
set -o errexit
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
# Fetch the script for configuring the instance.
|
. /etc/kube-configure-helper.sh
|
||||||
curl --fail --silent --show-error \
|
. /etc/kube-env
|
||||||
-H "X-Google-Metadata-Request: True" \
|
|
||||||
-o /etc/kube-configure.sh \
|
|
||||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
|
|
||||||
. /etc/kube-configure.sh
|
|
||||||
|
|
||||||
echo "Configuring hostname"
|
echo "Configuring hostname"
|
||||||
config_hostname
|
config_hostname
|
||||||
echo "Configuring IP firewall rules"
|
echo "Configuring IP firewall rules"
|
||||||
config_ip_firewall
|
config_ip_firewall
|
||||||
echo "Downloading kube-env file"
|
|
||||||
download_kube_env
|
|
||||||
echo "Creating required directories"
|
echo "Creating required directories"
|
||||||
. /etc/kube-env
|
|
||||||
create_dirs
|
create_dirs
|
||||||
echo "Creating kubelet kubeconfig file"
|
echo "Creating kubelet kubeconfig file"
|
||||||
create_kubelet_kubeconfig
|
create_kubelet_kubeconfig
|
||||||
@ -50,13 +71,13 @@ Content-Disposition: attachment; filename="kube-install-packages.conf"
|
|||||||
|
|
||||||
description "Install packages needed to run kubernetes"
|
description "Install packages needed to run kubernetes"
|
||||||
|
|
||||||
start on stopped kube-env
|
start on stopped kube-install-minion
|
||||||
|
|
||||||
script
|
script
|
||||||
set -o errexit
|
set -o errexit
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
install_critical_packages
|
install_critical_packages
|
||||||
end script
|
end script
|
||||||
|
|
||||||
@ -76,31 +97,10 @@ script
|
|||||||
set -o errexit
|
set -o errexit
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
install_additional_packages
|
install_additional_packages
|
||||||
end script
|
end script
|
||||||
|
|
||||||
--====================================
|
|
||||||
MIME-Version: 1.0
|
|
||||||
Content-Type: text/upstart-job; charset="us-ascii"
|
|
||||||
Content-Transfer-Encoding: 7bit
|
|
||||||
Content-Disposition: attachment; filename="kube-install-minion.conf"
|
|
||||||
|
|
||||||
#upstart-job
|
|
||||||
|
|
||||||
description "Download and install k8s binaries and configurations"
|
|
||||||
|
|
||||||
start on stopped kube-env
|
|
||||||
|
|
||||||
script
|
|
||||||
set -o errexit
|
|
||||||
set -o nounset
|
|
||||||
|
|
||||||
. /etc/kube-configure.sh
|
|
||||||
. /etc/kube-env
|
|
||||||
install_kube_binary_config
|
|
||||||
end script
|
|
||||||
|
|
||||||
--====================================
|
--====================================
|
||||||
MIME-Version: 1.0
|
MIME-Version: 1.0
|
||||||
Content-Type: text/upstart-job; charset="us-ascii"
|
Content-Type: text/upstart-job; charset="us-ascii"
|
||||||
@ -111,7 +111,7 @@ Content-Disposition: attachment; filename="kubelet.conf"
|
|||||||
|
|
||||||
description "Run kubelet service"
|
description "Run kubelet service"
|
||||||
|
|
||||||
start on stopped kube-install-minion and stopped kube-install-packages
|
start on stopped kube-install-packages and stopped kube-env
|
||||||
|
|
||||||
respawn
|
respawn
|
||||||
|
|
||||||
@ -120,7 +120,7 @@ script
|
|||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
echo "Start kubelet upstart job"
|
echo "Start kubelet upstart job"
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
. /etc/kube-env
|
. /etc/kube-env
|
||||||
# Assemble command line flags based on env variables, which will put the string
|
# Assemble command line flags based on env variables, which will put the string
|
||||||
# of flags in variable KUBELET_CMD_FLAGS.
|
# of flags in variable KUBELET_CMD_FLAGS.
|
||||||
@ -162,7 +162,7 @@ script
|
|||||||
set -o errexit
|
set -o errexit
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
. /etc/kube-env
|
. /etc/kube-env
|
||||||
restart_docker_daemon
|
restart_docker_daemon
|
||||||
end script
|
end script
|
||||||
@ -183,7 +183,7 @@ script
|
|||||||
set -o errexit
|
set -o errexit
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
. /etc/kube-env
|
. /etc/kube-env
|
||||||
prepare_log_file "/var/log/kube-proxy.log"
|
prepare_log_file "/var/log/kube-proxy.log"
|
||||||
# Load the docker image from file /run/kube-docker-files/kube-proxy.tar.
|
# Load the docker image from file /run/kube-docker-files/kube-proxy.tar.
|
||||||
@ -239,7 +239,7 @@ script
|
|||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
# Kube-system pod manifest files are located at /run/kube-manifests/kubernetes.
|
# Kube-system pod manifest files are located at /run/kube-manifests/kubernetes.
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
. /etc/kube-env
|
. /etc/kube-env
|
||||||
# Fluentd
|
# Fluentd
|
||||||
start_fluentd
|
start_fluentd
|
||||||
@ -268,7 +268,7 @@ script
|
|||||||
# Wait for a minute to let docker, kubelet, and kube-proxy processes finish initialization.
|
# Wait for a minute to let docker, kubelet, and kube-proxy processes finish initialization.
|
||||||
# TODO(andyzheng0831): replace it with a more reliable method if possible.
|
# TODO(andyzheng0831): replace it with a more reliable method if possible.
|
||||||
sleep 60
|
sleep 60
|
||||||
. /etc/kube-configure.sh
|
. /etc/kube-configure-helper.sh
|
||||||
. /etc/kube-env
|
. /etc/kube-env
|
||||||
health_monitoring
|
health_monitoring
|
||||||
end script
|
end script
|
||||||
|
|||||||
@ -13,7 +13,7 @@ cluster/gce/configure-vm.sh: cloud_config: ${CLOUD_CONFIG}
|
|||||||
cluster/gce/configure-vm.sh: env-to-grains "runtime_config"
|
cluster/gce/configure-vm.sh: env-to-grains "runtime_config"
|
||||||
cluster/gce/configure-vm.sh: kubelet_api_servers: '${KUBELET_APISERVER}'
|
cluster/gce/configure-vm.sh: kubelet_api_servers: '${KUBELET_APISERVER}'
|
||||||
cluster/gce/coreos/helper.sh:# cloud_config yaml file should be passed
|
cluster/gce/coreos/helper.sh:# cloud_config yaml file should be passed
|
||||||
cluster/gce/trusty/configure.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
|
cluster/gce/trusty/configure-helper.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
|
||||||
cluster/gce/util.sh: local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
|
cluster/gce/util.sh: local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
|
||||||
cluster/juju/charms/trusty/kubernetes-master/files/controller-manager.upstart.tmpl: --address=%(bind_address)s \
|
cluster/juju/charms/trusty/kubernetes-master/files/controller-manager.upstart.tmpl: --address=%(bind_address)s \
|
||||||
cluster/juju/charms/trusty/kubernetes-master/files/scheduler.upstart.tmpl: --address=%(bind_address)s \
|
cluster/juju/charms/trusty/kubernetes-master/files/scheduler.upstart.tmpl: --address=%(bind_address)s \
|
||||||
@ -54,7 +54,6 @@ cluster/lib/logging.sh: local source_file=${BASH_SOURCE[$frame_no]}
|
|||||||
cluster/lib/logging.sh: local source_file=${BASH_SOURCE[$stack_skip]}
|
cluster/lib/logging.sh: local source_file=${BASH_SOURCE[$stack_skip]}
|
||||||
cluster/log-dump.sh: for node_name in "${NODE_NAMES[@]}"; do
|
cluster/log-dump.sh: for node_name in "${NODE_NAMES[@]}"; do
|
||||||
cluster/log-dump.sh: local -r node_name="${1}"
|
cluster/log-dump.sh: local -r node_name="${1}"
|
||||||
cluster/log-dump.sh: local -r node_name="${1}"
|
|
||||||
cluster/log-dump.sh:readonly report_dir="${1:-_artifacts}"
|
cluster/log-dump.sh:readonly report_dir="${1:-_artifacts}"
|
||||||
cluster/mesos/docker/km/build.sh: km_path=$(find-binary km darwin/amd64)
|
cluster/mesos/docker/km/build.sh: km_path=$(find-binary km darwin/amd64)
|
||||||
cluster/rackspace/util.sh: local node_ip=$(nova show --minimal ${NODE_NAMES[$i]} \
|
cluster/rackspace/util.sh: local node_ip=$(nova show --minimal ${NODE_NAMES[$i]} \
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user