diff --git a/cluster/gce/coreos/README.md b/cluster/gce/coreos/README.md new file mode 100644 index 00000000000..64d36a8a5ae --- /dev/null +++ b/cluster/gce/coreos/README.md @@ -0,0 +1,11 @@ +# Container-VM Image + +[Container-VM Image](https://cloud.google.com/compute/docs/containers/vm-image/) +is a container-optimized OS image for the Google Cloud Platform (GCP). It is +primarily for running Google services on GCP. Unlike the open preview version +of container-vm, the new Container-VM Image is based on the open source +ChromiumOS project, allowing us greater control over the build management, +security compliance, and customizations for GCP. + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/gce/gci/README.md?pixel)]() diff --git a/cluster/gce/coreos/configure-helper.sh b/cluster/gce/coreos/configure-helper.sh new file mode 100644 index 00000000000..01bfbff6e96 --- /dev/null +++ b/cluster/gce/coreos/configure-helper.sh @@ -0,0 +1,1302 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is for configuring kubernetes master and node instances. It is +# uploaded in the manifests tar ball. + +# TODO: this script duplicates templating logic from cluster/saltbase/salt +# using sed. It should use an actual template parser on the manifest +# files. + +set -o errexit +set -o nounset +set -o pipefail + +function setup-os-params { + # Reset core_pattern. On GCI, the default core_pattern pipes the core dumps to + # /sbin/crash_reporter which is more restrictive in saving crash dumps. So for + # now, set a generic core_pattern that users can work with. + echo "core.%e.%p.%t" > /proc/sys/kernel/core_pattern +} + +function config-ip-firewall { + echo "Configuring IP firewall rules" + # The GCI image has host firewall which drop most inbound/forwarded packets. + # We need to add rules to accept all TCP/UDP/ICMP packets. + if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then + echo "Add rules to accept all inbound TCP/UDP/ICMP packets" + iptables -A INPUT -w -p TCP -j ACCEPT + iptables -A INPUT -w -p UDP -j ACCEPT + iptables -A INPUT -w -p ICMP -j ACCEPT + fi + if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then + echo "Add rules to accept all forwarded TCP/UDP/ICMP packets" + iptables -A FORWARD -w -p TCP -j ACCEPT + iptables -A FORWARD -w -p UDP -j ACCEPT + iptables -A FORWARD -w -p ICMP -j ACCEPT + fi +} + +function create-dirs { + echo "Creating required directories" + mkdir -p /var/lib/kubelet + mkdir -p /etc/kubernetes/manifests + if [[ "${KUBERNETES_MASTER:-}" == "false" ]]; then + mkdir -p /var/lib/kube-proxy + fi +} + +# Formats the given device ($1) if needed and mounts it at given mount point +# ($2). +function safe-format-and-mount() { + device=$1 + mountpoint=$2 + + # Format only if the disk is not already formatted. + if ! tune2fs -l "${device}" ; then + echo "Formatting '${device}'" + mkfs.ext4 -F -E lazy_itable_init=0,lazy_journal_init=0,discard "${device}" + fi + + mkdir -p "${mountpoint}" + echo "Mounting '${device}' at '${mountpoint}'" + mount -o discard,defaults "${device}" "${mountpoint}" +} + +# Local ssds, if present, are mounted at /mnt/disks/ssdN. +function ensure-local-ssds() { + for ssd in /dev/disk/by-id/google-local-ssd-*; do + if [ -e "${ssd}" ]; then + ssdnum=`echo ${ssd} | sed -e 's/\/dev\/disk\/by-id\/google-local-ssd-\([0-9]*\)/\1/'` + ssdmount="/mnt/disks/ssd${ssdnum}/" + mkdir -p ${ssdmount} + safe-format-and-mount "${ssd}" ${ssdmount} + echo "Mounted local SSD $ssd at ${ssdmount}" + chmod a+w ${ssdmount} + else + echo "No local SSD disks found." + fi + done +} + +# Installs logrotate configuration files +function setup-logrotate() { + mkdir -p /etc/logrotate.d/ + cat >/etc/logrotate.d/docker-containers < 100Mb OR if one day has elapsed + # * save rotated logs into a gzipped timestamped backup + # * log file timestamp (controlled by 'dateformat') includes seconds too. This + # ensures that logrotate can generate unique logfiles during each rotation + # (otherwise it skips rotation if 'maxsize' is reached multiple times in a + # day). + # * keep only 5 old (rotated) logs, and will discard older logs. + cat > /etc/logrotate.d/allvarlogs </dev/null; then + useradd -s /sbin/nologin -d /var/etcd etcd + fi + chown -R etcd "${mount_point}/var/etcd" + chgrp -R etcd "${mount_point}/var/etcd" +} + +# After the first boot and on upgrade, these files exist on the master-pd +# and should never be touched again (except perhaps an additional service +# account, see NB below.) +function create-master-auth { + echo "Creating master auth files" + local -r auth_dir="/etc/srv/kubernetes" + if [[ ! -e "${auth_dir}/ca.crt" && ! -z "${CA_CERT:-}" && ! -z "${MASTER_CERT:-}" && ! -z "${MASTER_KEY:-}" ]]; then + echo "${CA_CERT}" | base64 --decode > "${auth_dir}/ca.crt" + echo "${MASTER_CERT}" | base64 --decode > "${auth_dir}/server.cert" + echo "${MASTER_KEY}" | base64 --decode > "${auth_dir}/server.key" + fi + local -r basic_auth_csv="${auth_dir}/basic_auth.csv" + if [[ ! -e "${basic_auth_csv}" && -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then + echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${basic_auth_csv}" + fi + local -r known_tokens_csv="${auth_dir}/known_tokens.csv" + if [[ ! -e "${known_tokens_csv}" ]]; then + echo "${KUBE_BEARER_TOKEN},admin,admin" > "${known_tokens_csv}" + echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${known_tokens_csv}" + echo "${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy" >> "${known_tokens_csv}" + fi + local use_cloud_config="false" + cat </etc/gce.conf +[global] +EOF + if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then + use_cloud_config="true" + cat <>/etc/gce.conf +token-url = ${TOKEN_URL} +token-body = ${TOKEN_BODY} +project-id = ${PROJECT_ID} +network-name = ${NODE_NETWORK} +EOF + fi + if [[ -n "${NODE_INSTANCE_PREFIX:-}" ]]; then + use_cloud_config="true" + if [[ -n "${NODE_TAGS:-}" ]]; then + local -r node_tags="${NODE_TAGS}" + else + local -r node_tags="${NODE_INSTANCE_PREFIX}" + fi + cat <>/etc/gce.conf +node-tags = ${node_tags} +node-instance-prefix = ${NODE_INSTANCE_PREFIX} +EOF + fi + if [[ -n "${MULTIZONE:-}" ]]; then + use_cloud_config="true" + cat <>/etc/gce.conf +multizone = ${MULTIZONE} +EOF + fi + if [[ "${use_cloud_config}" != "true" ]]; then + rm -f /etc/gce.conf + fi + + if [[ -n "${GCP_AUTHN_URL:-}" ]]; then + cat </etc/gcp_authn.config +clusters: + - name: gcp-authentication-server + cluster: + server: ${GCP_AUTHN_URL} +users: + - name: kube-apiserver + user: + auth-provider: + name: gcp +current-context: webhook +contexts: +- context: + cluster: gcp-authentication-server + user: kube-apiserver + name: webhook +EOF + fi + + if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then + cat </etc/gcp_authz.config +clusters: + - name: gcp-authorization-server + cluster: + server: ${GCP_AUTHZ_URL} +users: + - name: kube-apiserver + user: + auth-provider: + name: gcp +current-context: webhook +contexts: +- context: + cluster: gcp-authorization-server + user: kube-apiserver + name: webhook +EOF + fi + +if [[ -n "${GCP_IMAGE_VERIFICATION_URL:-}" ]]; then + # This is the config file for the image review webhook. + cat </etc/gcp_image_review.config +clusters: + - name: gcp-image-review-server + cluster: + server: ${GCP_IMAGE_VERIFICATION_URL} +users: + - name: kube-apiserver + user: + auth-provider: + name: gcp +current-context: webhook +contexts: +- context: + cluster: gcp-image-review-server + user: kube-apiserver + name: webhook +EOF + # This is the config for the image review admission controller. + cat </etc/admission_controller.config +imagePolicy: + kubeConfigFile: /etc/gcp_image_review.config + allowTTL: 30 + denyTTL: 30 + retryBackoff: 500 + defaultAllow: true +EOF + fi +} + +function create-kubelet-kubeconfig { + echo "Creating kubelet kubeconfig file" + if [[ -z "${KUBELET_CA_CERT:-}" ]]; then + KUBELET_CA_CERT="${CA_CERT}" + fi + cat </var/lib/kubelet/kubeconfig +apiVersion: v1 +kind: Config +users: +- name: kubelet + user: + client-certificate-data: ${KUBELET_CERT} + client-key-data: ${KUBELET_KEY} +clusters: +- name: local + cluster: + certificate-authority-data: ${KUBELET_CA_CERT} +contexts: +- context: + cluster: local + user: kubelet + name: service-account-context +current-context: service-account-context +EOF +} + +# Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY +# to generate a kubeconfig file for the kubelet to securely connect to the apiserver. +# Set REGISTER_MASTER_KUBELET to true if kubelet on the master node +# should register to the apiserver. +function create-master-kubelet-auth { + # Only configure the kubelet on the master if the required variables are + # set in the environment. + if [[ -n "${KUBELET_APISERVER:-}" && -n "${KUBELET_CERT:-}" && -n "${KUBELET_KEY:-}" ]]; then + REGISTER_MASTER_KUBELET="true" + create-kubelet-kubeconfig + fi +} + +function create-kubeproxy-kubeconfig { + echo "Creating kube-proxy kubeconfig file" + cat </var/lib/kube-proxy/kubeconfig +apiVersion: v1 +kind: Config +users: +- name: kube-proxy + user: + token: ${KUBE_PROXY_TOKEN} +clusters: +- name: local + cluster: + certificate-authority-data: ${CA_CERT} +contexts: +- context: + cluster: local + user: kube-proxy + name: service-account-context +current-context: service-account-context +EOF +} + +function create-master-etcd-auth { + if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then + local -r auth_dir="/etc/srv/kubernetes" + echo "${ETCD_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-ca.crt" + echo "${ETCD_PEER_KEY}" | base64 --decode > "${auth_dir}/etcd-peer.key" + echo "${ETCD_PEER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-peer.crt" + fi +} + +function assemble-docker-flags { + echo "Assemble docker command line flags" + local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false" + if [[ "${TEST_CLUSTER:-}" == "true" ]]; then + docker_opts+=" --log-level=debug" + else + docker_opts+=" --log-level=warn" + fi + local use_net_plugin="true" + if [[ "${NETWORK_PROVIDER:-}" == "kubenet" || "${NETWORK_PROVIDER:-}" == "cni" ]]; then + # set docker0 cidr to private ip address range to avoid conflict with cbr0 cidr range + docker_opts+=" --bip=169.254.123.1/24" + else + use_net_plugin="false" + docker_opts+=" --bridge=cbr0" + fi + + # Decide whether to enable a docker registry mirror. This is taken from + # the "kube-env" metadata value. + if [[ -n "${DOCKER_REGISTRY_MIRROR_URL:-}" ]]; then + echo "Enable docker registry mirror at: ${DOCKER_REGISTRY_MIRROR_URL}" + docker_opts+=" --registry-mirror=${DOCKER_REGISTRY_MIRROR_URL}" + fi + + echo "DOCKER_OPTS=\"${docker_opts} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker + + if [[ "${use_net_plugin}" == "true" ]]; then + # If using a network plugin, extend the docker configuration to always remove + # the network checkpoint to avoid corrupt checkpoints. + # (https://github.com/docker/docker/issues/18283). + echo "Extend the default docker.service configuration" + mkdir -p /etc/systemd/system/docker.service.d + cat </etc/systemd/system/docker.service.d/01network.conf +[Service] +ExecStartPre=/bin/sh -x -c "rm -rf /var/lib/docker/network" +EOF + + systemctl daemon-reload + + # If using a network plugin, we need to explicitly restart docker daemon, because + # kubelet will not do it. + echo "Docker command line is updated. Restart docker to pick it up" + systemctl restart docker + fi +} + +# A helper function for loading a docker image. It keeps trying up to 5 times. +# +# $1: Full path of the docker image +function try-load-docker-image { + local -r img=$1 + echo "Try to load docker image file ${img}" + # Temporarily turn off errexit, because we don't want to exit on first failure. + set +e + local -r max_attempts=5 + local -i attempt_num=1 + until timeout 30 docker load -i "${img}"; do + if [[ "${attempt_num}" == "${max_attempts}" ]]; then + echo "Fail to load docker image file ${img} after ${max_attempts} retries. Exit!!" + exit 1 + else + attempt_num=$((attempt_num+1)) + sleep 5 + fi + done + # Re-enable errexit. + set -e +} + +# Loads kube-system docker images. It is better to do it before starting kubelet, +# as kubelet will restart docker daemon, which may interfere with loading images. +function load-docker-images { + echo "Start loading kube-system docker images" + local -r img_dir="${KUBE_HOME}/kube-docker-files" + if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then + try-load-docker-image "${img_dir}/kube-apiserver.tar" + try-load-docker-image "${img_dir}/kube-controller-manager.tar" + try-load-docker-image "${img_dir}/kube-scheduler.tar" + else + try-load-docker-image "${img_dir}/kube-proxy.tar" + fi +} + +# This function assembles the kubelet systemd service file and starts it +# using systemctl. +function start-kubelet { + echo "Start kubelet" + local kubelet_bin="${KUBE_HOME}/bin/kubelet" + local -r version="$("${kubelet_bin}" --version=true | cut -f2 -d " ")" + local -r builtin_kubelet="/usr/bin/kubelet" + if [[ "${TEST_CLUSTER:-}" == "true" ]]; then + # Determine which binary to use on test clusters. We use the built-in + # version only if the downloaded version is the same as the built-in + # version. This allows GCI to run some of the e2e tests to qualify the + # built-in kubelet. + if [[ -x "${builtin_kubelet}" ]]; then + local -r builtin_version="$("${builtin_kubelet}" --version=true | cut -f2 -d " ")" + if [[ "${builtin_version}" == "${version}" ]]; then + kubelet_bin="${builtin_kubelet}" + fi + fi + fi + echo "Using kubelet binary at ${kubelet_bin}" + local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}" + flags+=" --allow-privileged=true" + flags+=" --babysit-daemons=true" + flags+=" --cgroup-root=/" + flags+=" --cloud-provider=gce" + flags+=" --cluster-dns=${DNS_SERVER_IP}" + flags+=" --cluster-domain=${DNS_DOMAIN}" + flags+=" --config=/etc/kubernetes/manifests" + flags+=" --experimental-mounter-path=${KUBE_HOME}/bin/mounter" + flags+=" --experimental-check-node-capabilities-before-mount=true" + + if [[ -n "${KUBELET_PORT:-}" ]]; then + flags+=" --port=${KUBELET_PORT}" + fi + if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then + flags+=" --enable-debugging-handlers=false" + flags+=" --hairpin-mode=none" + if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then + flags+=" --api-servers=https://${KUBELET_APISERVER}" + flags+=" --register-schedulable=false" + flags+=" --register-with-taints=node.alpha.kubernetes.io/ismaster=:NoSchedule" + else + # Standalone mode (not widely used?) + flags+=" --pod-cidr=${MASTER_IP_RANGE}" + fi + else # For nodes + flags+=" --enable-debugging-handlers=true" + flags+=" --api-servers=https://${KUBERNETES_MASTER_NAME}" + if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \ + [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \ + [[ "${HAIRPIN_MODE:-}" == "none" ]]; then + flags+=" --hairpin-mode=${HAIRPIN_MODE}" + fi + fi + # Network plugin + if [[ -n "${NETWORK_PROVIDER:-}" ]]; then + if [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then + flags+=" --cni-bin-dir=/home/kubernetes/bin" + else + flags+=" --network-plugin-dir=/home/kubernetes/bin" + fi + flags+=" --network-plugin=${NETWORK_PROVIDER}" + fi + if [[ -n "${NON_MASQUERADE_CIDR:-}" ]]; then + flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}" + fi + if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then + flags+=" --manifest-url=${MANIFEST_URL}" + flags+=" --manifest-url-header=${MANIFEST_URL_HEADER}" + fi + if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then + flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" + fi + if [[ -n "${NODE_LABELS:-}" ]]; then + flags+=" --node-labels=${NODE_LABELS}" + fi + if [[ -n "${EVICTION_HARD:-}" ]]; then + flags+=" --eviction-hard=${EVICTION_HARD}" + fi + if [[ -n "${FEATURE_GATES:-}" ]]; then + flags+=" --feature-gates=${FEATURE_GATES}" + fi + + local -r kubelet_env_file="/etc/default/kubelet" + echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}" + + # Write the systemd service file for kubelet. + cat </etc/systemd/system/kubelet.service +[Unit] +Description=Kubernetes kubelet +Requires=network-online.target +After=network-online.target + +[Service] +Restart=always +RestartSec=10 +EnvironmentFile=${kubelet_env_file} +ExecStart=${kubelet_bin} \$KUBELET_OPTS + +[Install] +WantedBy=multi-user.target +EOF + + # Flush iptables nat table + iptables -t nat -F || true + + systemctl start kubelet.service +} + +# Create the log file and set its properties. +# +# $1 is the file to create. +function prepare-log-file { + touch $1 + chmod 644 $1 + chown root:root $1 +} + +# Starts kube-proxy pod. +function start-kube-proxy { + echo "Start kube-proxy pod" + prepare-log-file /var/log/kube-proxy.log + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" + remove-salt-config-comments "${src_file}" + + local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig" + local kube_docker_registry="gcr.io/google_containers" + if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then + kube_docker_registry=${KUBE_DOCKER_REGISTRY} + fi + local -r kube_proxy_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-proxy.docker_tag) + local api_servers="--master=https://${KUBERNETES_MASTER_NAME}" + local params="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=2"}" + if [[ -n "${FEATURE_GATES:-}" ]]; then + params+=" --feature-gates=${FEATURE_GATES}" + fi + if [[ -n "${KUBEPROXY_TEST_ARGS:-}" ]]; then + params+=" ${KUBEPROXY_TEST_ARGS}" + fi + sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file} + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file} + sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file} + sed -i -e "s@{{params}}@${params}@g" ${src_file} + sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file} + sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file} + if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then + sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file} + fi + cp "${src_file}" /etc/kubernetes/manifests +} + +# Replaces the variables in the etcd manifest file with the real values, and then +# copy the file to the manifest dir +# $1: value for variable 'suffix' +# $2: value for variable 'port' +# $3: value for variable 'server_port' +# $4: value for variable 'cpulimit' +# $5: pod name, which should be either etcd or etcd-events +function prepare-etcd-manifest { + local host_name=$(hostname) + local etcd_cluster="" + local cluster_state="new" + local etcd_protocol="http" + local etcd_creds="" + + if [[ -n "${ETCD_CA_KEY:-}" && -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then + etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth " + etcd_protocol="https" + fi + + for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do + etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3" + if [[ -n "${etcd_cluster}" ]]; then + etcd_cluster+="," + cluster_state="existing" + fi + etcd_cluster+="${etcd_host}" + done + + local -r temp_file="/tmp/$5" + cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd.manifest" "${temp_file}" + remove-salt-config-comments "${temp_file}" + sed -i -e "s@{{ *suffix *}}@$1@g" "${temp_file}" + sed -i -e "s@{{ *port *}}@$2@g" "${temp_file}" + sed -i -e "s@{{ *server_port *}}@$3@g" "${temp_file}" + sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${temp_file}" + sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}" + sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${temp_file}" + sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" + sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" + if [[ "${STORAGE_BACKEND:-}" == "etcd3" ]]; then + sed -i -e "s@{{ *quota_bytes *}}@--quota-backend-bytes=4294967296@g" "${temp_file}" + else + sed -i -e "s@{{ *quota_bytes *}}@@g" "${temp_file}" + fi + sed -i -e "s@{{ *cluster_state *}}@$cluster_state@g" "${temp_file}" + if [[ -n "${ETCD_IMAGE:-}" ]]; then + sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@${ETCD_IMAGE}@g" "${temp_file}" + else + sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${temp_file}" + fi + sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}" + sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}" + if [[ -n "${ETCD_VERSION:-}" ]]; then + sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${temp_file}" + else + sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@\1@g" "${temp_file}" + fi + # Replace the volume host path. + sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${temp_file}" + mv "${temp_file}" /etc/kubernetes/manifests +} + +function start-etcd-empty-dir-cleanup-pod { + cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd-empty-dir-cleanup/etcd-empty-dir-cleanup.yaml" "/etc/kubernetes/manifests" +} + +# Starts etcd server pod (and etcd-events pod if needed). +# More specifically, it prepares dirs and files, sets the variable value +# in the manifests, and copies them to /etc/kubernetes/manifests. +function start-etcd-servers { + echo "Start etcd pods" + if [[ -d /etc/etcd ]]; then + rm -rf /etc/etcd + fi + if [[ -e /etc/default/etcd ]]; then + rm -f /etc/default/etcd + fi + if [[ -e /etc/systemd/system/etcd.service ]]; then + rm -f /etc/systemd/system/etcd.service + fi + if [[ -e /etc/init.d/etcd ]]; then + rm -f /etc/init.d/etcd + fi + prepare-log-file /var/log/etcd.log + prepare-etcd-manifest "" "2379" "2380" "200m" "etcd.manifest" + + prepare-log-file /var/log/etcd-events.log + prepare-etcd-manifest "-events" "4002" "2381" "100m" "etcd-events.manifest" +} + +# Calculates the following variables based on env variables, which will be used +# by the manifests of several kube-master components. +# CLOUD_CONFIG_OPT +# CLOUD_CONFIG_VOLUME +# CLOUD_CONFIG_MOUNT +# DOCKER_REGISTRY +function compute-master-manifest-variables { + CLOUD_CONFIG_OPT="" + CLOUD_CONFIG_VOLUME="" + CLOUD_CONFIG_MOUNT="" + if [[ -f /etc/gce.conf ]]; then + CLOUD_CONFIG_OPT="--cloud-config=/etc/gce.conf" + CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\"}}," + CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}," + fi + DOCKER_REGISTRY="gcr.io/google_containers" + if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then + DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}" + fi +} + +# A helper function for removing salt configuration and comments from a file. +# This is mainly for preparing a manifest file. +# +# $1: Full path of the file to manipulate +function remove-salt-config-comments { + # Remove salt configuration. + sed -i "/^[ |\t]*{[#|%]/d" $1 + # Remove comments. + sed -i "/^[ |\t]*#/d" $1 +} + +# Starts kubernetes apiserver. +# It prepares the log file, loads the docker image, calculates variables, sets them +# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. +# +# Assumed vars (which are calculated in function compute-master-manifest-variables) +# CLOUD_CONFIG_OPT +# CLOUD_CONFIG_VOLUME +# CLOUD_CONFIG_MOUNT +# DOCKER_REGISTRY +function start-kube-apiserver { + echo "Start kubernetes api-server" + prepare-log-file /var/log/kube-apiserver.log + + # Calculate variables and assemble the command line. + local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" + params+=" --address=127.0.0.1" + params+=" --allow-privileged=true" + params+=" --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl" + params+=" --cloud-provider=gce" + params+=" --client-ca-file=/etc/srv/kubernetes/ca.crt" + params+=" --etcd-servers=http://127.0.0.1:2379" + params+=" --etcd-servers-overrides=/events#http://127.0.0.1:4002" + params+=" --secure-port=443" + params+=" --tls-cert-file=/etc/srv/kubernetes/server.cert" + params+=" --tls-private-key-file=/etc/srv/kubernetes/server.key" + params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv" + if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then + params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv" + fi + if [[ -n "${STORAGE_BACKEND:-}" ]]; then + params+=" --storage-backend=${STORAGE_BACKEND}" + fi + if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then + params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" + fi + if [[ -n "${NUM_NODES:-}" ]]; then + # If the cluster is large, increase max-requests-inflight limit in apiserver. + if [[ "${NUM_NODES}" -ge 1000 ]]; then + params+=" --max-requests-inflight=1500 --max-mutating-requests-inflight=500" + fi + # Set amount of memory available for apiserver based on number of nodes. + # TODO: Once we start setting proper requests and limits for apiserver + # we should reuse the same logic here instead of current heuristic. + params+=" --target-ram-mb=$((${NUM_NODES} * 60))" + fi + if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then + params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" + fi + if [[ -n "${ETCD_QUORUM_READ:-}" ]]; then + params+=" --etcd-quorum-read=${ETCD_QUORUM_READ}" + fi + + local admission_controller_config_mount="" + local admission_controller_config_volume="" + local image_policy_webhook_config_mount="" + local image_policy_webhook_config_volume="" + if [[ -n "${ADMISSION_CONTROL:-}" ]]; then + params+=" --admission-control=${ADMISSION_CONTROL}" + if [[ ${ADMISSION_CONTROL} == *"ImagePolicyWebhook"* ]]; then + params+=" --admission-control-config-file=/etc/admission_controller.config" + # Mount the file to configure admission controllers if ImagePolicyWebhook is set. + admission_controller_config_mount="{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"/etc/admission_controller.config\", \"readOnly\": false}," + admission_controller_config_volume="{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"/etc/admission_controller.config\"}}," + # Mount the file to configure the ImagePolicyWebhook's webhook. + image_policy_webhook_config_mount="{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," + image_policy_webhook_config_volume="{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\"}}," + fi + fi + + if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then + params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" + fi + if [[ -n "${RUNTIME_CONFIG:-}" ]]; then + params+=" --runtime-config=${RUNTIME_CONFIG}" + fi + if [[ -n "${FEATURE_GATES:-}" ]]; then + params+=" --feature-gates=${FEATURE_GATES}" + fi + if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then + local -r vm_external_ip=$(curl --retry 5 --retry-delay 3 --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip") + params+=" --advertise-address=${vm_external_ip}" + params+=" --ssh-user=${PROXY_SSH_USER}" + params+=" --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile" + elif [ -n "${MASTER_ADVERTISE_ADDRESS:-}" ]; then + params="${params} --advertise-address=${MASTER_ADVERTISE_ADDRESS}" + fi + + local webhook_authn_config_mount="" + local webhook_authn_config_volume="" + if [[ -n "${GCP_AUTHN_URL:-}" ]]; then + params+=" --authentication-token-webhook-config-file=/etc/gcp_authn.config" + webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false}," + webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\"}}," + fi + + params+=" --authorization-mode=ABAC" + local webhook_config_mount="" + local webhook_config_volume="" + if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then + params+=",Webhook --authorization-webhook-config-file=/etc/gcp_authz.config" + webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false}," + webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\"}}," + fi + local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" + + if [[ -n "${KUBE_USER:-}" || ! -e /etc/srv/kubernetes/abac-authz-policy.jsonl ]]; then + local -r abac_policy_json="${src_dir}/abac-authz-policy.jsonl" + remove-salt-config-comments "${abac_policy_json}" + if [[ -n "${KUBE_USER:-}" ]]; then + sed -i -e "s/{{kube_user}}/${KUBE_USER}/g" "${abac_policy_json}" + else + sed -i -e "/{{kube_user}}/d" "${abac_policy_json}" + fi + cp "${abac_policy_json}" /etc/srv/kubernetes/ + fi + + src_file="${src_dir}/kube-apiserver.manifest" + remove-salt-config-comments "${src_file}" + # Evaluate variables. + local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag) + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" + sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}" + sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" + sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}" + sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" + sed -i -e "s@{{secure_port}}@443@g" "${src_file}" + sed -i -e "s@{{secure_port}}@8080@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" + sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}" + sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}" + sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}" + sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" + sed -i -e "s@{{admission_controller_config_mount}}@${admission_controller_config_mount}@g" "${src_file}" + sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}" + sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}" + sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests +} + +# Starts kubernetes controller manager. +# It prepares the log file, loads the docker image, calculates variables, sets them +# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. +# +# Assumed vars (which are calculated in function compute-master-manifest-variables) +# CLOUD_CONFIG_OPT +# CLOUD_CONFIG_VOLUME +# CLOUD_CONFIG_MOUNT +# DOCKER_REGISTRY +function start-kube-controller-manager { + echo "Start kubernetes controller-manager" + prepare-log-file /var/log/kube-controller-manager.log + # Calculate variables and assemble the command line. + local params="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=2"} ${CONTROLLER_MANAGER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" + params+=" --cloud-provider=gce" + params+=" --master=127.0.0.1:8080" + params+=" --root-ca-file=/etc/srv/kubernetes/ca.crt" + params+=" --service-account-private-key-file=/etc/srv/kubernetes/server.key" + if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then + params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" + fi + if [[ -n "${INSTANCE_PREFIX:-}" ]]; then + params+=" --cluster-name=${INSTANCE_PREFIX}" + fi + if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then + params+=" --cluster-cidr=${CLUSTER_IP_RANGE}" + fi + if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then + params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" + fi + if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then + params+=" --allocate-node-cidrs=true" + elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then + params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}" + fi + if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then + params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" + fi + if [[ -n "${FEATURE_GATES:-}" ]]; then + params+=" --feature-gates=${FEATURE_GATES}" + fi + local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag) + + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest" + remove-salt-config-comments "${src_file}" + # Evaluate variables. + sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube-controller-manager_docker_tag'\]}}@${kube_rc_docker_tag}@g" "${src_file}" + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" + sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests +} + +# Starts kubernetes scheduler. +# It prepares the log file, loads the docker image, calculates variables, sets them +# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. +# +# Assumed vars (which are calculated in compute-master-manifest-variables) +# DOCKER_REGISTRY +function start-kube-scheduler { + echo "Start kubernetes scheduler" + prepare-log-file /var/log/kube-scheduler.log + + # Calculate variables and set them in the manifest. + params="${SCHEDULER_TEST_LOG_LEVEL:-"--v=2"} ${SCHEDULER_TEST_ARGS:-}" + if [[ -n "${FEATURE_GATES:-}" ]]; then + params+=" --feature-gates=${FEATURE_GATES}" + fi + if [[ -n "${SCHEDULING_ALGORITHM_PROVIDER:-}" ]]; then + params+=" --algorithm-provider=${SCHEDULING_ALGORITHM_PROVIDER}" + fi + local -r kube_scheduler_docker_tag=$(cat "${KUBE_HOME}/kube-docker-files/kube-scheduler.docker_tag") + + # Remove salt comments and replace variables with values. + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-scheduler.manifest" + remove-salt-config-comments "${src_file}" + + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests +} + +# Starts cluster autoscaler. +# Assumed vars (which are calculated in function compute-master-manifest-variables) +# CLOUD_CONFIG_OPT +# CLOUD_CONFIG_VOLUME +# CLOUD_CONFIG_MOUNT +function start-cluster-autoscaler { + if [[ "${ENABLE_CLUSTER_AUTOSCALER:-}" == "true" ]]; then + echo "Start kubernetes cluster autoscaler" + prepare-log-file /var/log/cluster-autoscaler.log + + # Remove salt comments and replace variables with values + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/cluster-autoscaler.manifest" + remove-salt-config-comments "${src_file}" + + local params="${AUTOSCALER_MIG_CONFIG} ${CLOUD_CONFIG_OPT}" + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" + sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" + sed -i -e "s@{%.*%}@@g" "${src_file}" + + cp "${src_file}" /etc/kubernetes/manifests + fi +} + +# A helper function for copying addon manifests and set dir/files +# permissions. +# +# $1: addon category under /etc/kubernetes +# $2: manifest source dir +function setup-addon-manifests { + local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/$2" + local -r dst_dir="/etc/kubernetes/$1/$2" + if [[ ! -d "${dst_dir}" ]]; then + mkdir -p "${dst_dir}" + fi + local files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml") + if [[ -n "${files}" ]]; then + cp "${src_dir}/"*.yaml "${dst_dir}" + fi + files=$(find "${src_dir}" -maxdepth 1 -name "*.json") + if [[ -n "${files}" ]]; then + cp "${src_dir}/"*.json "${dst_dir}" + fi + files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml.in") + if [[ -n "${files}" ]]; then + cp "${src_dir}/"*.yaml.in "${dst_dir}" + fi + chown -R root:root "${dst_dir}" + chmod 755 "${dst_dir}" + chmod 644 "${dst_dir}"/* +} + +# Prepares the manifests of k8s addons, and starts the addon manager. +function start-kube-addons { + echo "Prepare kube-addons manifests and start kube addon manager" + local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" + local -r dst_dir="/etc/kubernetes/addons" + # Set up manifests of other addons. + if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \ + [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \ + [[ "${ENABLE_CLUSTER_MONITORING:-}" == "standalone" ]] || \ + [[ "${ENABLE_CLUSTER_MONITORING:-}" == "googleinfluxdb" ]]; then + local -r file_dir="cluster-monitoring/${ENABLE_CLUSTER_MONITORING}" + setup-addon-manifests "addons" "${file_dir}" + # Replace the salt configurations with variable values. + base_metrics_memory="140Mi" + base_eventer_memory="190Mi" + base_metrics_cpu="80m" + nanny_memory="90Mi" + local -r metrics_memory_per_node="4" + local -r metrics_cpu_per_node="0.5" + local -r eventer_memory_per_node="500" + local -r nanny_memory_per_node="200" + if [[ -n "${NUM_NODES:-}" && "${NUM_NODES}" -ge 1 ]]; then + num_kube_nodes="$((${NUM_NODES}+1))" + nanny_memory="$((${num_kube_nodes} * ${nanny_memory_per_node} + 90 * 1024))Ki" + fi + controller_yaml="${dst_dir}/${file_dir}" + if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "googleinfluxdb" ]]; then + controller_yaml="${controller_yaml}/heapster-controller-combined.yaml" + else + controller_yaml="${controller_yaml}/heapster-controller.yaml" + fi + remove-salt-config-comments "${controller_yaml}" + sed -i -e "s@{{ *base_metrics_memory *}}@${base_metrics_memory}@g" "${controller_yaml}" + sed -i -e "s@{{ *base_metrics_cpu *}}@${base_metrics_cpu}@g" "${controller_yaml}" + sed -i -e "s@{{ *base_eventer_memory *}}@${base_eventer_memory}@g" "${controller_yaml}" + sed -i -e "s@{{ *metrics_memory_per_node *}}@${metrics_memory_per_node}@g" "${controller_yaml}" + sed -i -e "s@{{ *eventer_memory_per_node *}}@${eventer_memory_per_node}@g" "${controller_yaml}" + sed -i -e "s@{{ *nanny_memory *}}@${nanny_memory}@g" "${controller_yaml}" + sed -i -e "s@{{ *metrics_cpu_per_node *}}@${metrics_cpu_per_node}@g" "${controller_yaml}" + fi + if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then + setup-addon-manifests "addons" "dns" + local -r dns_controller_file="${dst_dir}/dns/kubedns-controller.yaml" + local -r dns_svc_file="${dst_dir}/dns/kubedns-svc.yaml" + mv "${dst_dir}/dns/kubedns-controller.yaml.in" "${dns_controller_file}" + mv "${dst_dir}/dns/kubedns-svc.yaml.in" "${dns_svc_file}" + # Replace the salt configurations with variable values. + sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${dns_controller_file}" + sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${dns_svc_file}" + + if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then + setup-addon-manifests "addons" "dns-horizontal-autoscaler" + fi + + if [[ "${FEDERATION:-}" == "true" ]]; then + local federations_domain_map="${FEDERATIONS_DOMAIN_MAP:-}" + if [[ -z "${federations_domain_map}" && -n "${FEDERATION_NAME:-}" && -n "${DNS_ZONE_NAME:-}" ]]; then + federations_domain_map="${FEDERATION_NAME}=${DNS_ZONE_NAME}" + fi + if [[ -n "${federations_domain_map}" ]]; then + sed -i -e "s@{{ *pillar\['federations_domain_map'\] *}}@- --federations=${federations_domain_map}@g" "${dns_controller_file}" + else + sed -i -e "/{{ *pillar\['federations_domain_map'\] *}}/d" "${dns_controller_file}" + fi + else + sed -i -e "/{{ *pillar\['federations_domain_map'\] *}}/d" "${dns_controller_file}" + fi + fi + if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then + setup-addon-manifests "addons" "registry" + local -r registry_pv_file="${dst_dir}/registry/registry-pv.yaml" + local -r registry_pvc_file="${dst_dir}/registry/registry-pvc.yaml" + mv "${dst_dir}/registry/registry-pv.yaml.in" "${registry_pv_file}" + mv "${dst_dir}/registry/registry-pvc.yaml.in" "${registry_pvc_file}" + # Replace the salt configurations with variable values. + remove-salt-config-comments "${controller_yaml}" + sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pv_file}" + sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pvc_file}" + sed -i -e "s@{{ *pillar\['cluster_registry_disk_name'\] *}}@${CLUSTER_REGISTRY_DISK}@g" "${registry_pvc_file}" + fi + # TODO(piosz): figure out how to not run fluentd-es pod from fluentd daemon set on master. + # Running fluentd-es on the master is pointless, as it can't communicate + # with elasticsearch from there in the default configuration. + if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \ + [[ "${LOGGING_DESTINATION:-}" == "elasticsearch" ]] && \ + [[ "${ENABLE_CLUSTER_LOGGING:-}" == "true" ]]; then + setup-addon-manifests "addons" "fluentd-elasticsearch" + fi + if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \ + [[ "${LOGGING_DESTINATION:-}" == "gcp" ]]; then + setup-addon-manifests "addons" "fluentd-gcp" + fi + if [[ "${ENABLE_CLUSTER_UI:-}" == "true" ]]; then + setup-addon-manifests "addons" "dashboard" + fi + if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "true" ]]; then + setup-addon-manifests "addons" "node-problem-detector" + fi + if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then + setup-addon-manifests "admission-controls" "limit-range" + fi + if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then + setup-addon-manifests "addons" "calico-policy-controller" + fi + if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then + setup-addon-manifests "addons" "storage-class/gce" + fi + + # Place addon manager pod manifest. + cp "${src_dir}/kube-addon-manager.yaml" /etc/kubernetes/manifests +} + +# Starts a fluentd static pod for logging for gcp in case master is not registered. +function start-fluentd-static-pod { + echo "Start fluentd pod" + if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \ + [[ "${LOGGING_DESTINATION:-}" == "gcp" ]] && \ + [[ "${KUBERNETES_MASTER:-}" == "true" ]] && \ + [[ "${REGISTER_MASTER_KUBELET:-false}" == "false" ]]; then + cp "${KUBE_HOME}/kube-manifests/kubernetes/fluentd-gcp.yaml" /etc/kubernetes/manifests/ + fi +} + +# Starts an image-puller - used in test clusters. +function start-image-puller { + echo "Start image-puller" + cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/e2e-image-puller.manifest" \ + /etc/kubernetes/manifests/ +} + +# Starts kube-registry proxy +function start-kube-registry-proxy { + echo "Start kube-registry-proxy" + cp "${KUBE_HOME}/kube-manifests/kubernetes/kube-registry-proxy.yaml" /etc/kubernetes/manifests +} + +# Starts a l7 loadbalancing controller for ingress. +function start-lb-controller { + if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then + echo "Start GCE L7 pod" + prepare-log-file /var/log/glbc.log + setup-addon-manifests "addons" "cluster-loadbalancing/glbc" + cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/glbc.manifest" \ + /etc/kubernetes/manifests/ + fi +} + +# Starts rescheduler. +function start-rescheduler { + if [[ "${ENABLE_RESCHEDULER:-}" == "true" ]]; then + echo "Start Rescheduler" + prepare-log-file /var/log/rescheduler.log + cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/rescheduler.manifest" \ + /etc/kubernetes/manifests/ + fi +} + +# Setup working directory for kubelet. +function setup-kubelet-dir { + echo "Making /var/lib/kubelet executable for kubelet" + mount -B /var/lib/kubelet /var/lib/kubelet/ + mount -B -o remount,exec,suid,dev /var/lib/kubelet +} + +function reset-motd { + # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl) + local -r version="$("${KUBE_HOME}"/bin/kubelet --version=true | cut -f2 -d " ")" + # This logic grabs either a release tag (v1.2.1 or v1.2.1-alpha.1), + # or the git hash that's in the build info. + local gitref="$(echo "${version}" | sed -r "s/(v[0-9]+\.[0-9]+\.[0-9]+)(-[a-z]+\.[0-9]+)?.*/\1\2/g")" + local devel="" + if [[ "${gitref}" != "${version}" ]]; then + devel=" +Note: This looks like a development version, which might not be present on GitHub. +If it isn't, the closest tag is at: + https://github.com/kubernetes/kubernetes/tree/${gitref} +" + gitref="${version//*+/}" + fi + cat > /etc/motd < /etc/profile.d/kube_env.sh +} + +function pre-warm-mounter { + echo "prewarming mounter" + ${KUBE_HOME}/bin/mounter &> /dev/null +} + +########### Main Function ########### +echo "Start to configure instance for kubernetes" + +KUBE_HOME="/home/kubernetes" +if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then + echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization." + exit 1 +fi + +source "${KUBE_HOME}/kube-env" + +if [[ -n "${KUBE_USER:-}" ]]; then + if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then + echo "Bad KUBE_USER format." + exit 1 + fi +fi + +setup-os-params +config-ip-firewall +create-dirs +setup-kubelet-dir +ensure-local-ssds +setup-logrotate +if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then + mount-master-pd + create-master-auth + create-master-kubelet-auth + create-master-etcd-auth +else + create-kubelet-kubeconfig + create-kubeproxy-kubeconfig +fi + +override-kubectl +# Run the containerized mounter once to pre-cache the container image. +pre-warm-mounter +assemble-docker-flags +load-docker-images +start-kubelet + +if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then + compute-master-manifest-variables + start-etcd-servers + start-etcd-empty-dir-cleanup-pod + start-kube-apiserver + start-kube-controller-manager + start-kube-scheduler + start-kube-addons + start-cluster-autoscaler + start-lb-controller + start-rescheduler + start-fluentd-static-pod +else + start-kube-proxy + # Kube-registry-proxy. + if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then + start-kube-registry-proxy + fi + if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then + start-image-puller + fi +fi +reset-motd +echo "Done for the configuration for kubernetes" diff --git a/cluster/gce/coreos/configure-node.sh b/cluster/gce/coreos/configure-node.sh deleted file mode 100755 index 6dc47b88458..00000000000 --- a/cluster/gce/coreos/configure-node.sh +++ /dev/null @@ -1,352 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -readonly KNOWN_TOKENS_FILE="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv" -readonly BASIC_AUTH_FILE="/srv/salt-overlay/salt/kube-apiserver/basic_auth.csv" - -# evaluate-manifest evalutes the source manifest with the environment variables. -function evaluate-manifest() { - local src=$1 - local dst=$2 - cp ${src} ${dst} - sed -i 's/\"/\\\"/g' ${dst} # eval will remove the double quotes if they are not escaped - eval "echo \"$(< ${dst})\"" > ${dst} -} - -# evaluate-manifests-dir evalutes the source manifests within $1 and put the result -# in $2. -function evaluate-manifests-dir() { - local src=$1 - local dst=$2 - mkdir -p ${dst} - - for f in ${src}/* - do - evaluate-manifest $f ${dst}/${f##*/} - done -} - -function configure-kube-proxy() { - echo "Configuring kube-proxy" - mkdir -p /var/lib/kube-proxy - evaluate-manifest ${MANIFESTS_DIR}/kubeproxy-config.yaml /var/lib/kube-proxy/kubeconfig -} - -function configure-logging() { - if [[ "${LOGGING_DESTINATION}" == "gcp" ]];then - echo "Configuring fluentd-gcp" - # fluentd-gcp - evaluate-manifest ${MANIFESTS_DIR}/fluentd-gcp.yaml /etc/kubernetes/manifests/fluentd-gcp.yaml - elif [[ "${LOGGING_DESTINATION}" == "elasticsearch" ]];then - echo "Configuring fluentd-es" - # fluentd-es - evaluate-manifest ${MANIFESTS_DIR}/fluentd-es.yaml /etc/kubernetes/manifests/fluentd-es.yaml - fi -} - -function configure-admission-controls() { - echo "Configuring admission controls" - mkdir -p /etc/kubernetes/admission-controls - cp -r ${SALT_DIR}/salt/kube-admission-controls/limit-range /etc/kubernetes/admission-controls/ -} - -function configure-etcd() { - echo "Configuring etcd" - touch /var/log/etcd.log - evaluate-manifest ${MANIFESTS_DIR}/etcd.yaml /etc/kubernetes/manifests/etcd.yaml -} - -function configure-etcd-events() { - echo "Configuring etcd-events" - touch /var/log/etcd-events.log - evaluate-manifest ${MANIFESTS_DIR}/etcd-events.yaml /etc/kubernetes/manifests/etcd-events.yaml -} - -function configure-addon-manager() { - echo "Configuring addon-manager" - evaluate-manifest ${MANIFESTS_DIR}/kube-addon-manager.yaml /etc/kubernetes/manifests/kube-addon-manager.yaml -} - -function configure-kube-apiserver() { - echo "Configuring kube-apiserver" - - # Wait for etcd to be up. - wait-url-up http://127.0.0.1:2379/version - - touch /var/log/kube-apiserver.log - - # Copying known_tokens and basic_auth file. - cp ${SALT_OVERLAY}/salt/kube-apiserver/*.csv /srv/kubernetes/ - evaluate-manifest ${MANIFESTS_DIR}/kube-apiserver.yaml /etc/kubernetes/manifests/kube-apiserver.yaml -} - -function configure-kube-scheduler() { - echo "Configuring kube-scheduler" - touch /var/log/kube-scheduler.log - evaluate-manifest ${MANIFESTS_DIR}/kube-scheduler.yaml /etc/kubernetes/manifests/kube-scheduler.yaml -} - -function configure-kube-controller-manager() { - # Wait for api server. - wait-url-up http://127.0.0.1:8080/version - echo "Configuring kube-controller-manager" - touch /var/log/kube-controller-manager.log - evaluate-manifest ${MANIFESTS_DIR}/kube-controller-manager.yaml /etc/kubernetes/manifests/kube-controller-manager.yaml -} - -# Wait until $1 become reachable. -function wait-url-up() { - until curl --silent $1 - do - sleep 5 - done -} - -# Configure addon yamls, and run salt/kube-addons/kube-addons.sh -function configure-master-addons() { - echo "Configuring master addons" - - local addon_dir=/etc/kubernetes/addons - mkdir -p ${addon_dir} - - # Copy namespace.yaml - evaluate-manifest ${MANIFESTS_DIR}/addons/namespace.yaml ${addon_dir}/namespace.yaml - - if [[ "${ENABLE_L7_LOADBALANCING}" == "glbc" ]]; then - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/cluster-loadbalancing/glbc ${addon_dir}/cluster-loadbalancing/glbc - fi - - if [[ "${ENABLE_CLUSTER_DNS}" == "true" ]]; then - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/dns ${addon_dir}/dns - if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER}" == "true" ]]; then - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/dns-horizontal-autoscaler ${addon_dir}/dns-horizontal-autoscaler - fi - fi - - if [[ "${ENABLE_CLUSTER_UI}" == "true" ]]; then - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/dashboard ${addon_dir}/dashboard - fi - - if [[ "${ENABLE_CLUSTER_LOGGING}" == "true" ]]; then - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/fluentd-elasticsearch ${addon_dir}/fluentd-elasticsearch - fi - - if [[ "${ENABLE_CLUSTER_MONITORING}" == "influxdb" ]]; then - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/cluster-monitoring/influxdb ${addon_dir}/cluster-monitoring/influxdb - elif [[ "${ENABLE_CLUSTER_MONITORING}" == "google" ]]; then - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/cluster-monitoring/google ${addon_dir}/cluster-monitoring/google - elif [[ "${ENABLE_CLUSTER_MONITORING}" == "standalone" ]]; then - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/cluster-monitoring/standalone ${addon_dir}/cluster-monitoring/standalone - elif [[ "${ENABLE_CLUSTER_MONITORING}" == "googleinfluxdb" ]]; then - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/cluster-monitoring/googleinfluxdb ${addon_dir}/cluster-monitoring/googleinfluxdb - fi - - # Note that, KUBE_ENABLE_INSECURE_REGISTRY is not supported yet. - if [[ "${ENABLE_CLUSTER_REGISTRY}" == "true" ]]; then - CLUSTER_REGISTRY_DISK_SIZE=$(convert-bytes-gce-kube "${CLUSTER_REGISTRY_DISK_SIZE}") - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/registry ${addon_dir}/registry - fi - - if [[ "${ENABLE_NODE_PROBLEM_DETECTOR}" == "true" ]]; then - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/node-problem-detector ${addon_dir}/node-problem-detector - fi - if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then - setup-addon-manifests "addons" "storage-class/gce" - fi - - evaluate-manifests-dir ${MANIFESTS_DIR}/addons/storage-class/gce ${addon_dir}/storage-class -} - -function configure-master-components() { - configure-admission-controls - configure-etcd - configure-etcd-events - configure-kube-apiserver - configure-kube-scheduler - configure-kube-controller-manager - configure-master-addons - configure-addon-manager -} - -# TODO(yifan): Merge this with mount-master-pd() in configure-vm.sh -# Pass ${save_format_and_mount} as an argument. -function mount-master-pd() { - if [[ ! -e /dev/disk/by-id/google-master-pd ]]; then - return - fi - device_info=$(ls -l /dev/disk/by-id/google-master-pd) - relative_path=${device_info##* } - device_path="/dev/disk/by-id/${relative_path}" - - # Format and mount the disk, create directories on it for all of the master's - # persistent data, and link them to where they're used. - echo "Mounting master-pd" - mkdir -p /mnt/master-pd - safe_format_and_mount=${SALT_DIR}/salt/helpers/safe_format_and_mount - chmod +x ${safe_format_and_mount} - ${safe_format_and_mount} -m "mkfs.ext4 -F" "${device_path}" /mnt/master-pd &>/var/log/master-pd-mount.log || \ - { echo "!!! master-pd mount failed, review /var/log/master-pd-mount.log !!!"; return 1; } - # Contains all the data stored in etcd - mkdir -m 700 -p /mnt/master-pd/var/etcd - # Contains the dynamically generated apiserver auth certs and keys - mkdir -p /mnt/master-pd/srv/kubernetes - # Contains the cluster's initial config parameters and auth tokens - mkdir -p /mnt/master-pd/srv/salt-overlay - # Directory for kube-apiserver to store SSH key (if necessary) - mkdir -p /mnt/master-pd/srv/sshproxy - - ln -s -f /mnt/master-pd/var/etcd /var/etcd - ln -s -f /mnt/master-pd/srv/kubernetes /srv/kubernetes - ln -s -f /mnt/master-pd/srv/sshproxy /srv/sshproxy - ln -s -f /mnt/master-pd/srv/salt-overlay /srv/salt-overlay - - # This is a bit of a hack to get around the fact that salt has to run after the - # PD and mounted directory are already set up. We can't give ownership of the - # directory to etcd until the etcd user and group exist, but they don't exist - # until salt runs if we don't create them here. We could alternatively make the - # permissions on the directory more permissive, but this seems less bad. - if ! id etcd &>/dev/null; then - useradd -s /sbin/nologin -d /var/etcd etcd - fi - chown -R etcd /mnt/master-pd/var/etcd - chgrp -R etcd /mnt/master-pd/var/etcd -} - -# The job of this function is simple, but the basic regular expression syntax makes -# this difficult to read. What we want to do is convert from [0-9]+B, KB, KiB, MB, etc -# into [0-9]+, Ki, Mi, Gi, etc. -# This is done in two steps: -# 1. Convert from [0-9]+X?i?B into [0-9]X? (X denotes the prefix, ? means the field -# is optional. -# 2. Attach an 'i' to the end of the string if we find a letter. -# The two step process is needed to handle the edge case in which we want to convert -# a raw byte count, as the result should be a simple number (e.g. 5B -> 5). -# -# TODO(yifan): Reuse the one defined in configure-vm.sh to remove duplication. -function convert-bytes-gce-kube() { - local -r storage_space=$1 - echo "${storage_space}" | sed -e 's/^\([0-9]\+\)\([A-Z]\)\?i\?B$/\1\2/g' -e 's/\([A-Z]\)$/\1i/' -} - -# TODO(yifan): Use create-salt-master-auth() in configure-vm.sh -function create-salt-master-auth() { - if [[ ! -e /srv/kubernetes/ca.crt ]]; then - if [[ ! -z "${CA_CERT:-}" ]] && [[ ! -z "${MASTER_CERT:-}" ]] && [[ ! -z "${MASTER_KEY:-}" ]]; then - mkdir -p /srv/kubernetes - (umask 077; - echo "${CA_CERT}" | base64 --decode > /srv/kubernetes/ca.crt; - echo "${MASTER_CERT}" | base64 --decode > /srv/kubernetes/server.cert; - echo "${MASTER_KEY}" | base64 --decode > /srv/kubernetes/server.key; - # Kubecfg cert/key are optional and included for backwards compatibility. - # TODO(roberthbailey): Remove these two lines once GKE no longer requires - # fetching clients certs from the master VM. - echo "${KUBECFG_CERT:-}" | base64 --decode > /srv/kubernetes/kubecfg.crt; - echo "${KUBECFG_KEY:-}" | base64 --decode > /srv/kubernetes/kubecfg.key) - fi - fi - if [ ! -e "${BASIC_AUTH_FILE}" ]; then - mkdir -p /srv/salt-overlay/salt/kube-apiserver - (umask 077; - echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${BASIC_AUTH_FILE}") - fi - if [ ! -e "${KNOWN_TOKENS_FILE}" ]; then - mkdir -p /srv/salt-overlay/salt/kube-apiserver - (umask 077; - echo "${KUBE_BEARER_TOKEN},admin,admin" > "${KNOWN_TOKENS_FILE}"; - echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${KNOWN_TOKENS_FILE}"; - echo "${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy" >> "${KNOWN_TOKENS_FILE}") - - # Generate tokens for other "service accounts". Append to known_tokens. - # - # NB: If this list ever changes, this script actually has to - # change to detect the existence of this file, kill any deleted - # old tokens and add any new tokens (to handle the upgrade case). - local -r service_accounts=("system:scheduler" "system:controller_manager" "system:logging" "system:monitoring") - for account in "${service_accounts[@]}"; do - token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - echo "${token},${account},${account}" >> "${KNOWN_TOKENS_FILE}" - done - fi -} - -# $1 is the directory containing all of the docker images -function load-docker-images() { - local success - local restart_docker - while true; do - success=true - restart_docker=false - for image in "$1/"*; do - timeout 30 docker load -i "${image}" &>/dev/null - rc=$? - if [[ "$rc" == 124 ]]; then - restart_docker=true - elif [[ "$rc" != 0 ]]; then - success=false - fi - done - if [[ "$success" == "true" ]]; then break; fi - if [[ "$restart_docker" == "true" ]]; then systemctl restart docker; fi - sleep 15 - done -} - -function load-master-components-images() { - echo "Loading images for master components" - export RKT_BIN=/opt/rkt/rkt - export DOCKER2ACI_BIN=/opt/docker2aci/docker2aci - ${SALT_DIR}/install.sh ${KUBE_BIN_TAR} - ${SALT_DIR}/salt/kube-master-addons/kube-master-addons.sh - - # Get the image tags. - KUBE_APISERVER_DOCKER_TAG=$(cat ${KUBE_BIN_DIR}/kube-apiserver.docker_tag) - KUBE_CONTROLLER_MANAGER_DOCKER_TAG=$(cat ${KUBE_BIN_DIR}/kube-controller-manager.docker_tag) - KUBE_SCHEDULER_DOCKER_TAG=$(cat ${KUBE_BIN_DIR}/kube-scheduler.docker_tag) -} - -########## -# main # -########## - -KUBE_BIN_TAR=/opt/downloads/kubernetes-server-linux-amd64.tar.gz -KUBE_BIN_DIR=/opt/kubernetes/server/bin -SALT_DIR=/opt/kubernetes/saltbase -SALT_OVERLAY=/srv/salt-overlay -MANIFESTS_DIR=/opt/kube-manifests/kubernetes - -# On CoreOS, the hosts is in /usr/share/baselayout/hosts -# So we need to manually populdate the hosts file here on gce. -echo "127.0.0.1 localhost" >> /etc/hosts -echo "::1 localhost" >> /etc/hosts - -if [[ "${KUBERNETES_MASTER}" == "true" ]]; then - mount-master-pd - create-salt-master-auth - load-master-components-images - configure-master-components -else - configure-kube-proxy -fi - -if [[ "${ENABLE_NODE_LOGGING}" == "true" ]];then - configure-logging -fi - -echo "Finish configuration successfully!" diff --git a/cluster/gce/coreos/configure.sh b/cluster/gce/coreos/configure.sh new file mode 100644 index 00000000000..11fe6f08d67 --- /dev/null +++ b/cluster/gce/coreos/configure.sh @@ -0,0 +1,213 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Due to the GCE custom metadata size limit, we split the entire script into two +# files configure.sh and configure-helper.sh. The functionality of downloading +# kubernetes configuration, manifests, docker images, and binary files are +# put in configure.sh, which is uploaded via GCE custom metadata. + +set -o errexit +set -o nounset +set -o pipefail + +function set-broken-motd { + cat > /etc/motd < "${KUBE_HOME}/kube-env") + rm -f "${tmp_kube_env}" +} + +function validate-hash { + local -r file="$1" + local -r expected="$2" + + actual=$(sha1sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +# Retry a download until we get it. Takes a hash and a set of URLs. +# +# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. +# $2+ are the URLs to download. +function download-or-bust { + local -r hash="$1" + shift 1 + + local -r urls=( $* ) + while true; do + for url in "${urls[@]}"; do + local file="${url##*/}" + rm -f "${file}" + if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${url}"; then + echo "== Failed to download ${url}. Retrying. ==" + elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done +} + +function split-commas { + echo $1 | tr "," "\n" +} + +function install-gci-mounter-tools { + local -r rkt_version="v1.18.0" + local -r gci_mounter_version="v2" + local -r rkt_binary_sha1="75fc8f29c79bc9e505f3e7f6e8fadf2425c21967" + local -r rkt_stage1_fly_sha1="474df5a1f934960ba669b360ab713d0a54283091" + local -r gci_mounter_sha1="851e841d8640d6a05e64e22c493f5ac3c4cba561" + download-or-bust "${rkt_binary_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/rkt" + download-or-bust "${rkt_stage1_fly_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/stage1-fly.aci" + download-or-bust "${gci_mounter_sha1}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-${gci_mounter_version}.aci" + local -r rkt_dst="${KUBE_HOME}/bin/" + mv "${KUBE_HOME}/rkt" "${rkt_dst}/rkt" + mv "${KUBE_HOME}/stage1-fly.aci" "${rkt_dst}/stage1-fly.aci" + mv "${KUBE_HOME}/gci-mounter-${gci_mounter_version}.aci" "${rkt_dst}/gci-mounter-${gci_mounter_version}.aci" + chmod a+x "${rkt_dst}/rkt" +} + +# Downloads kubernetes binaries and kube-system manifest tarball, unpacks them, +# and places them into suitable directories. Files are placed in /home/kubernetes. +function install-kube-binary-config { + cd "${KUBE_HOME}" + local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") ) + local -r server_binary_tar="${server_binary_tar_urls[0]##*/}" + if [[ -n "${SERVER_BINARY_TAR_HASH:-}" ]]; then + local -r server_binary_tar_hash="${SERVER_BINARY_TAR_HASH}" + else + echo "Downloading binary release sha1 (not found in env)" + download-or-bust "" "${server_binary_tar_urls[@]/.tar.gz/.tar.gz.sha1}" + local -r server_binary_tar_hash=$(cat "${server_binary_tar}.sha1") + fi + echo "Downloading binary release tar" + download-or-bust "${server_binary_tar_hash}" "${server_binary_tar_urls[@]}" + tar xzf "${KUBE_HOME}/${server_binary_tar}" -C "${KUBE_HOME}" --overwrite + # Copy docker_tag and image files to ${KUBE_HOME}/kube-docker-files. + src_dir="${KUBE_HOME}/kubernetes/server/bin" + dst_dir="${KUBE_HOME}/kube-docker-files" + mkdir -p "${dst_dir}" + cp "${src_dir}/"*.docker_tag "${dst_dir}" + if [[ "${KUBERNETES_MASTER:-}" == "false" ]]; then + cp "${src_dir}/kube-proxy.tar" "${dst_dir}" + else + cp "${src_dir}/kube-apiserver.tar" "${dst_dir}" + cp "${src_dir}/kube-controller-manager.tar" "${dst_dir}" + cp "${src_dir}/kube-scheduler.tar" "${dst_dir}" + cp -r "${KUBE_HOME}/kubernetes/addons" "${dst_dir}" + fi + local -r kube_bin="${KUBE_HOME}/bin" + mv "${src_dir}/kubelet" "${kube_bin}" + mv "${src_dir}/kubectl" "${kube_bin}" + + if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]] || \ + [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then + #TODO(andyzheng0831): We should make the cni version number as a k8s env variable. + local -r cni_tar="cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz" + local -r cni_sha1="19d49f7b2b99cd2493d5ae0ace896c64e289ccbb" + download-or-bust "${cni_sha1}" "https://storage.googleapis.com/kubernetes-release/network-plugins/${cni_tar}" + local -r cni_dir="${KUBE_HOME}/cni" + mkdir -p "${cni_dir}" + tar xzf "${KUBE_HOME}/${cni_tar}" -C "${cni_dir}" --overwrite + mv "${cni_dir}/bin"/* "${kube_bin}" + rmdir "${cni_dir}/bin" + rm -f "${KUBE_HOME}/${cni_tar}" + fi + + mv "${KUBE_HOME}/kubernetes/LICENSES" "${KUBE_HOME}" + mv "${KUBE_HOME}/kubernetes/kubernetes-src.tar.gz" "${KUBE_HOME}" + + # Put kube-system pods manifests in ${KUBE_HOME}/kube-manifests/. + dst_dir="${KUBE_HOME}/kube-manifests" + mkdir -p "${dst_dir}" + local -r manifests_tar_urls=( $(split-commas "${KUBE_MANIFESTS_TAR_URL}") ) + local -r manifests_tar="${manifests_tar_urls[0]##*/}" + if [ -n "${KUBE_MANIFESTS_TAR_HASH:-}" ]; then + local -r manifests_tar_hash="${KUBE_MANIFESTS_TAR_HASH}" + else + echo "Downloading k8s manifests sha1 (not found in env)" + download-or-bust "" "${manifests_tar_urls[@]/.tar.gz/.tar.gz.sha1}" + local -r manifests_tar_hash=$(cat "${manifests_tar}.sha1") + fi + echo "Downloading k8s manifests tar" + download-or-bust "${manifests_tar_hash}" "${manifests_tar_urls[@]}" + tar xzf "${KUBE_HOME}/${manifests_tar}" -C "${dst_dir}" --overwrite + local -r kube_addon_registry="${KUBE_ADDON_REGISTRY:-gcr.io/google_containers}" + if [[ "${kube_addon_registry}" != "gcr.io/google_containers" ]]; then + find "${dst_dir}" -name \*.yaml -or -name \*.yaml.in | \ + xargs sed -ri "s@(image:\s.*)gcr.io/google_containers@\1${kube_addon_registry}@" + find "${dst_dir}" -name \*.manifest -or -name \*.json | \ + xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@" + fi + cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh" + cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter" + cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh" + chmod -R 755 "${kube_bin}" + + # Install gci mounter related artifacts to allow mounting storage volumes in GCI + install-gci-mounter-tools + + # Clean up. + rm -rf "${KUBE_HOME}/kubernetes" + rm -f "${KUBE_HOME}/${server_binary_tar}" + rm -f "${KUBE_HOME}/${server_binary_tar}.sha1" + rm -f "${KUBE_HOME}/${manifests_tar}" + rm -f "${KUBE_HOME}/${manifests_tar}.sha1" +} + +######### Main Function ########## +echo "Start to install kubernetes files" +set-broken-motd +KUBE_HOME="/home/kubernetes" +download-kube-env +source "${KUBE_HOME}/kube-env" +install-kube-binary-config +echo "Done for installing kubernetes files" + diff --git a/cluster/gce/coreos/health-monitor.sh b/cluster/gce/coreos/health-monitor.sh new file mode 100644 index 00000000000..4d50e4ee8dd --- /dev/null +++ b/cluster/gce/coreos/health-monitor.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is for master and node instance health monitoring, which is +# packed in kube-manifest tarball. It is executed through a systemd service +# in cluster/gce/gci/.yaml. The env variables come from an env +# file provided by the systemd service. + +set -o nounset +set -o pipefail + +# We simply kill the process when there is a failure. Another systemd service will +# automatically restart the process. +function docker_monitoring { + while [ 1 ]; do + if ! timeout 10 docker ps > /dev/null; then + echo "Docker daemon failed!" + pkill docker + # Wait for a while, as we don't want to kill it again before it is really up. + sleep 30 + else + sleep "${SLEEP_SECONDS}" + fi + done +} + +function kubelet_monitoring { + echo "Wait for 2 minutes for kubelet to be fuctional" + # TODO(andyzheng0831): replace it with a more reliable method if possible. + sleep 120 + local -r max_seconds=10 + while [ 1 ]; do + if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then + echo "Kubelet is unhealthy!" + curl --insecure https://127.0.0.1:${KUBELET_PORT:-10250}/healthz + pkill kubelet + # Wait for a while, as we don't want to kill it again before it is really up. + sleep 60 + else + sleep "${SLEEP_SECONDS}" + fi + done +} + + +############## Main Function ################ +if [[ "$#" -ne 1 ]]; then + echo "Usage: health-monitor.sh " + exit 1 +fi + +KUBE_ENV="/home/kubernetes/kube-env" +if [[ ! -e "${KUBE_ENV}" ]]; then + echo "The ${KUBE_ENV} file does not exist!! Terminate health monitoring" + exit 1 +fi + +SLEEP_SECONDS=10 +component=$1 +echo "Start kubernetes health monitoring for ${component}" +source "${KUBE_ENV}" +if [[ "${component}" == "docker" ]]; then + docker_monitoring +elif [[ "${component}" == "kubelet" ]]; then + kubelet_monitoring +else + echo "Health monitoring for component "${component}" is not supported!" +fi diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh new file mode 100755 index 00000000000..e0486c80c7b --- /dev/null +++ b/cluster/gce/coreos/helper.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions and constant for GCI distro + +# Creates the GCI specific metadata files if they do not exit. +# Assumed var +# KUBE_TEMP +function ensure-gci-metadata-files { + if [[ ! -f "${KUBE_TEMP}/gci-update.txt" ]]; then + echo -n "update_disabled" > "${KUBE_TEMP}/gci-update.txt" + fi + if [[ ! -f "${KUBE_TEMP}/gci-ensure-gke-docker.txt" ]]; then + echo -n "true" > "${KUBE_TEMP}/gci-ensure-gke-docker.txt" + fi + if [[ ! -f "${KUBE_TEMP}/gci-docker-version.txt" ]]; then + echo -n "${GCI_DOCKER_VERSION:-}" > "${KUBE_TEMP}/gci-docker-version.txt" + fi +} diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-loadbalancing/glbc/default-svc.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-loadbalancing/glbc/default-svc.yaml deleted file mode 100644 index cd07ec50eb4..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-loadbalancing/glbc/default-svc.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - # This must match the --default-backend-service argument of the l7 lb - # controller and is required because GCE mandates a default backend. - name: default-http-backend - namespace: kube-system - labels: - k8s-app: glbc - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "GLBCDefaultBackend" -spec: - # The default backend must be of type NodePort. - type: NodePort - ports: - - port: 80 - targetPort: 8080 - protocol: TCP - name: http - selector: - k8s-app: glbc \ No newline at end of file diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-loadbalancing/glbc/glbc-controller.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-loadbalancing/glbc/glbc-controller.yaml deleted file mode 100644 index a60adc1628f..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-loadbalancing/glbc/glbc-controller.yaml +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: l7-lb-controller-v0.6.0 - namespace: kube-system - labels: - k8s-app: glbc - version: v0.6.0 - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "GLBC" -spec: - # There should never be more than 1 controller alive simultaneously. - replicas: 1 - selector: - k8s-app: glbc - version: v0.6.0 - template: - metadata: - labels: - k8s-app: glbc - version: v0.6.0 - name: glbc - kubernetes.io/cluster-service: "true" - spec: - terminationGracePeriodSeconds: 600 - containers: - - name: default-http-backend - # Any image is permissible as long as: - # 1. It serves a 404 page at / - # 2. It serves 200 on a /healthz endpoint - image: gcr.io/google_containers/defaultbackend:1.0 - livenessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - ports: - - containerPort: 8080 - resources: - limits: - cpu: 10m - memory: 20Mi - requests: - cpu: 10m - memory: 20Mi - - image: gcr.io/google_containers/glbc:0.6.0 - livenessProbe: - httpGet: - path: /healthz - port: 8081 - scheme: HTTP - initialDelaySeconds: 30 - # healthz reaches out to GCE - periodSeconds: 30 - timeoutSeconds: 5 - name: l7-lb-controller - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 50Mi - args: - - --default-backend-service=kube-system/default-http-backend - - --sync-period=60s - - --cluster-uid=${KUBE_UID} diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/google/heapster-controller.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/google/heapster-controller.yaml deleted file mode 100644 index a52d46b8110..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/google/heapster-controller.yaml +++ /dev/null @@ -1,127 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: heapster-v1.2.0 - namespace: kube-system - labels: - k8s-app: heapster - kubernetes.io/cluster-service: "true" - version: v1.2.0 -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: heapster - version: v1.2.0 - template: - metadata: - labels: - k8s-app: heapster - version: v1.2.0 - spec: - containers: - - image: gcr.io/google_containers/heapster:v1.2.0 - name: heapster - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 100m - memory: 300Mi - command: - - /heapster - - --source=kubernetes.summary_api:'' - - --sink=gcm - volumeMounts: - - name: ssl-certs - mountPath: /etc/ssl/certs - readOnly: true - - name: usrsharecacerts - mountPath: /usr/share/ca-certificates - readOnly: true - - image: gcr.io/google_containers/heapster:v1.2.0 - name: eventer - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 100m - memory: 300Mi - command: - - /eventer - - --source=kubernetes:'' - - --sink=gcl - volumeMounts: - - name: ssl-certs - mountPath: /etc/ssl/certs - readOnly: true - - name: usrsharecacerts - mountPath: /usr/share/ca-certificates - readOnly: true - - image: gcr.io/google_containers/addon-resizer:1.0 - name: heapster-nanny - resources: - limits: - cpu: 50m - memory: 100Mi - requests: - cpu: 50m - memory: 100Mi - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - /pod_nanny - - --cpu=100m - - --extra-cpu=0m - - --memory=300Mi - - --extra-memory=4Mi - - --threshold=5 - - --deployment=heapster-v1.2.0 - - --container=heapster - - --poll-period=300000 - - image: gcr.io/google_containers/addon-resizer:1.0 - name: eventer-nanny - resources: - limits: - cpu: 50m - memory: 100Mi - requests: - cpu: 50m - memory: 100Mi - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - /pod_nanny - - --cpu=100m - - --extra-cpu=0m - - --memory=300Mi - - --extra-memory=307200Ki - - --threshold=5 - - --deployment=heapster-v1.2.0 - - --container=eventer - - --poll-period=300000 - volumes: - - name: ssl-certs - hostPath: - path: "/etc/ssl/certs" - - name: usrsharecacerts - hostPath: - path: "/usr/share/ca-certificates" diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/google/heapster-service.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/google/heapster-service.yaml deleted file mode 100644 index 31e8b96006d..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/google/heapster-service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: heapster - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "Heapster" -spec: - ports: - - port: 80 - targetPort: 8082 - selector: - k8s-app: heapster diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml deleted file mode 100644 index 714f634963b..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml +++ /dev/null @@ -1,128 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: heapster-v1.1.0 - namespace: kube-system - labels: - k8s-app: heapster - kubernetes.io/cluster-service: "true" - version: v1.1.0 -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: heapster - version: v1.1.0 - template: - metadata: - labels: - k8s-app: heapster - version: v1.1.0 - spec: - containers: - - image: gcr.io/google_containers/heapster:v1.2.0 - name: heapster - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 100m - memory: 300Mi - command: - - /heapster - - --source=kubernetes.summary_api:'' - - --sink=influxdb:http://monitoring-influxdb:8086 - - --sink=gcm:?metrics=autoscaling - volumeMounts: - - name: ssl-certs - mountPath: /etc/ssl/certs - readOnly: true - - name: usrsharecacerts - mountPath: /usr/share/ca-certificates - readOnly: true - - image: gcr.io/google_containers/heapster:v1.2.0 - name: eventer - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 100m - memory: 300Mi - command: - - /eventer - - --source=kubernetes:'' - - --sink=gcl - volumeMounts: - - name: ssl-certs - mountPath: /etc/ssl/certs - readOnly: true - - name: usrsharecacerts - mountPath: /usr/share/ca-certificates - readOnly: true - - image: gcr.io/google_containers/addon-resizer:1.0 - name: heapster-nanny - resources: - limits: - cpu: 50m - memory: 100Mi - requests: - cpu: 50m - memory: 100Mi - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - /pod_nanny - - --cpu=100m - - --extra-cpu=0m - - --memory=300Mi - - --extra-memory=4Mi - - --threshold=5 - - --deployment=heapster-v1.1.0 - - --container=heapster - - --poll-period=300000 - - image: gcr.io/google_containers/addon-resizer:1.0 - name: eventer-nanny - resources: - limits: - cpu: 50m - memory: 100Mi - requests: - cpu: 50m - memory: 100Mi - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - /pod_nanny - - --cpu=100m - - --extra-cpu=0m - - --memory=300Mi - - --extra-memory=307200Ki - - --threshold=5 - - --deployment=heapster-v1.1.0 - - --container=eventer - - --poll-period=300000 - volumes: - - name: ssl-certs - hostPath: - path: "/etc/ssl/certs" - - name: usrsharecacerts - hostPath: - path: "/usr/share/ca-certificates" diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/grafana-service.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/grafana-service.yaml deleted file mode 100644 index 9140e8b0c2c..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/grafana-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: monitoring-grafana - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "Grafana" -spec: - # On production clusters, consider setting up auth for grafana, and - # exposing Grafana either using a LoadBalancer or a public IP. - # type: LoadBalancer - ports: - - port: 80 - targetPort: 3000 - selector: - k8s-app: influxGrafana - diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/heapster-controller.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/heapster-controller.yaml deleted file mode 100644 index e3485c4a152..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/heapster-controller.yaml +++ /dev/null @@ -1,106 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: heapster-v1.2.0 - namespace: kube-system - labels: - k8s-app: heapster - kubernetes.io/cluster-service: "true" - version: v1.2.0 -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: heapster - version: v1.2.0 - template: - metadata: - labels: - k8s-app: heapster - version: v1.2.0 - spec: - containers: - - image: gcr.io/google_containers/heapster:v1.2.0 - name: heapster - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 100m - memory: 300Mi - command: - - /heapster - - --source=kubernetes.summary_api:'' - - --sink=influxdb:http://monitoring-influxdb:8086 - - image: gcr.io/google_containers/heapster:v1.2.0 - name: eventer - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 100m - memory: 300Mi - command: - - /eventer - - --source=kubernetes:'' - - --sink=influxdb:http://monitoring-influxdb:8086 - - image: gcr.io/google_containers/addon-resizer:1.0 - name: heapster-nanny - resources: - limits: - cpu: 50m - memory: 100Mi - requests: - cpu: 50m - memory: 100Mi - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - /pod_nanny - - --cpu=100m - - --extra-cpu=0m - - --memory=300Mi - - --extra-memory=4Mi - - --threshold=5 - - --deployment=heapster-v1.2.0 - - --container=heapster - - --poll-period=300000 - - image: gcr.io/google_containers/addon-resizer:1.0 - name: eventer-nanny - resources: - limits: - cpu: 50m - memory: 100Mi - requests: - cpu: 50m - memory: 100Mi - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - /pod_nanny - - --cpu=100m - - --extra-cpu=0m - - --memory=300Mi - - --extra-memory=307200Ki - - --threshold=5 - - --deployment=heapster-v1.2.0 - - --container=eventer - - --poll-period=300000 diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/heapster-service.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/heapster-service.yaml deleted file mode 100644 index e406d69c44c..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/heapster-service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: heapster - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "Heapster" -spec: - ports: - - port: 80 - targetPort: 8082 - selector: - k8s-app: heapster diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml deleted file mode 100644 index 16cb5d8e8eb..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: monitoring-influxdb-grafana-v3 - namespace: kube-system - labels: - k8s-app: influxGrafana - version: v3 - kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: influxGrafana - version: v3 - template: - metadata: - labels: - k8s-app: influxGrafana - version: v3 - kubernetes.io/cluster-service: "true" - spec: - containers: - - image: gcr.io/google_containers/heapster_influxdb:v0.7 - name: influxdb - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 500Mi - requests: - cpu: 100m - memory: 500Mi - ports: - - containerPort: 8083 - - containerPort: 8086 - volumeMounts: - - name: influxdb-persistent-storage - mountPath: /data - - image: gcr.io/google_containers/heapster_grafana:v3.1.1 - name: grafana - env: - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - env: - # This variable is required to setup templates in Grafana. - - name: INFLUXDB_SERVICE_URL - value: http://monitoring-influxdb:8086 - # The following env variables are required to make Grafana accessible via - # the kubernetes api-server proxy. On production clusters, we recommend - # removing these env variables, setup auth for grafana, and expose the grafana - # service using a LoadBalancer or a public IP. - - name: GF_AUTH_BASIC_ENABLED - value: "false" - - name: GF_AUTH_ANONYMOUS_ENABLED - value: "true" - - name: GF_AUTH_ANONYMOUS_ORG_ROLE - value: Admin - - name: GF_SERVER_ROOT_URL - value: /api/v1/proxy/namespaces/kube-system/services/monitoring-grafana/ - volumeMounts: - - name: grafana-persistent-storage - mountPath: /var - volumes: - - name: influxdb-persistent-storage - emptyDir: {} - - name: grafana-persistent-storage - emptyDir: {} - diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/influxdb-service.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/influxdb-service.yaml deleted file mode 100644 index 066e052476e..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/influxdb-service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: monitoring-influxdb - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "InfluxDB" -spec: - ports: - - name: http - port: 8083 - targetPort: 8083 - - name: api - port: 8086 - targetPort: 8086 - selector: - k8s-app: influxGrafana - diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/standalone/heapster-controller.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/standalone/heapster-controller.yaml deleted file mode 100644 index fdf422d66cd..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/standalone/heapster-controller.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: heapster-v1.2.0 - namespace: kube-system - labels: - k8s-app: heapster - kubernetes.io/cluster-service: "true" - version: v1.2.0 -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: heapster - version: v1.2.0 - template: - metadata: - labels: - k8s-app: heapster - version: v1.2.0 - spec: - containers: - - image: gcr.io/google_containers/heapster:v1.2.0 - name: heapster - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 100m - memory: 300Mi - command: - - /heapster - - --source=kubernetes.summary_api:'' - - image: gcr.io/google_containers/addon-resizer:1.0 - name: heapster-nanny - resources: - limits: - cpu: 50m - memory: 100Mi - requests: - cpu: 50m - memory: 100Mi - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - /pod_nanny - - --cpu=100m - - --extra-cpu=0m - - --memory=300Mi - - --extra-memory=4Mi - - --threshold=5 - - --deployment=heapster-v1.2.0 - - --container=heapster - - --poll-period=300000 diff --git a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/standalone/heapster-service.yaml b/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/standalone/heapster-service.yaml deleted file mode 100644 index 31e8b96006d..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/standalone/heapster-service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: heapster - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "Heapster" -spec: - ports: - - port: 80 - targetPort: 8082 - selector: - k8s-app: heapster diff --git a/cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml b/cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml deleted file mode 100644 index d4e60df4920..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Keep this file in sync with addons/dashboard/dashboard-controller.yaml -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: kubernetes-dashboard - namespace: kube-system - labels: - k8s-app: kubernetes-dashboard - kubernetes.io/cluster-service: "true" -spec: - selector: - matchLabels: - k8s-app: kubernetes-dashboard - template: - metadata: - labels: - k8s-app: kubernetes-dashboard - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' - spec: - containers: - - name: kubernetes-dashboard - image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.5.0 - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 50Mi - requests: - cpu: 100m - memory: 50Mi - ports: - - containerPort: 9090 - livenessProbe: - httpGet: - path: / - port: 9090 - initialDelaySeconds: 30 - timeoutSeconds: 30 diff --git a/cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml b/cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml deleted file mode 100644 index 195b503de10..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kubernetes-dashboard - namespace: kube-system - labels: - k8s-app: kubernetes-dashboard - kubernetes.io/cluster-service: "true" -spec: - selector: - k8s-app: kubernetes-dashboard - ports: - - port: 80 - targetPort: 9090 diff --git a/cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml b/cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml deleted file mode 100644 index fd22f64a6df..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml +++ /dev/null @@ -1,145 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: kube-dns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" -spec: - # replicas: not specified here: - # 1. In order to make Addon Manager do not reconcile this replicas parameter. - # 2. Default is 1. - # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. - strategy: - rollingUpdate: - maxSurge: 10% - maxUnavailable: 0 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - labels: - k8s-app: kube-dns - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' - spec: - containers: - - name: kubedns - image: gcr.io/google_containers/kubedns-amd64:1.9 - resources: - # TODO: Set memory limits when we've profiled the container for large - # clusters, then set request = limit to keep this container in - # guaranteed class. Currently, this container falls into the - # "burstable" category so the kubelet doesn't backoff from restarting it. - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - livenessProbe: - httpGet: - path: /healthz-kubedns - port: 8080 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /readiness - port: 8081 - scheme: HTTP - # we poll on pod startup for the Kubernetes master service and - # only setup the /readiness HTTP server once that's available. - initialDelaySeconds: 3 - timeoutSeconds: 5 - args: - # command = "/kube-dns" - - --domain=${DNS_DOMAIN}. - - --dns-port=10053 - - --config-map=kube-dns - - --v=2 - env: - - name: PROMETHEUS_PORT - value: "10055" - ports: - - containerPort: 10053 - name: dns-local - protocol: UDP - - containerPort: 10053 - name: dns-tcp-local - protocol: TCP - - containerPort: 10055 - name: metrics - protocol: TCP - - name: dnsmasq - image: gcr.io/google_containers/kube-dnsmasq-amd64:1.4 - livenessProbe: - httpGet: - path: /healthz-dnsmasq - port: 8080 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - args: - - --cache-size=1000 - - --no-resolv - - --server=127.0.0.1#10053 - - --log-facility=- - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - - name: dnsmasq-metrics - image: gcr.io/google_containers/dnsmasq-metrics-amd64:1.0 - livenessProbe: - httpGet: - path: /metrics - port: 10054 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - args: - - --v=2 - - --logtostderr - ports: - - containerPort: 10054 - name: metrics - protocol: TCP - resources: - requests: - memory: 10Mi - - name: healthz - image: gcr.io/google_containers/exechealthz-amd64:1.2 - resources: - limits: - memory: 50Mi - requests: - cpu: 10m - # Note that this container shouldn't really need 50Mi of memory. The - # limits are set higher than expected pending investigation on #29688. - # The extra memory was stolen from the kubedns container to keep the - # net memory requested by the pod constant. - memory: 50Mi - args: - - --cmd=nslookup kubernetes.default.svc.${DNS_DOMAIN} 127.0.0.1 >/dev/null - - --url=/healthz-dnsmasq - - --cmd=nslookup kubernetes.default.svc.${DNS_DOMAIN} 127.0.0.1:10053 >/dev/null - - --url=/healthz-kubedns - - --port=8080 - - --quiet - ports: - - containerPort: 8080 - protocol: TCP - dnsPolicy: Default # Don't use cluster DNS. diff --git a/cluster/gce/coreos/kube-manifests/addons/dns/skydns-svc.yaml b/cluster/gce/coreos/kube-manifests/addons/dns/skydns-svc.yaml deleted file mode 100644 index deeb0d9ba3d..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/dns/skydns-svc.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kube-dns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "KubeDNS" -spec: - selector: - k8s-app: kube-dns - clusterIP: ${DNS_SERVER_IP} - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP diff --git a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml b/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml deleted file mode 100644 index d24d6ed2fee..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: elasticsearch-logging-v1 - namespace: kube-system - labels: - k8s-app: elasticsearch-logging - version: v1 - kubernetes.io/cluster-service: "true" -spec: - replicas: 2 - selector: - k8s-app: elasticsearch-logging - version: v1 - template: - metadata: - labels: - k8s-app: elasticsearch-logging - version: v1 - kubernetes.io/cluster-service: "true" - spec: - containers: - - image: gcr.io/google_containers/elasticsearch:v2.4.1 - name: elasticsearch-logging - resources: - # need more cpu upon initialization, therefore burstable class - limits: - cpu: 1000m - requests: - cpu: 100m - ports: - - containerPort: 9200 - name: db - protocol: TCP - - containerPort: 9300 - name: transport - protocol: TCP - volumeMounts: - - name: es-persistent-storage - mountPath: /data - volumes: - - name: es-persistent-storage - emptyDir: {} diff --git a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-service.yaml b/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-service.yaml deleted file mode 100644 index abf1fd3f684..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: elasticsearch-logging - namespace: kube-system - labels: - k8s-app: elasticsearch-logging - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "Elasticsearch" -spec: - ports: - - port: 9200 - protocol: TCP - targetPort: db - selector: - k8s-app: elasticsearch-logging diff --git a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/kibana-controller.yaml b/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/kibana-controller.yaml deleted file mode 100644 index 3ec688e1f32..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/kibana-controller.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: kibana-logging-v1 - namespace: kube-system - labels: - k8s-app: kibana-logging - version: v1 - kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kibana-logging - version: v1 - template: - metadata: - labels: - k8s-app: kibana-logging - version: v1 - kubernetes.io/cluster-service: "true" - spec: - containers: - - name: kibana-logging - image: gcr.io/google_containers/kibana:v4.6.1 - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - requests: - cpu: 100m - env: - - name: "ELASTICSEARCH_URL" - value: "http://elasticsearch-logging:9200" - ports: - - containerPort: 5601 - name: ui - protocol: TCP \ No newline at end of file diff --git a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/kibana-service.yaml b/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/kibana-service.yaml deleted file mode 100644 index 43efada2c50..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/kibana-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kibana-logging - namespace: kube-system - labels: - k8s-app: kibana-logging - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "Kibana" -spec: - ports: - - port: 5601 - protocol: TCP - targetPort: ui - selector: - k8s-app: kibana-logging diff --git a/cluster/gce/coreos/kube-manifests/addons/namespace.yaml b/cluster/gce/coreos/kube-manifests/addons/namespace.yaml deleted file mode 100644 index 986f4b48221..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: kube-system diff --git a/cluster/gce/coreos/kube-manifests/addons/node-problem-detector/node-problem-detector.yaml b/cluster/gce/coreos/kube-manifests/addons/node-problem-detector/node-problem-detector.yaml deleted file mode 100644 index e206cdf2092..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/node-problem-detector/node-problem-detector.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: node-problem-detector-v0.1 - namespace: kube-system - labels: - k8s-app: node-problem-detector - version: v0.1 - kubernetes.io/cluster-service: "true" -spec: - template: - metadata: - labels: - k8s-app: node-problem-detector - version: v0.1 - kubernetes.io/cluster-service: "true" - spec: - hostNetwork: true - containers: - - name: node-problem-detector - image: gcr.io/google_containers/node-problem-detector:v0.1 - env: - # Config the host ip and port of apiserver. - - name: "KUBERNETES_SERVICE_HOST" - value: ${INSTANCE_PREFIX}-master - - name: "KUBERNETES_SERVICE_PORT" - value: "443" - securityContext: - privileged: true - resources: - limits: - cpu: "200m" - memory: "100Mi" - requests: - cpu: "20m" - memory: "20Mi" - volumeMounts: - - name: log - mountPath: /log - readOnly: true - volumes: - - name: log - hostPath: - path: /var/log/ diff --git a/cluster/gce/coreos/kube-manifests/addons/registry/registry-pv.yaml b/cluster/gce/coreos/kube-manifests/addons/registry/registry-pv.yaml deleted file mode 100644 index bb4ceb532bb..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/registry/registry-pv.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: PersistentVolume -apiVersion: v1 -metadata: - name: kube-system-kube-registry-pv - labels: - kubernetes.io/cluster-service: "true" -spec: - capacity: - storage: ${CLUSTER_REGISTRY_DISK_SIZE} - accessModes: - - ReadWriteOnce - gcePersistentDisk: - pdName: ${CLUSTER_REGISTRY_DISK} - fsType: "ext4" diff --git a/cluster/gce/coreos/kube-manifests/addons/registry/registry-pvc.yaml b/cluster/gce/coreos/kube-manifests/addons/registry/registry-pvc.yaml deleted file mode 100644 index 4f6c8da6d7b..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/registry/registry-pvc.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: kube-registry-pvc - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: ${CLUSTER_REGISTRY_DISK_SIZE} diff --git a/cluster/gce/coreos/kube-manifests/addons/registry/registry-rc.yaml b/cluster/gce/coreos/kube-manifests/addons/registry/registry-rc.yaml deleted file mode 100644 index 2a8bd477a68..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/registry/registry-rc.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-registry-v0 - namespace: kube-system - labels: - k8s-app: kube-registry - version: v0 - kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kube-registry - version: v0 - template: - metadata: - labels: - k8s-app: kube-registry - version: v0 - kubernetes.io/cluster-service: "true" - spec: - containers: - - name: registry - image: registry:2 - resources: - limits: - cpu: 100m - memory: 100Mi - env: - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY - value: /var/lib/registry - volumeMounts: - - name: image-store - mountPath: /var/lib/registry - ports: - - containerPort: 5000 - name: registry - protocol: TCP - volumes: - - name: image-store - persistentVolumeClaim: - claimName: kube-registry-pvc diff --git a/cluster/gce/coreos/kube-manifests/addons/registry/registry-svc.yaml b/cluster/gce/coreos/kube-manifests/addons/registry/registry-svc.yaml deleted file mode 100644 index b9f1cc40b99..00000000000 --- a/cluster/gce/coreos/kube-manifests/addons/registry/registry-svc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kube-registry - namespace: kube-system - labels: - k8s-app: kube-registry - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "KubeRegistry" -spec: - selector: - k8s-app: kube-registry - ports: - - name: registry - port: 5000 - protocol: TCP diff --git a/cluster/gce/coreos/kube-manifests/etcd-events.yaml b/cluster/gce/coreos/kube-manifests/etcd-events.yaml deleted file mode 100644 index 7479b1a1b93..00000000000 --- a/cluster/gce/coreos/kube-manifests/etcd-events.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: etcd-server-events - namespace: kube-system -spec: - containers: - - command: - - /bin/sh - - -c - - /usr/local/bin/etcd - --listen-peer-urls=http://127.0.0.1:2381 - --advertise-client-urls=http://127.0.0.1:4002 - --listen-client-urls=http://127.0.0.1:4002 - --data-dir=/var/etcd/data-events - 1>>/var/log/etcd-events.log 2>&1 - image: gcr.io/google_containers/etcd:2.2.1 - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /health - port: 4002 - scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: etcd-container - ports: - - containerPort: 2381 - hostPort: 2381 - name: serverport - protocol: TCP - - containerPort: 4002 - hostPort: 4002 - name: clientport - protocol: TCP - resources: - limits: - cpu: 100m - requests: - cpu: 100m - volumeMounts: - - mountPath: /var/etcd - name: varetcd - - mountPath: /var/log/etcd-events.log - name: varlogetcd - dnsPolicy: ClusterFirst - hostNetwork: true - nodeName: ${INSTANCE_PREFIX}-master - restartPolicy: Always - terminationGracePeriodSeconds: 30 - volumes: - - hostPath: - path: /mnt/master-pd/var/etcd - name: varetcd - - hostPath: - path: /var/log/etcd-events.log - name: varlogetcd diff --git a/cluster/gce/coreos/kube-manifests/etcd.yaml b/cluster/gce/coreos/kube-manifests/etcd.yaml deleted file mode 100644 index 5c07614ed90..00000000000 --- a/cluster/gce/coreos/kube-manifests/etcd.yaml +++ /dev/null @@ -1,55 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: etcd-server - namespace: kube-system -spec: - containers: - - command: - - /bin/sh - - -c - - /usr/local/bin/etcd - --listen-peer-urls=http://127.0.0.1:2380 - --advertise-client-urls=http://127.0.0.1:2379 - --listen-client-urls=http://127.0.0.1:2379 - --data-dir=/var/etcd/data - 1>>/var/log/etcd.log 2>&1 - image: gcr.io/google_containers/etcd:2.2.1 - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /health - port: 2379 - scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: etcd-container - ports: - - containerPort: 2380 - hostPort: 2380 - name: serverport - protocol: TCP - - containerPort: 2379 - hostPort: 2379 - name: clientport - protocol: TCP - resources: - requests: - cpu: 200m - volumeMounts: - - mountPath: /var/etcd - name: varetcd - - mountPath: /var/log/etcd.log - name: varlogetcd - dnsPolicy: ClusterFirst - hostNetwork: true - restartPolicy: Always - terminationGracePeriodSeconds: 30 - volumes: - - hostPath: - path: /mnt/master-pd/var/etcd - name: varetcd - - hostPath: - path: /var/log/etcd.log - name: varlogetcd diff --git a/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml b/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml deleted file mode 100644 index ed55115c43a..00000000000 --- a/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: kube-addon-manager - namespace: kube-system - labels: - component: kube-addon-manager -spec: - hostNetwork: true - containers: - - name: kube-addon-manager - image: gcr.io/google-containers/kube-addon-manager:v6.1 - command: - - /bin/bash - - -c - - /opt/kube-addons.sh 1>>/var/log/kube-addon-manager.log 2>&1 - resources: - requests: - cpu: 5m - memory: 50Mi - volumeMounts: - - mountPath: /etc/kubernetes/ - name: addons - readOnly: true - - mountPath: /var/log - name: varlog - readOnly: false - volumes: - - hostPath: - path: /etc/kubernetes/ - name: addons - - hostPath: - path: /var/log - name: varlog diff --git a/cluster/gce/coreos/kube-manifests/kube-apiserver.yaml b/cluster/gce/coreos/kube-manifests/kube-apiserver.yaml deleted file mode 100644 index 62f82276462..00000000000 --- a/cluster/gce/coreos/kube-manifests/kube-apiserver.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: kube-apiserver - namespace: kube-system -spec: - containers: - - command: - - /bin/sh - - -c - - /usr/local/bin/kube-apiserver - --address=127.0.0.1 - --etcd-servers=http://127.0.0.1:2379 - --etcd-servers-overrides=/events#http://127.0.0.1:4002 - --cloud-provider=gce - --admission-control=${ADMISSION_CONTROL} - --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE} - --client-ca-file=/srv/kubernetes/ca.crt - --basic-auth-file=/srv/kubernetes/basic_auth.csv - --tls-cert-file=/srv/kubernetes/server.cert - --tls-private-key-file=/srv/kubernetes/server.key - --secure-port=443 - --token-auth-file=/srv/kubernetes/known_tokens.csv - --v=2 - --allow-privileged=True - 1>>/var/log/kube-apiserver.log 2>&1 - image: gcr.io/google_containers/kube-apiserver:${KUBE_APISERVER_DOCKER_TAG} - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-apiserver - ports: - - containerPort: 443 - hostPort: 443 - name: https - protocol: TCP - - containerPort: 8080 - hostPort: 8080 - name: local - protocol: TCP - resources: - requests: - cpu: 250m - volumeMounts: - - mountPath: /srv/kubernetes - name: srvkube - readOnly: true - - mountPath: /var/log/kube-apiserver.log - name: logfile - - mountPath: /etc/ssl - name: etcssl - readOnly: true - - mountPath: /usr/share/ca-certificates - name: usrsharecacerts - readOnly: true - - mountPath: /srv/sshproxy - name: srvsshproxy - dnsPolicy: ClusterFirst - hostNetwork: true - restartPolicy: Always - terminationGracePeriodSeconds: 30 - volumes: - - hostPath: - path: /srv/kubernetes - name: srvkube - - hostPath: - path: /var/log/kube-apiserver.log - name: logfile - - hostPath: - path: /etc/ssl - name: etcssl - - hostPath: - path: /usr/share/ca-certificates - name: usrsharecacerts - - hostPath: - path: /srv/sshproxy - name: srvsshproxy diff --git a/cluster/gce/coreos/kube-manifests/kube-controller-manager.yaml b/cluster/gce/coreos/kube-manifests/kube-controller-manager.yaml deleted file mode 100644 index 3abc68da92c..00000000000 --- a/cluster/gce/coreos/kube-manifests/kube-controller-manager.yaml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: kube-controller-manager - namespace: kube-system -spec: - containers: - - command: - - /bin/sh - - -c - - /usr/local/bin/kube-controller-manager - --master=127.0.0.1:8080 - --cluster-name=${INSTANCE_PREFIX} - --cluster-cidr=${CLUSTER_IP_RANGE} - --service-cluster-ip-range="${SERVICE_CLUSTER_IP_RANGE}" - --allocate-node-cidrs=true - --cloud-provider=gce - --service-account-private-key-file=/srv/kubernetes/server.key - --v=2 - --root-ca-file=/srv/kubernetes/ca.crt - 1>>/var/log/kube-controller-manager.log 2>&1 - image: gcr.io/google_containers/kube-controller-manager:${KUBE_CONTROLLER_MANAGER_DOCKER_TAG} - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10252 - scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-controller-manager - resources: - limits: - cpu: 200m - requests: - cpu: 200m - volumeMounts: - - mountPath: /srv/kubernetes - name: srvkube - readOnly: true - - mountPath: /var/log/kube-controller-manager.log - name: logfile - - mountPath: /etc/ssl - name: etcssl - readOnly: true - - mountPath: /usr/share/ca-certificates - name: usrsharecacerts - readOnly: true - dnsPolicy: ClusterFirst - hostNetwork: true - restartPolicy: Always - terminationGracePeriodSeconds: 30 - volumes: - - hostPath: - path: /srv/kubernetes - name: srvkube - - hostPath: - path: /var/log/kube-controller-manager.log - name: logfile - - hostPath: - path: /etc/ssl - name: etcssl - - hostPath: - path: /usr/share/ca-certificates - name: usrsharecacerts diff --git a/cluster/gce/coreos/kube-manifests/kube-scheduler.yaml b/cluster/gce/coreos/kube-manifests/kube-scheduler.yaml deleted file mode 100644 index da504c56a76..00000000000 --- a/cluster/gce/coreos/kube-manifests/kube-scheduler.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: kube-scheduler - namespace: kube-system -spec: - containers: - - command: - - /bin/sh - - -c - - /usr/local/bin/kube-scheduler - --master=127.0.0.1:8080 - --v=2 - 1>>/var/log/kube-scheduler.log 2>&1 - image: gcr.io/google_containers/kube-scheduler:${KUBE_SCHEDULER_DOCKER_TAG} - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10251 - scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-scheduler - resources: - requests: - cpu: 100m - volumeMounts: - - mountPath: /var/log/kube-scheduler.log - name: logfile - dnsPolicy: ClusterFirst - hostNetwork: true - nodeName: ${INSTANCE_PREFIX}-master - restartPolicy: Always - terminationGracePeriodSeconds: 30 - volumes: - - hostPath: - path: /var/log/kube-scheduler.log - name: logfile diff --git a/cluster/gce/coreos/kube-manifests/kube-system.json b/cluster/gce/coreos/kube-manifests/kube-system.json deleted file mode 100644 index 395b9722bf6..00000000000 --- a/cluster/gce/coreos/kube-manifests/kube-system.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "apiVersion": "v1", - "kind": "Namespace", - "metadata": { - "name": "kube-system" - } -} diff --git a/cluster/gce/coreos/kube-manifests/kubelet-config.yaml b/cluster/gce/coreos/kube-manifests/kubelet-config.yaml deleted file mode 100644 index 8524abe8aa9..00000000000 --- a/cluster/gce/coreos/kube-manifests/kubelet-config.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Config -users: -- name: kubelet - user: - client-certificate-data: ${KUBELET_CERT} - client-key-data: ${KUBELET_KEY} -clusters: -- name: local - cluster: - certificate-authority-data: ${CA_CERT} -contexts: -- context: - cluster: local - user: kubelet - name: service-account-context -current-context: service-account-context diff --git a/cluster/gce/coreos/kube-manifests/kubeproxy-config.yaml b/cluster/gce/coreos/kube-manifests/kubeproxy-config.yaml deleted file mode 100644 index c111eb0fa16..00000000000 --- a/cluster/gce/coreos/kube-manifests/kubeproxy-config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Config -users: -- name: kube-proxy - user: - token: ${KUBE_PROXY_TOKEN} -clusters: -- name: local - cluster: - certificate-authority-data: ${CA_CERT} -contexts: -- context: - cluster: local - user: kube-proxy - name: service-account-context -current-context: service-account-context diff --git a/cluster/gce/coreos/master-docker.yaml b/cluster/gce/coreos/master-docker.yaml deleted file mode 100644 index 89685aadd9c..00000000000 --- a/cluster/gce/coreos/master-docker.yaml +++ /dev/null @@ -1,168 +0,0 @@ -#cloud-config - -coreos: - update: - reboot-strategy: off - units: - - name: locksmithd.service - mask: true - - name: kube-env.service - command: start - content: | - [Unit] - Description=Fetch kubernetes-node-environment - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /etc/kube-env.yaml \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env - # Transform the yaml to env file. - ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env - ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env - - - name: kubernetes-install-cni.service - command: start - content: | - [Unit] - Description=Download cni - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/usr/bin/mkdir -p /opt/cni - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz - ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ - - - name: kubernetes-download-salt.service - command: start - content: | - [Unit] - Description=Download salt - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /opt/downloads - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ - /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL} - # TODO(yifan): Check hash. - ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite - - - name: kubernetes-download-manifests.service - command: start - content: | - [Unit] - Description=Download manifests - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /opt/downloads - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ - /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL} - # TODO(yifan): Check hash. - ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests - ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite - - - name: kubernetes-install-node.service - command: start - content: | - [Unit] - Description=Install Kubernetes Server - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /opt/downloads - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ - /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} - # TODO(yifan): Check hash. - ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite - - - name: kubelet.service - command: start - content: | - [Unit] - Description=Run Kubelet service - Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service - After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service - [Service] - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /run/configure-kubelet.sh \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet - ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh - ExecStartPre=/run/configure-kubelet.sh - ExecStart=/opt/kubernetes/server/bin/kubelet \ - --api-servers=https://${INSTANCE_PREFIX}-master \ - --enable-debugging-handlers=false \ - --cloud-provider=gce \ - --config=/etc/kubernetes/manifests \ - --allow-privileged=true \ - --v=2 \ - --cluster-dns=${DNS_SERVER_IP} \ - --cluster-domain=${DNS_DOMAIN} \ - --logtostderr=true \ - --container-runtime=docker \ - --pod-cidr=${MASTER_IP_RANGE} \ - --register-schedulable=false - Restart=always - RestartSec=10 - KillMode=process - - - name: docker.service - drop-ins: - - name: 50-docker-opts.conf - content: | - [Service] - Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false' - MountFlags=slave - LimitNOFILE=1048576 - LimitNPROC=1048576 - LimitCORE=infinity - Restart=always - RestartSec=2s - StartLimitInterval=0 - - - name: kubernetes-configure-node.service - command: start - content: | - [Unit] - Description=Configure Node For Kubernetes service - Requires=kubernetes-install-node.service - After=kubernetes-install-node.service - Requires=kubernetes-download-salt.service - After=kubernetes-download-salt.service - Requires=kubernetes-download-manifests.service - After=kubernetes-download-manifests.service - # Need the kubelet/docker running because we will use docker load for docker images. - Requires=kubelet.service - After=kubelet.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /run/configure-node.sh \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node - ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh - ExecStart=/run/configure-node.sh diff --git a/cluster/gce/coreos/master-helper.sh b/cluster/gce/coreos/master-helper.sh index 87744cd2bf6..db67224f84e 100755 --- a/cluster/gce/coreos/master-helper.sh +++ b/cluster/gce/coreos/master-helper.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2015 The Kubernetes Authors. +# Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# A library of helper functions and constant for coreos os distro +# A library of helper functions and constant for GCI distro +source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" # create-master-instance creates the master instance. If called with # an argument, the argument is used as the name to a reserved IP @@ -24,23 +25,60 @@ # It requires a whole slew of assumed variables, partially due to to # the call to write-master-env. Listing them would be rather # futile. Instead, we list the required calls to ensure any additional +# # variables are set: # ensure-temp-dir # detect-project # get-bearer-token -# -function create-master-instance() { +function create-master-instance { local address_opt="" [[ -n ${1:-} ]] && address_opt="--address ${1}" + write-master-env + ensure-gci-metadata-files + create-master-instance-internal "${MASTER_NAME}" "${address_opt}" +} + +function replicate-master-instance() { + local existing_master_zone="${1}" + local existing_master_name="${2}" + local existing_master_replicas="${3}" + + local kube_env="$(get-metadata "${existing_master_zone}" "${existing_master_name}" kube-env)" + # Substitute INITIAL_ETCD_CLUSTER to enable etcd clustering. + kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER")" + kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER: '${existing_master_replicas},${REPLICA_NAME}'")" + ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" | sed "s/^.*: '//" | sed "s/'$//")" + ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" | sed "s/^.*: '//" | sed "s/'$//")" + + create-etcd-certs "${REPLICA_NAME}" "${ETCD_CA_CERT}" "${ETCD_CA_KEY}" + + kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")" + kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")" + kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_CERT")" + kube_env="$(echo -e "${kube_env}\nETCD_PEER_CERT: '${ETCD_PEER_CERT_BASE64}'")" + + echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml + get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-ensure-gke-docker > "${KUBE_TEMP}/gci-ensure-gke-docker.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-docker-version > "${KUBE_TEMP}/gci-docker-version.txt" + + create-master-instance-internal "${REPLICA_NAME}" +} + + +function create-master-instance-internal() { + local -r master_name="${1}" + local -r address_option="${2:-}" + local preemptible_master="" if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then preemptible_master="--preemptible --maintenance-policy TERMINATE" fi - write-master-env - gcloud compute instances create "${MASTER_NAME}" \ - ${address_opt} \ + gcloud compute instances create "${master_name}" \ + ${address_option} \ --project "${PROJECT}" \ --zone "${ZONE}" \ --machine-type "${MASTER_SIZE}" \ @@ -51,8 +89,18 @@ function create-master-instance() { --scopes "storage-ro,compute-rw,monitoring,logging-write" \ --can-ip-forward \ --metadata-from-file \ - "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master-${CONTAINER_RUNTIME}.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \ - --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ + "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" \ + --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \ - ${preemptible_master} + ${preemptible_master} +} + +function get-metadata() { + local zone="${1}" + local name="${2}" + local key="${3}" + gcloud compute ssh "${name}" \ + --project "${PROJECT}" \ + --zone "${zone}" \ + --command "curl \"http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}\" -H \"Metadata-Flavor: Google\"" 2>/dev/null } diff --git a/cluster/gce/coreos/master-rkt.yaml b/cluster/gce/coreos/master-rkt.yaml deleted file mode 100644 index 8138a0469cb..00000000000 --- a/cluster/gce/coreos/master-rkt.yaml +++ /dev/null @@ -1,220 +0,0 @@ -#cloud-config - -coreos: - update: - reboot-strategy: off - units: - - name: locksmithd.service - mask: true - - name: kube-env.service - command: start - content: | - [Unit] - Description=Fetch kubernetes-node-environment - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /etc/kube-env.yaml \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env - # Transform the yaml to env file. - ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env - ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env - - - name: kubernetes-install-cni.service - command: start - content: | - [Unit] - Description=Download cni - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/usr/bin/mkdir -p /opt/cni - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz - ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ - - - name: kubernetes-install-docker2aci.service - command: start - content: | - [Unit] - Description=Download docker2aci - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/usr/bin/mkdir -p /opt/docker2aci - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/docker2aci.tar.gz https://github.com/appc/docker2aci/releases/download/v0.11.1/docker2aci-v0.11.1.tar.gz - ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/docker2aci.tar.gz -C /opt/docker2aci/ --overwrite - - - name: kubernetes-install-rkt.service - command: start - content: | - [Unit] - Description=Fetch rkt - Documentation=http://github.com/coreos/rkt - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/ - ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz - ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite - - - name: kubernetes-load-rkt-stage1.service - command: start - content: | - [Unit] - Description=Load rkt stage1 images - Documentation=http://github.com/coreos/rkt - Requires=network-online.target - After=network-online.target - Requires=kubernetes-install-rkt.service - After=kubernetes-install-rkt.service - [Service] - Type=oneshot - ExecStart=/opt/rkt/rkt fetch /opt/rkt/stage1-coreos.aci /opt/rkt/stage1-kvm.aci /opt/rkt/stage1-fly.aci --insecure-options=image - - - name: kubernetes-download-salt.service - command: start - content: | - [Unit] - Description=Download salt - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /opt/downloads - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ - /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL} - # TODO(yifan): Check hash. - ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite - - - name: kubernetes-download-manifests.service - command: start - content: | - [Unit] - Description=Download manifests - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /opt/downloads - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ - /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL} - # TODO(yifan): Check hash. - ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests - ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite - - - name: kubernetes-install-node.service - command: start - content: | - [Unit] - Description=Install Kubernetes Server - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /opt/downloads - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ - /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} - # TODO(yifan): Check hash. - ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite - - - name: rkt-api-service.service - command: start - content: | - [Unit] - Description=Start rkt API service as Daemon - Requires=kubernetes-install-rkt.service - After=kubernetes-install-rkt.service - [Service] - ExecStart=/opt/rkt/rkt api-service - Restart=always - RestartSec=10 - - - name: kubelet.service - command: start - content: | - [Unit] - Description=Run Kubelet service - Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service kubernetes-load-rkt-stage1.service - After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service kubernetes-load-rkt-stage1.service - [Service] - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /run/configure-kubelet.sh \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet - ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh - ExecStartPre=/run/configure-kubelet.sh - ExecStart=/opt/kubernetes/server/bin/kubelet \ - --api-servers=https://${INSTANCE_PREFIX}-master \ - --enable-debugging-handlers=false \ - --cloud-provider=gce \ - --config=/etc/kubernetes/manifests \ - --allow-privileged=true \ - --v=2 \ - --cluster-dns=${DNS_SERVER_IP} \ - --cluster-domain=${DNS_DOMAIN} \ - --logtostderr=true \ - --container-runtime=rkt \ - --rkt-path=/opt/rkt/rkt \ - --rkt-stage1-image=${RKT_STAGE_IMAGE} \ - --pod-cidr=${MASTER_IP_RANGE} \ - --register-schedulable=false - Restart=always - RestartSec=10 - KillMode=process - - - name: docker.service - command: stop - - - name: kubernetes-configure-node.service - command: start - content: | - [Unit] - Description=Configure Node For Kubernetes service - Requires=kubernetes-install-node.service - After=kubernetes-install-node.service - Requires=kubernetes-install-rkt.service - After=kubernetes-install-rkt.service - Requires=kubernetes-download-salt.service - After=kubernetes-download-salt.service - Requires=kubernetes-download-manifests.service - After=kubernetes-download-manifests.service - Requires=kubernetes-install-docker2aci.service - After=kubernetes-install-docker2aci.service - Requires=kubelet.service - After=kubelet.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /run/configure-node.sh \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node - ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh - ExecStart=/run/configure-node.sh diff --git a/cluster/gce/coreos/master.yaml b/cluster/gce/coreos/master.yaml new file mode 100644 index 00000000000..78b39164ae4 --- /dev/null +++ b/cluster/gce/coreos/master.yaml @@ -0,0 +1,124 @@ +#cloud-config + +write_files: + - path: /etc/systemd/system/kube-master-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-master-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes master + After=kube-master-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-docker-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for docker + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh docker + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-master-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + +runcmd: + - systemctl daemon-reload + - systemctl enable kube-master-installation.service + - systemctl enable kube-master-configuration.service + - systemctl enable kube-docker-monitor.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl start kubernetes.target diff --git a/cluster/gce/coreos/mounter/Changelog b/cluster/gce/coreos/mounter/Changelog new file mode 100644 index 00000000000..ced9cdd097d --- /dev/null +++ b/cluster/gce/coreos/mounter/Changelog @@ -0,0 +1,7 @@ +## v1 (Thu Oct 20 2016 Vishnu Kannan ) + - Creating a container with mount tools pre-installed + - Digest: sha256:9b3c1f04ad6b8947af4eb98f1eff2dc54c5664e3469b4cdf722ec5dd2a1dc064 + +## v2 (Fri Oct 28 2016 Vishnu Kannan ) + - Adding netbase package. + - Digest: sha256:c7dfe059fbbf976fc4284a87eb18adf0f8e0c4cf30a30f5a852842c772a64c2d diff --git a/cluster/gce/coreos/mounter/Dockerfile b/cluster/gce/coreos/mounter/Dockerfile new file mode 100644 index 00000000000..bfdc3f6b7da --- /dev/null +++ b/cluster/gce/coreos/mounter/Dockerfile @@ -0,0 +1,20 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:xenial +MAINTAINER vishh@google.com + +RUN apt-get update && apt-get install -y netbase nfs-common=1:1.2.8-9ubuntu12 glusterfs-client=3.7.6-1ubuntu1 + +ENTRYPOINT ["/bin/mount"] diff --git a/cluster/gce/coreos/configure-kubelet.sh b/cluster/gce/coreos/mounter/Makefile old mode 100755 new mode 100644 similarity index 51% rename from cluster/gce/coreos/configure-kubelet.sh rename to cluster/gce/coreos/mounter/Makefile index 4395e909b3b..72efa3b77c4 --- a/cluster/gce/coreos/configure-kubelet.sh +++ b/cluster/gce/coreos/mounter/Makefile @@ -1,6 +1,4 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. +# Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,20 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -o errexit -set -o nounset -set -o pipefail +TAG=v2 +REGISTRY=gcr.io/google_containers +IMAGE=gci-mounter -MANIFESTS_DIR=/opt/kube-manifests/kubernetes +all: container -echo "Configuring hostname" -hostnamectl set-hostname $(hostname | cut -f1 -d.) +container: + docker build --pull -t ${REGISTRY}/${IMAGE}:${TAG} . -echo "Configuring kubelet" -mkdir -p /var/lib/kubelet -mkdir -p /etc/kubernetes/manifests -src=${MANIFESTS_DIR}/kubelet-config.yaml -dst=/var/lib/kubelet/kubeconfig -cp ${src} ${dst} -sed -i 's/\"/\\\"/g' ${dst} # eval will remove the double quotes if they are not escaped -eval "echo \"$(< ${dst})\"" > ${dst} +push: + gcloud docker -- push ${REGISTRY}/${IMAGE}:${TAG} + +upload: + ./stage-upload.sh ${TAG} ${REGISTRY}/${IMAGE}:${TAG} + +.PHONY: all container push diff --git a/cluster/gce/coreos/mounter/mounter b/cluster/gce/coreos/mounter/mounter new file mode 100755 index 00000000000..2fba3255f2b --- /dev/null +++ b/cluster/gce/coreos/mounter/mounter @@ -0,0 +1,55 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +MOUNTER_VERSION=v2 +MOUNTER_USER=root +ROOT_DIR=/home/kubernetes/bin +RKT_BINARY=${ROOT_DIR}/rkt +STAGE1_ACI=${ROOT_DIR}/stage1-fly.aci +MOUNTER_ACI=${ROOT_DIR}/gci-mounter-${MOUNTER_VERSION}.aci +MOUNTER_IMAGE=gcr.io/google_containers/gci-mounter:${MOUNTER_VERSION} + +function gc { + # Attempt to garbage collect rkt pods with 5 retries. + # Rkt pods end up creating new copies of mounts on the host. Hence it is ideal to clean them up right away. + attempt=0 + until [ $attempt -ge 5 ]; do + ${RKT_BINARY} gc --grace-period=0s &> /dev/null + attempt=$[$attempt+1] + sleep 1 + done +} + +# Garbage collect old rkt containers on exit +trap gc EXIT + +if [[ ! $(${RKT_BINARY} image list | grep ${MOUNTER_IMAGE}) ]]; then + ${RKT_BINARY} fetch --insecure-options=image file://${MOUNTER_ACI} +fi + +echo "Running mount using a rkt fly container" + +${RKT_BINARY} run --stage1-path=${STAGE1_ACI} \ + --insecure-options=image \ + --volume=kubelet,kind=host,source=/var/lib/kubelet,readOnly=false,recursive=true \ + --mount volume=kubelet,target=/var/lib/kubelet \ + ${MOUNTER_IMAGE} --user=${MOUNTER_USER} --exec /bin/mount -- "$@" + +echo "Successfully ran mount using a rkt fly container" diff --git a/cluster/gce/coreos/mounter/stage-upload.sh b/cluster/gce/coreos/mounter/stage-upload.sh new file mode 100755 index 00000000000..66780da381e --- /dev/null +++ b/cluster/gce/coreos/mounter/stage-upload.sh @@ -0,0 +1,86 @@ +#!/bin/sh + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Due to the GCE custom metadata size limit, we split the entire script into two +# files configure.sh and configure-helper.sh. The functionality of downloading +# kubernetes configuration, manifests, docker images, and binary files are +# put in configure.sh, which is uploaded via GCE custom metadata. + +set -o errexit +set -o pipefail +set -o nounset + +RKT_VERSION="v1.18.0" +DOCKER2ACI_VERSION="v0.13.0" +MOUNTER_VERSION=$1 +DOCKER_IMAGE=docker://$2 +MOUNTER_ACI_IMAGE=gci-mounter-${MOUNTER_VERSION}.aci +RKT_GCS_DIR=gs://kubernetes-release/rkt/ +MOUNTER_GCS_DIR=gs://kubernetes-release/gci-mounter/ + +TMPDIR=/tmp +# Setup a working directory +DOWNLOAD_DIR=$(mktemp --tmpdir=${TMPDIR} -d gci-mounter-build.XXXXXXXXXX) + +# Setup a staging directory +STAGING_DIR=$(mktemp --tmpdir=${TMPDIR} -d gci-mounter-staging.XXXXXXXXXX) +RKT_DIR=${STAGING_DIR}/${RKT_VERSION} +ACI_DIR=${STAGING_DIR}/gci-mounter +CWD=${PWD} + +# Cleanup the temporary directories +function cleanup { + rm -rf ${DOWNLOAD_DIR} + rm -rf ${STAGING_DIR} + cd ${CWD} +} + +# Delete temporary directories on exit +trap cleanup EXIT + +mkdir ${RKT_DIR} +mkdir ${ACI_DIR} + +# Download rkt +cd ${DOWNLOAD_DIR} +echo "Downloading rkt ${RKT_VERSION}" +wget "https://github.com/coreos/rkt/releases/download/${RKT_VERSION}/rkt-${RKT_VERSION}.tar.gz" &> /dev/null +echo "Extracting rkt ${RKT_VERSION}" +tar xzf rkt-${RKT_VERSION}.tar.gz + +# Stage rkt into working directory +cp rkt-${RKT_VERSION}/rkt ${RKT_DIR}/rkt +cp rkt-${RKT_VERSION}/stage1-fly.aci ${RKT_DIR}/ + +# Convert docker image to aci and stage it +echo "Downloading docker2aci ${DOCKER2ACI_VERSION}" +wget "https://github.com/appc/docker2aci/releases/download/${DOCKER2ACI_VERSION}/docker2aci-${DOCKER2ACI_VERSION}.tar.gz" &> /dev/null +echo "Extracting docker2aci ${DOCKER2ACI_VERSION}" +tar xzf docker2aci-${DOCKER2ACI_VERSION}.tar.gz +ACI_IMAGE=$(${DOWNLOAD_DIR}/docker2aci-${DOCKER2ACI_VERSION}/docker2aci ${DOCKER_IMAGE} 2>/dev/null | tail -n 1) +cp ${ACI_IMAGE} ${ACI_DIR}/${MOUNTER_ACI_IMAGE} + +# Upload the contents to gcs +echo "Uploading rkt artifacts in ${RKT_DIR} to ${RKT_GCS_DIR}" +gsutil cp -R ${RKT_DIR} ${RKT_GCS_DIR} +echo "Uploading gci mounter ACI in ${ACI_DIR} to ${MOUNTER_GCS_DIR}" +gsutil cp ${ACI_DIR}/${MOUNTER_ACI_IMAGE} ${MOUNTER_GCS_DIR} + +echo "Upload completed" +echo "Update rkt, stag1-fly.aci & gci-mounter ACI versions and SHA1 in cluster/gce/gci/configure.sh" +echo "${RKT_VERSION}/rkt sha1: $(sha1sum ${RKT_DIR}/rkt)" +echo "${RKT_VERSION}/stage1-fly.aci sha1: $(sha1sum ${RKT_DIR}/stage1-fly.aci)" +echo "${MOUNTER_ACI_IMAGE} hash: $(sha1sum ${ACI_DIR}/${MOUNTER_ACI_IMAGE})" diff --git a/cluster/gce/coreos/node-docker.yaml b/cluster/gce/coreos/node-docker.yaml deleted file mode 100644 index 87cc48429d4..00000000000 --- a/cluster/gce/coreos/node-docker.yaml +++ /dev/null @@ -1,159 +0,0 @@ -#cloud-config - -coreos: - update: - reboot-strategy: off - units: - - name: locksmithd.service - mask: true - - name: kube-env.service - command: start - content: | - [Unit] - Description=Fetch kubernetes-node-environment - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /etc/kube-env.yaml \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env - # Transform the yaml to env file. - ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env - ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env - - - name: kubernetes-install-cni.service - command: start - content: | - [Unit] - Description=Download cni - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/usr/bin/mkdir -p /opt/cni - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz - ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ - - - name: kubernetes-download-manifests.service - command: start - content: | - [Unit] - Description=Download manifests - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /opt/downloads - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ - /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL} - # TODO(yifan): Check hash. - ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests - ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite - - - name: kubernetes-install-node.service - command: start - content: | - [Unit] - Description=Install Kubernetes Server - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ - /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} - ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite - - - name: kubelet.service - command: start - content: | - [Unit] - Description=Run Kubelet service - Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service - After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service - [Service] - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /run/configure-kubelet.sh \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet - ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh - ExecStartPre=/run/configure-kubelet.sh - ExecStart=/opt/kubernetes/server/bin/kubelet \ - --api-servers=https://${INSTANCE_PREFIX}-master \ - --enable-debugging-handlers=true \ - --cloud-provider=gce \ - --config=/etc/kubernetes/manifests \ - --allow-privileged=true \ - --v=2 \ - --cluster-dns=${DNS_SERVER_IP} \ - --cluster-domain=${DNS_DOMAIN} \ - --logtostderr=true \ - --container-runtime=docker \ - --network-plugin=${NETWORK_PROVIDER} - Restart=always - RestartSec=10 - KillMode=process - - - name: kube-proxy.service - command: start - content: | - [Unit] - Description=Start Kube-proxy service as Daemon - Requires=kubernetes-configure-node.service - After=kubernetes-configure-node.service - [Service] - EnvironmentFile=/etc/kube-env - ExecStart=/opt/kubernetes/server/bin/kube-proxy \ - --master=https://${KUBERNETES_MASTER_NAME} \ - --kubeconfig=/var/lib/kube-proxy/kubeconfig \ - --v=2 \ - --logtostderr=true - Restart=always - RestartSec=10 - - - name: docker.service - drop-ins: - - name: 50-docker-opts.conf - content: | - [Service] - Environment='DOCKER_OPTS= --iptables=false --ip-masq=false' - MountFlags=slave - LimitNOFILE=1048576 - LimitNPROC=1048576 - LimitCORE=infinity - Restart=always - RestartSec=2s - StartLimitInterval=0 - - - name: kubernetes-configure-node.service - command: start - content: | - [Unit] - Description=Configure Node For Kubernetes service - Requires=kubernetes-install-node.service - After=kubernetes-install-node.service - Requires=kubernetes-download-manifests.service - After=kubernetes-download-manifests.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /run/configure-node.sh \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node - ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh - ExecStart=/run/configure-node.sh diff --git a/cluster/gce/coreos/node-helper.sh b/cluster/gce/coreos/node-helper.sh index 674c5729e23..3217b4e09da 100755 --- a/cluster/gce/coreos/node-helper.sh +++ b/cluster/gce/coreos/node-helper.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2015 The Kubernetes Authors. +# Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,15 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -# A library of helper functions and constant for coreos os distro +# A library of helper functions and constant for GCI distro +source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" -# $1: template name (required) -function create-node-instance-template() { +# $1: template name (required). +function create-node-instance-template { local template_name="$1" - create-node-template "$template_name" "${scope_flags}" \ + ensure-gci-metadata-files + create-node-template "$template_name" "${scope_flags[*]}" \ "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ - "user-data=${KUBE_ROOT}/cluster/gce/coreos/node-${CONTAINER_RUNTIME}.yaml" \ - "configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh" \ - "configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh" \ - "cluster-name=${KUBE_TEMP}/cluster-name.txt" + "user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml" \ + "configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" \ + "cluster-name=${KUBE_TEMP}/cluster-name.txt" \ + "gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \ + "gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" \ + "gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" } diff --git a/cluster/gce/coreos/node-rkt.yaml b/cluster/gce/coreos/node-rkt.yaml deleted file mode 100644 index 20349762d82..00000000000 --- a/cluster/gce/coreos/node-rkt.yaml +++ /dev/null @@ -1,196 +0,0 @@ -#cloud-config - -coreos: - update: - reboot-strategy: off - units: - - name: locksmithd.service - mask: true - - name: kube-env.service - command: start - content: | - [Unit] - Description=Fetch kubernetes-node-environment - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /etc/kube-env.yaml \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env - # Transform the yaml to env file. - ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env - ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env - - - name: kubernetes-install-cni.service - command: start - content: | - [Unit] - Description=Download cni - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/usr/bin/mkdir -p /opt/cni - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz - ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ - - - name: kubernetes-install-rkt.service - command: start - content: | - [Unit] - Description=Fetch rkt - Documentation=http://github.com/coreos/rkt - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/ - ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz - ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite - - - name: kubernetes-load-rkt-stage1.service - command: start - content: | - [Unit] - Description=Load rkt stage1 images - Documentation=http://github.com/coreos/rkt - Requires=network-online.target - After=network-online.target - Requires=kubernetes-install-rkt.service - After=kubernetes-install-rkt.service - [Service] - Type=oneshot - ExecStart=/opt/rkt/rkt fetch /opt/rkt/stage1-coreos.aci /opt/rkt/stage1-kvm.aci /opt/rkt/stage1-fly.aci --insecure-options=image - - - name: kubernetes-download-manifests.service - command: start - content: | - [Unit] - Description=Download manifests - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /opt/downloads - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ - /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL} - # TODO(yifan): Check hash. - ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests - ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite - - - name: kubernetes-install-node.service - command: start - content: | - [Unit] - Description=Install Kubernetes Server - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ - /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} - ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite - - - name: rkt-api-service.service - command: start - content: | - [Unit] - Description=Start rkt API service as Daemon - Requires=kubernetes-install-rkt.service - After=kubernetes-install-rkt.service - [Service] - ExecStart=/opt/rkt/rkt api-service - Restart=always - RestartSec=10 - - - name: kubelet.service - command: start - content: | - [Unit] - Description=Run Kubelet service - Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service kubernetes-load-rkt-stage1.service - After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service kubernetes-load-rkt-stage1.service - [Service] - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /run/configure-kubelet.sh \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet - ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh - ExecStartPre=/run/configure-kubelet.sh - ExecStart=/opt/kubernetes/server/bin/kubelet \ - --api-servers=https://${INSTANCE_PREFIX}-master \ - --enable-debugging-handlers=true \ - --cloud-provider=gce \ - --config=/etc/kubernetes/manifests \ - --allow-privileged=true \ - --v=2 \ - --cluster-dns=${DNS_SERVER_IP} \ - --cluster-domain=${DNS_DOMAIN} \ - --logtostderr=true \ - --container-runtime=rkt \ - --rkt-path=/opt/rkt/rkt \ - --rkt-stage1-image=${RKT_STAGE1_IMAGE} \ - --network-plugin=kubenet - Restart=always - RestartSec=10 - KillMode=process - - - name: kube-proxy.service - command: start - content: | - [Unit] - Description=Start Kube-proxy service as Daemon - Requires=kubernetes-configure-node.service - After=kubernetes-configure-node.service - [Service] - EnvironmentFile=/etc/kube-env - ExecStart=/opt/kubernetes/server/bin/kube-proxy \ - --master=https://${KUBERNETES_MASTER_NAME} \ - --kubeconfig=/var/lib/kube-proxy/kubeconfig \ - --v=2 \ - --logtostderr=true - Restart=always - RestartSec=10 - - - name: docker.service - command: stop - - - name: kubernetes-configure-node.service - command: start - content: | - [Unit] - Description=Configure Node For Kubernetes service - Requires=kubernetes-install-node.service - After=kubernetes-install-node.service - Requires=kubernetes-install-rkt.service - After=kubernetes-install-rkt.service - Requires=kubernetes-download-manifests.service - After=kubernetes-download-manifests.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/curl --fail --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o /run/configure-node.sh \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node - ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh - ExecStart=/run/configure-node.sh diff --git a/cluster/gce/coreos/node.yaml b/cluster/gce/coreos/node.yaml new file mode 100644 index 00000000000..d0cc12e752d --- /dev/null +++ b/cluster/gce/coreos/node.yaml @@ -0,0 +1,124 @@ +#cloud-config + +write_files: + - path: /etc/systemd/system/kube-node-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-node-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes node + After=kube-node-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-docker-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for docker + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh docker + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-node-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + +runcmd: + - systemctl daemon-reload + - systemctl enable kube-node-installation.service + - systemctl enable kube-node-configuration.service + - systemctl enable kube-docker-monitor.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl start kubernetes.target diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index bc903ab571d..11fe6f08d67 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -202,7 +202,6 @@ function install-kube-binary-config { rm -f "${KUBE_HOME}/${manifests_tar}.sha1" } - ######### Main Function ########## echo "Start to install kubernetes files" set-broken-motd @@ -211,3 +210,4 @@ download-kube-env source "${KUBE_HOME}/kube-env" install-kube-binary-config echo "Done for installing kubernetes files" + diff --git a/cluster/gce/gci/master-helper.sh b/cluster/gce/gci/master-helper.sh index e430aa0a6fe..db67224f84e 100755 --- a/cluster/gce/gci/master-helper.sh +++ b/cluster/gce/gci/master-helper.sh @@ -15,7 +15,6 @@ # limitations under the License. # A library of helper functions and constant for GCI distro - source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" # create-master-instance creates the master instance. If called with @@ -72,6 +71,12 @@ function replicate-master-instance() { function create-master-instance-internal() { local -r master_name="${1}" local -r address_option="${2:-}" + + local preemptible_master="" + if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then + preemptible_master="--preemptible --maintenance-policy TERMINATE" + fi + gcloud compute instances create "${master_name}" \ ${address_option} \ --project "${PROJECT}" \ @@ -86,7 +91,8 @@ function create-master-instance-internal() { --metadata-from-file \ "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" \ --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ - --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" + --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \ + ${preemptible_master} } function get-metadata() {