diff --git a/build/lib/release.sh b/build/lib/release.sh index 870451601f6..c9932ea29d2 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -414,7 +414,6 @@ function kube::release::package_kube_manifests_tarball() { cp "${salt_dir}/e2e-image-puller/e2e-image-puller.manifest" "${gci_dst_dir}/" cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${gci_dst_dir}/gci-configure-helper.sh" cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${gci_dst_dir}/health-monitor.sh" - cp "${KUBE_ROOT}/cluster/gce/container-linux/configure-helper.sh" "${gci_dst_dir}/container-linux-configure-helper.sh" cp -r "${salt_dir}/kube-admission-controls/limit-range" "${gci_dst_dir}" local objects objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo) diff --git a/cluster/common.sh b/cluster/common.sh index a2b947f1748..2aa73622a64 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -436,8 +436,8 @@ function find-release-tars() { # This tarball is used by GCI, Ubuntu Trusty, and Container Linux. KUBE_MANIFESTS_TAR= - if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "container-linux" || "${MASTER_OS_DISTRIBUTION:-}" == "ubuntu" ]] || \ - [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "container-linux" || "${NODE_OS_DISTRIBUTION:-}" == "ubuntu" ]] ; then + if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "ubuntu" ]] || \ + [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "ubuntu" ]] ; then KUBE_MANIFESTS_TAR=$(find-tar kubernetes-manifests.tar.gz) fi } @@ -576,9 +576,7 @@ function build-kube-env { local server_binary_tar_url=$SERVER_BINARY_TAR_URL local salt_tar_url=$SALT_TAR_URL local kube_manifests_tar_url="${KUBE_MANIFESTS_TAR_URL:-}" - if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "container-linux" ]] || \ - [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "container-linux" ]] || \ - [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ + if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]] ; then # TODO: Support fallback .tar.gz settings on Container Linux server_binary_tar_url=$(split_csv "${SERVER_BINARY_TAR_URL}") @@ -696,8 +694,8 @@ EOF TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD}) EOF fi - if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "container-linux") || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ - [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "container-linux") || "${NODE_OS_DISTRIBUTION}" = "ubuntu" ]] ; then + if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci") || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ + [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci") || "${NODE_OS_DISTRIBUTION}" = "ubuntu" ]] ; then cat >>$file <>$file <>$file </dev/null; then - useradd -s /sbin/nologin -d /var/etcd etcd - fi - chown -R etcd "${mount_point}/var/etcd" - chgrp -R etcd "${mount_point}/var/etcd" -} - -# replace_prefixed_line ensures: -# 1. the specified file exists -# 2. existing lines with the specified ${prefix} are removed -# 3. a new line with the specified ${prefix}${suffix} is appended -function replace_prefixed_line { - local -r file="${1:-}" - local -r prefix="${2:-}" - local -r suffix="${3:-}" - - touch "${file}" - awk "substr(\$0,0,length(\"${prefix}\")) != \"${prefix}\" { print }" "${file}" > "${file}.filtered" && mv "${file}.filtered" "${file}" - echo "${prefix}${suffix}" >> "${file}" -} - -# After the first boot and on upgrade, these files exist on the master-pd -# and should never be touched again (except perhaps an additional service -# account, see NB below.) -function create-master-auth { - echo "Creating master auth files" - local -r auth_dir="/etc/srv/kubernetes" - if [[ ! -e "${auth_dir}/ca.crt" && ! -z "${CA_CERT:-}" && ! -z "${MASTER_CERT:-}" && ! -z "${MASTER_KEY:-}" ]]; then - echo "${CA_CERT}" | base64 --decode > "${auth_dir}/ca.crt" - echo "${MASTER_CERT}" | base64 --decode > "${auth_dir}/server.cert" - echo "${MASTER_KEY}" | base64 --decode > "${auth_dir}/server.key" - fi - local -r basic_auth_csv="${auth_dir}/basic_auth.csv" - if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then - replace_prefixed_line "${basic_auth_csv}" "${KUBE_PASSWORD},${KUBE_USER}," "admin,system:masters" - fi - local -r known_tokens_csv="${auth_dir}/known_tokens.csv" - if [[ -n "${KUBE_BEARER_TOKEN:-}" ]]; then - replace_prefixed_line "${known_tokens_csv}" "${KUBE_BEARER_TOKEN}," "admin,admin,system:masters" - fi - if [[ -n "${KUBE_CONTROLLER_MANAGER_TOKEN:-}" ]]; then - replace_prefixed_line "${known_tokens_csv}" "${KUBE_CONTROLLER_MANAGER_TOKEN}," "system:kube-controller-manager,uid:system:kube-controller-manager" - fi - if [[ -n "${KUBE_SCHEDULER_TOKEN:-}" ]]; then - replace_prefixed_line "${known_tokens_csv}" "${KUBE_SCHEDULER_TOKEN}," "system:kube-scheduler,uid:system:kube-scheduler" - fi - if [[ -n "${KUBELET_TOKEN:-}" ]]; then - replace_prefixed_line "${known_tokens_csv}" "${KUBELET_TOKEN}," "kubelet,uid:kubelet,system:nodes" - fi - if [[ -n "${KUBE_PROXY_TOKEN:-}" ]]; then - replace_prefixed_line "${known_tokens_csv}" "${KUBE_PROXY_TOKEN}," "system:kube-proxy,uid:kube_proxy" - fi - local use_cloud_config="false" - cat </etc/gce.conf -[global] -EOF - if [[ -n "${GCE_API_ENDPOINT:-}" ]]; then - cat <>/etc/gce.conf -api-endpoint = ${GCE_API_ENDPOINT} -EOF - fi - if [[ -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" ]]; then - use_cloud_config="true" - cat <>/etc/gce.conf -token-url = ${TOKEN_URL} -token-body = ${TOKEN_BODY} -EOF - fi - if [[ -n "${PROJECT_ID:-}" ]]; then - use_cloud_config="true" - cat <>/etc/gce.conf -project-id = ${PROJECT_ID} -EOF - fi - if [[ -n "${NETWORK_PROJECT_ID:-}" ]]; then - use_cloud_config="true" - cat <>/etc/gce.conf -network-project-id = ${NETWORK_PROJECT_ID} -EOF - fi - if [[ -n "${NODE_NETWORK:-}" ]]; then - use_cloud_config="true" - cat <>/etc/gce.conf -network-name = ${NODE_NETWORK} -EOF - fi - if [[ -n "${NODE_SUBNETWORK:-}" ]]; then - use_cloud_config="true" - cat <>/etc/gce.conf -subnetwork-name = ${NODE_SUBNETWORK} -EOF - fi - if [[ -n "${NODE_INSTANCE_PREFIX:-}" ]]; then - use_cloud_config="true" - if [[ -n "${NODE_TAGS:-}" ]]; then - # split NODE_TAGS into an array by comma. - IFS=',' read -r -a node_tags <<< ${NODE_TAGS} - else - local -r node_tags="${NODE_INSTANCE_PREFIX}" - fi - cat <>/etc/gce.conf -node-instance-prefix = ${NODE_INSTANCE_PREFIX} -EOF - for tag in ${node_tags[@]}; do - cat <>/etc/gce.conf -node-tags = ${tag} -EOF - done - fi - if [[ -n "${MULTIZONE:-}" ]]; then - use_cloud_config="true" - cat <>/etc/gce.conf -multizone = ${MULTIZONE} -EOF - fi - if [[ -n "${GCE_ALPHA_FEATURES:-}" ]]; then - use_cloud_config="true" - # split GCE_ALPHA_FEATURES into an array by comma. - IFS=',' read -r -a alpha_features <<< ${GCE_ALPHA_FEATURES} - for feature in ${alpha_features[@]}; do - cat <>/etc/gce.conf -alpha-features = ${feature} -EOF - done - fi - if [[ -n "${SECONDARY_RANGE_NAME:-}" ]]; then - use_cloud_config="true" - cat <> /etc/gce.conf -secondary-range-name = ${SECONDARY_RANGE_NAME} -EOF - fi - if [[ "${use_cloud_config}" != "true" ]]; then - rm -f /etc/gce.conf - fi - - if [[ -n "${GCP_AUTHN_URL:-}" ]]; then - cat </etc/gcp_authn.config -clusters: - - name: gcp-authentication-server - cluster: - server: ${GCP_AUTHN_URL} -users: - - name: kube-apiserver - user: - auth-provider: - name: gcp -current-context: webhook -contexts: -- context: - cluster: gcp-authentication-server - user: kube-apiserver - name: webhook -EOF - fi - - if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then - cat </etc/gcp_authz.config -clusters: - - name: gcp-authorization-server - cluster: - server: ${GCP_AUTHZ_URL} -users: - - name: kube-apiserver - user: - auth-provider: - name: gcp -current-context: webhook -contexts: -- context: - cluster: gcp-authorization-server - user: kube-apiserver - name: webhook -EOF - fi - -if [[ -n "${GCP_IMAGE_VERIFICATION_URL:-}" ]]; then - # This is the config file for the image review webhook. - cat </etc/gcp_image_review.config -clusters: - - name: gcp-image-review-server - cluster: - server: ${GCP_IMAGE_VERIFICATION_URL} -users: - - name: kube-apiserver - user: - auth-provider: - name: gcp -current-context: webhook -contexts: -- context: - cluster: gcp-image-review-server - user: kube-apiserver - name: webhook -EOF - # This is the config for the image review admission controller. - cat </etc/admission_controller.config -imagePolicy: - kubeConfigFile: /etc/gcp_image_review.config - allowTTL: 30 - denyTTL: 30 - retryBackoff: 500 - defaultAllow: true -EOF - fi -} - -# Arg 1: the address of the API server -function create-kubelet-kubeconfig() { - local apiserver_address="${1}" - if [[ -z "${apiserver_address}" ]]; then - echo "Must provide API server address to create Kubelet kubeconfig file!" - exit 1 - fi - echo "Creating kubelet kubeconfig file" - if [[ -z "${KUBELET_CA_CERT:-}" ]]; then - KUBELET_CA_CERT="${CA_CERT}" - fi - cat </var/lib/kubelet/kubeconfig -apiVersion: v1 -kind: Config -users: -- name: kubelet - user: - client-certificate-data: ${KUBELET_CERT} - client-key-data: ${KUBELET_KEY} -clusters: -- name: local - cluster: - server: ${apiserver_address} - certificate-authority-data: ${KUBELET_CA_CERT} -contexts: -- context: - cluster: local - user: kubelet - name: service-account-context -current-context: service-account-context -EOF -} - -# Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY -# to generate a kubeconfig file for the kubelet to securely connect to the apiserver. -# Set REGISTER_MASTER_KUBELET to true if kubelet on the master node -# should register to the apiserver. -function create-master-kubelet-auth { - # Only configure the kubelet on the master if the required variables are - # set in the environment. - if [[ -n "${KUBELET_APISERVER:-}" && -n "${KUBELET_CERT:-}" && -n "${KUBELET_KEY:-}" ]]; then - REGISTER_MASTER_KUBELET="true" - create-kubelet-kubeconfig "https://${KUBELET_APISERVER}" - fi -} - -function create-kubeproxy-user-kubeconfig { - echo "Creating kube-proxy user kubeconfig file" - cat </var/lib/kube-proxy/kubeconfig -apiVersion: v1 -kind: Config -users: -- name: kube-proxy - user: - token: ${KUBE_PROXY_TOKEN} -clusters: -- name: local - cluster: - certificate-authority-data: ${CA_CERT} -contexts: -- context: - cluster: local - user: kube-proxy - name: service-account-context -current-context: service-account-context -EOF -} - -function create-kubecontrollermanager-kubeconfig { - echo "Creating kube-controller-manager kubeconfig file" - mkdir -p /etc/srv/kubernetes/kube-controller-manager - cat </etc/srv/kubernetes/kube-controller-manager/kubeconfig -apiVersion: v1 -kind: Config -users: -- name: kube-controller-manager - user: - token: ${KUBE_CONTROLLER_MANAGER_TOKEN} -clusters: -- name: local - cluster: - insecure-skip-tls-verify: true - server: https://localhost:443 -contexts: -- context: - cluster: local - user: kube-controller-manager - name: service-account-context -current-context: service-account-context -EOF -} - -function create-kubescheduler-kubeconfig { - echo "Creating kube-scheduler kubeconfig file" - mkdir -p /etc/srv/kubernetes/kube-scheduler - cat </etc/srv/kubernetes/kube-scheduler/kubeconfig -apiVersion: v1 -kind: Config -users: -- name: kube-scheduler - user: - token: ${KUBE_SCHEDULER_TOKEN} -clusters: -- name: local - cluster: - insecure-skip-tls-verify: true - server: https://localhost:443 -contexts: -- context: - cluster: local - user: kube-scheduler - name: kube-scheduler -current-context: kube-scheduler -EOF -} - -function create-master-etcd-auth { - if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then - local -r auth_dir="/etc/srv/kubernetes" - echo "${ETCD_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-ca.crt" - echo "${ETCD_PEER_KEY}" | base64 --decode > "${auth_dir}/etcd-peer.key" - echo "${ETCD_PEER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-peer.crt" - fi -} - -function configure-docker-daemon { - echo "Configuring the Docker daemon" - local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false" - if [[ "${TEST_CLUSTER:-}" == "true" ]]; then - docker_opts+=" --log-level=debug" - else - docker_opts+=" --log-level=warn" - fi - local use_net_plugin="true" - if [[ "${NETWORK_PROVIDER:-}" == "kubenet" || "${NETWORK_PROVIDER:-}" == "cni" ]]; then - # set docker0 cidr to private ip address range to avoid conflict with cbr0 cidr range - docker_opts+=" --bip=169.254.123.1/24" - else - use_net_plugin="false" - docker_opts+=" --bridge=cbr0" - fi - - # Decide whether to enable a docker registry mirror. This is taken from - # the "kube-env" metadata value. - if [[ -n "${DOCKER_REGISTRY_MIRROR_URL:-}" ]]; then - echo "Enable docker registry mirror at: ${DOCKER_REGISTRY_MIRROR_URL}" - docker_opts+=" --registry-mirror=${DOCKER_REGISTRY_MIRROR_URL}" - fi - - mkdir -p /etc/systemd/system/docker.service.d/ - local kubernetes_conf_dropin="/etc/systemd/system/docker.service.d/00_kubelet.conf" - cat > "${kubernetes_conf_dropin}" < "${kubelet_env_file}" - - # Write the systemd service file for kubelet. - cat </etc/systemd/system/kubelet.service -[Unit] -Description=Kubernetes kubelet -Requires=network-online.target -After=network-online.target - -[Service] -Restart=always -RestartSec=10 -EnvironmentFile=${kubelet_env_file} -ExecStart=${kubelet_bin} \$KUBELET_OPTS - -[Install] -WantedBy=multi-user.target -EOF - - # Flush iptables nat table - iptables -t nat -F || true - - systemctl start kubelet.service -} - -# Create the log file and set its properties. -# -# $1 is the file to create. -function prepare-log-file { - touch $1 - chmod 644 $1 - chown root:root $1 -} - -# Prepares parameters for kube-proxy manifest. -# $1 source path of kube-proxy manifest. -function prepare-kube-proxy-manifest-variables { - local -r src_file=$1; - - remove-salt-config-comments "${src_file}" - - local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig" - local kube_docker_registry="gcr.io/google_containers" - if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then - kube_docker_registry=${KUBE_DOCKER_REGISTRY} - fi - local -r kube_proxy_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-proxy.docker_tag) - local api_servers="--master=https://${KUBERNETES_MASTER_NAME}" - local params="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=2"}" - if [[ -n "${FEATURE_GATES:-}" ]]; then - params+=" --feature-gates=${FEATURE_GATES}" - fi - params+=" --iptables-sync-period=1m --iptables-min-sync-period=10s --ipvs-sync-period=1m --ipvs-min-sync-period=10s" - if [[ -n "${KUBEPROXY_TEST_ARGS:-}" ]]; then - params+=" ${KUBEPROXY_TEST_ARGS}" - fi - local container_env="" - local kube_cache_mutation_detector_env_name="" - local kube_cache_mutation_detector_env_value="" - if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then - container_env="env:" - kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR" - kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" - fi - local pod_priority="" - if [[ "${ENABLE_POD_PRIORITY:-}" == "true" ]]; then - pod_priority="priorityClassName: system-node-critical" - fi - sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file} - sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file} - sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file} - sed -i -e "s@{{params}}@${params}@g" ${src_file} - sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file} - sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file} - sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file} - sed -i -e "s@{{pod_priority}}@${pod_priority}@g" ${src_file} - sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file} - sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file} - sed -i -e "s@{{kubernetes_service_host_env_value}}@${KUBERNETES_MASTER_NAME}@g" ${src_file} - if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then - sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file} - fi - if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then - # Work arounds for https://github.com/coreos/rkt/issues/3245 and https://github.com/coreos/rkt/issues/3264 - # This is an incredibly hacky workaround. It's fragile too. If the kube-proxy command changes too much, this breaks - # TODO, this could be done much better in many other places, such as an - # init script within the container, or even within kube-proxy's code. - local extra_workaround_cmd="ln -sf /proc/self/mounts /etc/mtab; \ - mount -o remount,rw /proc; \ - mount -o remount,rw /proc/sys; \ - mount -o remount,rw /sys; " - sed -i -e "s@-\\s\\+kube-proxy@- ${extra_workaround_cmd} kube-proxy@g" "${src_file}" - fi -} - -# Starts kube-proxy static pod. -function start-kube-proxy { - echo "Start kube-proxy static pod" - prepare-log-file /var/log/kube-proxy.log - local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" - prepare-kube-proxy-manifest-variables "$src_file" - - cp "${src_file}" /etc/kubernetes/manifests -} - -# Replaces the variables in the etcd manifest file with the real values, and then -# copy the file to the manifest dir -# $1: value for variable 'suffix' -# $2: value for variable 'port' -# $3: value for variable 'server_port' -# $4: value for variable 'cpulimit' -# $5: pod name, which should be either etcd or etcd-events -function prepare-etcd-manifest { - local host_name=${ETCD_HOSTNAME:-$(hostname -s)} - local etcd_cluster="" - local cluster_state="new" - local etcd_protocol="http" - local etcd_creds="" - - if [[ -n "${ETCD_CA_KEY:-}" && -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then - etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth " - etcd_protocol="https" - fi - - for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do - etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3" - if [[ -n "${etcd_cluster}" ]]; then - etcd_cluster+="," - cluster_state="existing" - fi - etcd_cluster+="${etcd_host}" - done - - local -r temp_file="/tmp/$5" - cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd.manifest" "${temp_file}" - remove-salt-config-comments "${temp_file}" - sed -i -e "s@{{ *suffix *}}@$1@g" "${temp_file}" - sed -i -e "s@{{ *port *}}@$2@g" "${temp_file}" - sed -i -e "s@{{ *server_port *}}@$3@g" "${temp_file}" - sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${temp_file}" - sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}" - sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${temp_file}" - sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" - sed -i -e "s@{{ *liveness_probe_initial_delay *}}@${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${temp_file}" - # Get default storage backend from manifest file. - local -r default_storage_backend=$(cat "${temp_file}" | \ - grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" | \ - sed -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g") - if [[ -n "${STORAGE_BACKEND:-}" ]]; then - sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}" - else - sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}" - fi - if [[ "${STORAGE_BACKEND:-${default_storage_backend}}" == "etcd3" ]]; then - sed -i -e "s@{{ *quota_bytes *}}@--quota-backend-bytes=4294967296@g" "${temp_file}" - else - sed -i -e "s@{{ *quota_bytes *}}@@g" "${temp_file}" - fi - sed -i -e "s@{{ *cluster_state *}}@$cluster_state@g" "${temp_file}" - if [[ -n "${ETCD_IMAGE:-}" ]]; then - sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@${ETCD_IMAGE}@g" "${temp_file}" - else - sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${temp_file}" - fi - if [[ -n "${ETCD_DOCKER_REPOSITORY:-}" ]]; then - sed -i -e "s@{{ *pillar\.get('etcd_docker_repository', '\(.*\)') *}}@${ETCD_DOCKER_REPOSITORY}@g" "${temp_file}" - else - sed -i -e "s@{{ *pillar\.get('etcd_docker_repository', '\(.*\)') *}}@\1@g" "${temp_file}" - fi - - sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}" - sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}" - if [[ -n "${ETCD_VERSION:-}" ]]; then - sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${temp_file}" - else - sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@\1@g" "${temp_file}" - fi - # Replace the volume host path. - sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${temp_file}" - mv "${temp_file}" /etc/kubernetes/manifests -} - -function start-etcd-empty-dir-cleanup-pod { - cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd-empty-dir-cleanup/etcd-empty-dir-cleanup.yaml" "/etc/kubernetes/manifests" -} - -# Starts etcd server pod (and etcd-events pod if needed). -# More specifically, it prepares dirs and files, sets the variable value -# in the manifests, and copies them to /etc/kubernetes/manifests. -function start-etcd-servers { - echo "Start etcd pods" - if [[ -d /etc/etcd ]]; then - rm -rf /etc/etcd - fi - if [[ -e /etc/default/etcd ]]; then - rm -f /etc/default/etcd - fi - if [[ -e /etc/systemd/system/etcd.service ]]; then - rm -f /etc/systemd/system/etcd.service - fi - if [[ -e /etc/init.d/etcd ]]; then - rm -f /etc/init.d/etcd - fi - prepare-log-file /var/log/etcd.log - prepare-etcd-manifest "" "2379" "2380" "200m" "etcd.manifest" - - prepare-log-file /var/log/etcd-events.log - prepare-etcd-manifest "-events" "4002" "2381" "100m" "etcd-events.manifest" -} - -# Calculates the following variables based on env variables, which will be used -# by the manifests of several kube-master components. -# CLOUD_CONFIG_OPT -# CLOUD_CONFIG_VOLUME -# CLOUD_CONFIG_MOUNT -# DOCKER_REGISTRY -function compute-master-manifest-variables { - CLOUD_CONFIG_OPT="" - CLOUD_CONFIG_VOLUME="" - CLOUD_CONFIG_MOUNT="" - if [[ -f /etc/gce.conf ]]; then - CLOUD_CONFIG_OPT="--cloud-config=/etc/gce.conf" - CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}}," - CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}," - fi - DOCKER_REGISTRY="gcr.io/google_containers" - if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then - DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}" - fi -} - -# A helper function for removing salt configuration and comments from a file. -# This is mainly for preparing a manifest file. -# -# $1: Full path of the file to manipulate -function remove-salt-config-comments { - # Remove salt configuration. - sed -i "/^[ |\t]*{[#|%]/d" $1 - # Remove comments. - sed -i "/^[ |\t]*#/d" $1 -} - -# Starts kubernetes apiserver. -# It prepares the log file, loads the docker image, calculates variables, sets them -# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. -# -# Assumed vars (which are calculated in function compute-master-manifest-variables) -# CLOUD_CONFIG_OPT -# CLOUD_CONFIG_VOLUME -# CLOUD_CONFIG_MOUNT -# DOCKER_REGISTRY -function start-kube-apiserver { - echo "Start kubernetes api-server" - prepare-log-file /var/log/kube-apiserver.log - prepare-log-file /var/log/kube-apiserver-audit.log - - # Calculate variables and assemble the command line. - local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" - params+=" --address=127.0.0.1" - params+=" --allow-privileged=true" - params+=" --cloud-provider=gce" - params+=" --client-ca-file=/etc/srv/kubernetes/ca.crt" - params+=" --etcd-servers=http://127.0.0.1:2379" - params+=" --etcd-servers-overrides=/events#http://127.0.0.1:4002" - params+=" --secure-port=443" - params+=" --tls-cert-file=/etc/srv/kubernetes/server.cert" - params+=" --tls-private-key-file=/etc/srv/kubernetes/server.key" - params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv" - params+=" --enable-aggregator-routing=true" - if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then - params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv" - fi - if [[ -n "${STORAGE_BACKEND:-}" ]]; then - params+=" --storage-backend=${STORAGE_BACKEND}" - fi - if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then - params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" - fi - if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC:-}" ]]; then - params+=" --request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT_SEC}s" - fi - if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then - params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" - fi - if [[ -n "${NUM_NODES:-}" ]]; then - # If the cluster is large, increase max-requests-inflight limit in apiserver. - if [[ "${NUM_NODES}" -ge 1000 ]]; then - params+=" --max-requests-inflight=1500 --max-mutating-requests-inflight=500" - fi - # Set amount of memory available for apiserver based on number of nodes. - # TODO: Once we start setting proper requests and limits for apiserver - # we should reuse the same logic here instead of current heuristic. - params+=" --target-ram-mb=$((${NUM_NODES} * 60))" - fi - if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then - params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" - fi - if [[ -n "${ETCD_QUORUM_READ:-}" ]]; then - params+=" --etcd-quorum-read=${ETCD_QUORUM_READ}" - fi - - if [[ "${ENABLE_APISERVER_BASIC_AUDIT:-}" == "true" ]]; then - # We currently only support enabling with a fixed path and with built-in log - # rotation "disabled" (large value) so it behaves like kube-apiserver.log. - # External log rotation should be set up the same as for kube-apiserver.log. - params+=" --audit-log-path=/var/log/kube-apiserver-audit.log" - params+=" --audit-log-maxage=0" - params+=" --audit-log-maxbackup=0" - # Lumberjack doesn't offer any way to disable size-based rotation. It also - # has an in-memory counter that doesn't notice if you truncate the file. - # 2000000000 (in MiB) is a large number that fits in 31 bits. If the log - # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver - # never restarts. Please manually restart apiserver before this time. - params+=" --audit-log-maxsize=2000000000" - fi - - if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then - params+=" --enable-logs-handler=false" - fi - - local admission_controller_config_mount="" - local admission_controller_config_volume="" - local image_policy_webhook_config_mount="" - local image_policy_webhook_config_volume="" - if [[ -n "${ADMISSION_CONTROL:-}" ]]; then - params+=" --admission-control=${ADMISSION_CONTROL}" - if [[ ${ADMISSION_CONTROL} == *"ImagePolicyWebhook"* ]]; then - params+=" --admission-control-config-file=/etc/admission_controller.config" - # Mount the file to configure admission controllers if ImagePolicyWebhook is set. - admission_controller_config_mount="{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"/etc/admission_controller.config\", \"readOnly\": false}," - admission_controller_config_volume="{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"/etc/admission_controller.config\", \"type\": \"FileOrCreate\"}}," - # Mount the file to configure the ImagePolicyWebhook's webhook. - image_policy_webhook_config_mount="{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," - image_policy_webhook_config_volume="{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\", \"type\": \"FileOrCreate\"}}," - fi - fi - - if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then - params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" - fi - if [[ -n "${RUNTIME_CONFIG:-}" ]]; then - params+=" --runtime-config=${RUNTIME_CONFIG}" - fi - if [[ -n "${FEATURE_GATES:-}" ]]; then - params+=" --feature-gates=${FEATURE_GATES}" - fi - if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then - local -r vm_external_ip=$(curl --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip") - if [[ -n "${PROXY_SSH_USER:-}" ]]; then - params+=" --advertise-address=${vm_external_ip}" - params+=" --ssh-user=${PROXY_SSH_USER}" - params+=" --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile" - else - params+=" --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - fi - elif [ -n "${MASTER_ADVERTISE_ADDRESS:-}" ]; then - params="${params} --advertise-address=${MASTER_ADVERTISE_ADDRESS}" - fi - - local webhook_authn_config_mount="" - local webhook_authn_config_volume="" - if [[ -n "${GCP_AUTHN_URL:-}" ]]; then - params+=" --authentication-token-webhook-config-file=/etc/gcp_authn.config" - webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false}," - webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\", \"type\": \"FileOrCreate\"}}," - fi - - local authorization_mode="RBAC" - local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" - - # Enable ABAC mode unless the user explicitly opts out with ENABLE_LEGACY_ABAC=false - if [[ "${ENABLE_LEGACY_ABAC:-}" != "false" ]]; then - echo "Warning: Enabling legacy ABAC policy. All service accounts will have superuser API access. Set ENABLE_LEGACY_ABAC=false to disable this." - # Create the ABAC file if it doesn't exist yet, or if we have a KUBE_USER set (to ensure the right user is given permissions) - if [[ -n "${KUBE_USER:-}" || ! -e /etc/srv/kubernetes/abac-authz-policy.jsonl ]]; then - local -r abac_policy_json="${src_dir}/abac-authz-policy.jsonl" - remove-salt-config-comments "${abac_policy_json}" - if [[ -n "${KUBE_USER:-}" ]]; then - sed -i -e "s/{{kube_user}}/${KUBE_USER}/g" "${abac_policy_json}" - else - sed -i -e "/{{kube_user}}/d" "${abac_policy_json}" - fi - cp "${abac_policy_json}" /etc/srv/kubernetes/ - fi - - params+=" --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl" - authorization_mode+=",ABAC" - fi - - local webhook_config_mount="" - local webhook_config_volume="" - if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then - authorization_mode+=",Webhook" - params+=" --authorization-webhook-config-file=/etc/gcp_authz.config" - webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false}," - webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\", \"type\": \"FileOrCreate\"}}," - fi - params+=" --authorization-mode=${authorization_mode}" - - local container_env="" - if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then - container_env="\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" - fi - if [[ -n "${ENABLE_PATCH_CONVERSION_DETECTOR:-}" ]]; then - if [[ -n "${container_env}" ]]; then - container_env="${container_env}, " - fi - container_env="\"name\": \"KUBE_PATCH_CONVERSION_DETECTOR\", \"value\": \"${ENABLE_PATCH_CONVERSION_DETECTOR}\"" - fi - if [[ -n "${container_env}" ]]; then - container_env="\"env\":[{${container_env}}]," - fi - - src_file="${src_dir}/kube-apiserver.manifest" - remove-salt-config-comments "${src_file}" - # Evaluate variables. - local -r kube_apiserver_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-apiserver.docker_tag) - sed -i -e "s@{{params}}@${params}@g" "${src_file}" - sed -i -e "s@{{container_env}}@${container_env}@g" "${src_file}" - sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" - sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}" - sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" - sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}" - sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" - sed -i -e "s@{{liveness_probe_initial_delay}}@${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${src_file}" - sed -i -e "s@{{secure_port}}@443@g" "${src_file}" - sed -i -e "s@{{secure_port}}@8080@g" "${src_file}" - sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" - sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" - sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}" - sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}" - sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}" - sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" - sed -i -e "s@{{admission_controller_config_mount}}@${admission_controller_config_mount}@g" "${src_file}" - sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}" - sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}" - sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}" - cp "${src_file}" /etc/kubernetes/manifests -} - -# Starts kubernetes controller manager. -# It prepares the log file, loads the docker image, calculates variables, sets them -# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. -# -# Assumed vars (which are calculated in function compute-master-manifest-variables) -# CLOUD_CONFIG_OPT -# CLOUD_CONFIG_VOLUME -# CLOUD_CONFIG_MOUNT -# DOCKER_REGISTRY -function start-kube-controller-manager { - echo "Start kubernetes controller-manager" - create-kubecontrollermanager-kubeconfig - prepare-log-file /var/log/kube-controller-manager.log - # Calculate variables and assemble the command line. - local params="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=2"} ${CONTROLLER_MANAGER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" - params+=" --use-service-account-credentials" - params+=" --cloud-provider=gce" - params+=" --kubeconfig=/etc/srv/kubernetes/kube-controller-manager/kubeconfig" - params+=" --root-ca-file=/etc/srv/kubernetes/ca.crt" - params+=" --service-account-private-key-file=/etc/srv/kubernetes/server.key" - if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then - params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" - fi - if [[ -n "${INSTANCE_PREFIX:-}" ]]; then - params+=" --cluster-name=${INSTANCE_PREFIX}" - fi - if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then - params+=" --cluster-cidr=${CLUSTER_IP_RANGE}" - fi - if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then - params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" - fi - if [[ -n "${CONCURRENT_SERVICE_SYNCS:-}" ]]; then - params+=" --concurrent-service-syncs=${CONCURRENT_SERVICE_SYNCS}" - fi - if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then - params+=" --allocate-node-cidrs=true" - elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then - params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}" - fi - if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then - params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" - fi - if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then - params+=" --cidr-allocator-type=CloudAllocator" - params+=" --configure-cloud-routes=false" - fi - if [[ -n "${FEATURE_GATES:-}" ]]; then - params+=" --feature-gates=${FEATURE_GATES}" - fi - local -r kube_rc_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-controller-manager.docker_tag) - local container_env="" - if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then - container_env="\"env\":[{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}]," - fi - - local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest" - remove-salt-config-comments "${src_file}" - # Evaluate variables. - sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" - sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube-controller-manager_docker_tag'\]}}@${kube_rc_docker_tag}@g" "${src_file}" - sed -i -e "s@{{params}}@${params}@g" "${src_file}" - sed -i -e "s@{{container_env}}@${container_env}@g" "${src_file}" - sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" - sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" - sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" - sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" - cp "${src_file}" /etc/kubernetes/manifests -} - -# Starts kubernetes scheduler. -# It prepares the log file, loads the docker image, calculates variables, sets them -# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. -# -# Assumed vars (which are calculated in compute-master-manifest-variables) -# DOCKER_REGISTRY -function start-kube-scheduler { - echo "Start kubernetes scheduler" - create-kubescheduler-kubeconfig - prepare-log-file /var/log/kube-scheduler.log - - # Calculate variables and set them in the manifest. - params="${SCHEDULER_TEST_LOG_LEVEL:-"--v=2"} ${SCHEDULER_TEST_ARGS:-}" - params+=" --kubeconfig=/etc/srv/kubernetes/kube-scheduler/kubeconfig" - if [[ -n "${FEATURE_GATES:-}" ]]; then - params+=" --feature-gates=${FEATURE_GATES}" - fi - if [[ -n "${SCHEDULING_ALGORITHM_PROVIDER:-}" ]]; then - params+=" --algorithm-provider=${SCHEDULING_ALGORITHM_PROVIDER}" - fi - local -r kube_scheduler_docker_tag=$(cat "${KUBE_HOME}/kube-docker-files/kube-scheduler.docker_tag") - - # Remove salt comments and replace variables with values. - local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-scheduler.manifest" - remove-salt-config-comments "${src_file}" - - sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" - sed -i -e "s@{{params}}@${params}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" "${src_file}" - cp "${src_file}" /etc/kubernetes/manifests -} - -# Starts cluster autoscaler. -# Assumed vars (which are calculated in function compute-master-manifest-variables) -# CLOUD_CONFIG_OPT -# CLOUD_CONFIG_VOLUME -# CLOUD_CONFIG_MOUNT -function start-cluster-autoscaler { - if [[ "${ENABLE_CLUSTER_AUTOSCALER:-}" == "true" ]]; then - echo "Start kubernetes cluster autoscaler" - prepare-log-file /var/log/cluster-autoscaler.log - - # Remove salt comments and replace variables with values - local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/cluster-autoscaler.manifest" - remove-salt-config-comments "${src_file}" - - local params="${AUTOSCALER_MIG_CONFIG} ${CLOUD_CONFIG_OPT} ${AUTOSCALER_EXPANDER_CONFIG:---expander=price}" - sed -i -e "s@{{params}}@${params}@g" "${src_file}" - sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" - sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" - sed -i -e "s@{%.*%}@@g" "${src_file}" - - cp "${src_file}" /etc/kubernetes/manifests - fi -} - -# A helper function for copying addon manifests and set dir/files -# permissions. -# -# $1: addon category under /etc/kubernetes -# $2: manifest source dir -function setup-addon-manifests { - local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/$2" - local -r dst_dir="/etc/kubernetes/$1/$2" - if [[ ! -d "${dst_dir}" ]]; then - mkdir -p "${dst_dir}" - fi - local files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml") - if [[ -n "${files}" ]]; then - cp "${src_dir}/"*.yaml "${dst_dir}" - fi - files=$(find "${src_dir}" -maxdepth 1 -name "*.json") - if [[ -n "${files}" ]]; then - cp "${src_dir}/"*.json "${dst_dir}" - fi - files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml.in") - if [[ -n "${files}" ]]; then - cp "${src_dir}/"*.yaml.in "${dst_dir}" - fi - chown -R root:root "${dst_dir}" - chmod 755 "${dst_dir}" - chmod 644 "${dst_dir}"/* -} - -# Updates parameters in yaml file for prometheus-to-sd configuration, or -# removes component if it is disabled. -function update-prometheus-to-sd-parameters { - if [[ "${ENABLE_PROMETHEUS_TO_SD:-}" == "true" ]]; then - sed -i -e "s@{{ *prometheus_to_sd_prefix *}}@${PROMETHEUS_TO_SD_PREFIX}@g" "$1" - sed -i -e "s@{{ *prometheus_to_sd_endpoint *}}@${PROMETHEUS_TO_SD_ENDPOINT}@g" "$1" - else - # Removes all lines between two patterns (throws away prometheus-to-sd) - sed -i -e "/# BEGIN_PROMETHEUS_TO_SD/,/# END_PROMETHEUS_TO_SD/d" "$1" - fi -} - -# Sets up the manifests of coreDNS for k8s addons. -function setup-coredns-manifest { - local -r coredns_file="${dst_dir}/dns/coredns.yaml" - mv "${dst_dir}/dns/coredns.yaml.in" "${coredns_file}" - # Replace the salt configurations with variable values. - sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${coredns_file}" - sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${coredns_file}" - sed -i -e "s@{{ *pillar\['service_cluster_ip_range'\] *}}@${SERVICE_CLUSTER_IP_RANGE}@g" "${coredns_file}" -} - -# Sets up the manifests of kube-dns for k8s addons. -function setup-kube-dns-manifest { - local -r kubedns_file="${dst_dir}/dns/kube-dns.yaml" - mv "${dst_dir}/dns/kube-dns.yaml.in" "${kubedns_file}" - if [ -n "${CUSTOM_KUBE_DNS_YAML:-}" ]; then - # Replace with custom GKE kube-dns deployment. - cat > "${kubedns_file}" < "$src_dir/kube-proxy/kube-proxy-ds.yaml" < /etc/systemd/system/rkt-api.service </dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) -KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - -# KUBERNETES_CONTAINER_RUNTIME is set by the `kube-env` file, but it's a bit of a mouthful -if [[ "${CONTAINER_RUNTIME:-}" == "" ]]; then - CONTAINER_RUNTIME="${KUBERNETES_CONTAINER_RUNTIME:-docker}" -fi - -create-dirs -ensure-local-ssds -if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then - mount-master-pd - create-master-auth - create-master-kubelet-auth - create-master-etcd-auth -else - create-kubelet-kubeconfig "https://${KUBERNETES_MASTER_NAME}" - if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then - create-kubeproxy-user-kubeconfig - fi -fi - -if [[ "${KUBERNETES_CONTAINER_RUNTIME:-}" == "rkt" ]]; then - systemctl stop docker - systemctl disable docker - setup-rkt - install-docker2aci - create-kube-controller-manager-dirs -else - configure-docker-daemon -fi - -load-docker-images -start-kubelet - -if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then - compute-master-manifest-variables - start-etcd-servers - start-etcd-empty-dir-cleanup-pod - start-kube-apiserver - start-kube-controller-manager - start-kube-scheduler - start-kube-addons - start-cluster-autoscaler - start-lb-controller - start-rescheduler -else - if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then - start-kube-proxy - fi - # Kube-registry-proxy. - if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then - start-kube-registry-proxy - fi - if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then - start-image-puller - fi -fi -echo "Done for the configuration for kubernetes" diff --git a/cluster/gce/container-linux/configure.sh b/cluster/gce/container-linux/configure.sh deleted file mode 100755 index 16dcf27a044..00000000000 --- a/cluster/gce/container-linux/configure.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/bash - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -# Use --retry-connrefused opt only if it's supported by curl. -CURL_RETRY_CONNREFUSED="" -if curl --help | grep -q -- '--retry-connrefused'; then - CURL_RETRY_CONNREFUSED='--retry-connrefused' -fi - -function download-kube-env { - # Fetch kube-env from GCE metadata server. - local -r tmp_kube_env="/tmp/kube-env.yaml" - curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \ - -H "X-Google-Metadata-Request: True" \ - -o "${tmp_kube_env}" \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env - # Convert the yaml format file into a shell-style file. - sed 's/: /=/' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env" - rm -f "${tmp_kube_env}" -} - -function validate-hash { - local -r file="$1" - local -r expected="$2" - - actual=$(sha1sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - - -# Retry a download until we get it. Takes a hash and a set of URLs. -# -# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. -# $2+ are the URLs to download. -function download-or-bust { - local -r hash="$1" - shift 1 - - local -r urls=( $* ) - while true; do - for url in "${urls[@]}"; do - local file="${url##*/}" - rm -f "${file}" - if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 ${CURL_RETRY_CONNREFUSED} "${url}"; then - echo "== Failed to download ${url}. Retrying. ==" - elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done -} - -function split-commas { - echo $1 | tr "," "\n" -} - -# Downloads kubernetes binaries and kube-system manifest tarball, unpacks them, -# and places them into suitable directories. Files are placed in /opt/kubernetes. -function install-kube-binary-config { - cd "${KUBE_HOME}" - local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") ) - local -r server_binary_tar="${server_binary_tar_urls[0]##*/}" - if [[ -n "${SERVER_BINARY_TAR_HASH:-}" ]]; then - local -r server_binary_tar_hash="${SERVER_BINARY_TAR_HASH}" - else - echo "Downloading binary release sha1 (not found in env)" - download-or-bust "" "${server_binary_tar_urls[@]/.tar.gz/.tar.gz.sha1}" - local -r server_binary_tar_hash=$(cat "${server_binary_tar}.sha1") - fi - echo "Downloading binary release tar" - download-or-bust "${server_binary_tar_hash}" "${server_binary_tar_urls[@]}" - tar xzf "${KUBE_HOME}/${server_binary_tar}" -C "${KUBE_HOME}" --overwrite - # Copy docker_tag and image files to ${KUBE_HOME}/kube-docker-files. - src_dir="${KUBE_HOME}/kubernetes/server/bin" - dst_dir="${KUBE_HOME}/kube-docker-files" - mkdir -p "${dst_dir}" - cp "${src_dir}/"*.docker_tag "${dst_dir}" - if [[ "${KUBERNETES_MASTER:-}" == "false" ]]; then - cp "${src_dir}/kube-proxy.tar" "${dst_dir}" - else - cp "${src_dir}/kube-apiserver.tar" "${dst_dir}" - cp "${src_dir}/kube-controller-manager.tar" "${dst_dir}" - cp "${src_dir}/kube-scheduler.tar" "${dst_dir}" - cp -r "${KUBE_HOME}/kubernetes/addons" "${dst_dir}" - fi - local -r kube_bin="${KUBE_HOME}/bin" - mv "${src_dir}/kubelet" "${kube_bin}" - mv "${src_dir}/kubectl" "${kube_bin}" - - if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]] || \ - [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then - local -r cni_version="v0.6.0" - local -r cni_tar="cni-plugins-amd64-${cni_version}.tgz" - local -r cni_sha1="d595d3ded6499a64e8dac02466e2f5f2ce257c9f" - download-or-bust "${cni_sha1}" "https://storage.googleapis.com/kubernetes-release/network-plugins/${cni_tar}" - local -r cni_dir="${KUBE_HOME}/cni" - mkdir -p "${cni_dir}/bin" - tar xzf "${KUBE_HOME}/${cni_tar}" -C "${cni_dir}/bin" --overwrite - mv "${cni_dir}/bin"/* "${kube_bin}" - rmdir "${cni_dir}/bin" - rm -f "${KUBE_HOME}/${cni_tar}" - fi - - mv "${KUBE_HOME}/kubernetes/LICENSES" "${KUBE_HOME}" - mv "${KUBE_HOME}/kubernetes/kubernetes-src.tar.gz" "${KUBE_HOME}" - - # Put kube-system pods manifests in ${KUBE_HOME}/kube-manifests/. - dst_dir="${KUBE_HOME}/kube-manifests" - mkdir -p "${dst_dir}" - local -r manifests_tar_urls=( $(split-commas "${KUBE_MANIFESTS_TAR_URL}") ) - local -r manifests_tar="${manifests_tar_urls[0]##*/}" - if [ -n "${KUBE_MANIFESTS_TAR_HASH:-}" ]; then - local -r manifests_tar_hash="${KUBE_MANIFESTS_TAR_HASH}" - else - echo "Downloading k8s manifests sha1 (not found in env)" - download-or-bust "" "${manifests_tar_urls[@]/.tar.gz/.tar.gz.sha1}" - local -r manifests_tar_hash=$(cat "${manifests_tar}.sha1") - fi - echo "Downloading k8s manifests tar" - download-or-bust "${manifests_tar_hash}" "${manifests_tar_urls[@]}" - tar xzf "${KUBE_HOME}/${manifests_tar}" -C "${dst_dir}" --overwrite - local -r kube_addon_registry="${KUBE_ADDON_REGISTRY:-gcr.io/google_containers}" - if [[ "${kube_addon_registry}" != "gcr.io/google_containers" ]]; then - find "${dst_dir}" -name \*.yaml -or -name \*.yaml.in | \ - xargs sed -ri "s@(image:\s.*)gcr.io/google_containers@\1${kube_addon_registry}@" - find "${dst_dir}" -name \*.manifest -or -name \*.json | \ - xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@" - fi - cp "${dst_dir}/kubernetes/gci-trusty/container-linux-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh" - chmod -R 755 "${kube_bin}" - - # Clean up. - rm -rf "${KUBE_HOME}/kubernetes" - rm -f "${KUBE_HOME}/${server_binary_tar}" - rm -f "${KUBE_HOME}/${server_binary_tar}.sha1" - rm -f "${KUBE_HOME}/${manifests_tar}" - rm -f "${KUBE_HOME}/${manifests_tar}.sha1" -} - -######### Main Function ########## -echo "Start to install kubernetes files" -KUBE_HOME="/opt/kubernetes" -mkdir -p "${KUBE_HOME}" -download-kube-env -source "${KUBE_HOME}/kube-env" -install-kube-binary-config -echo "Done for installing kubernetes files" - -# On Container Linux, the hosts is in /usr/share/baselayout/hosts -# So we need to manually populdate the hosts file here on gce. -echo "127.0.0.1 localhost" >> /etc/hosts -echo "::1 localhost" >> /etc/hosts - -echo "Configuring hostname" -hostnamectl set-hostname $(hostname | cut -f1 -d.) diff --git a/cluster/gce/container-linux/health-monitor.sh b/cluster/gce/container-linux/health-monitor.sh deleted file mode 100644 index 6e8f1b03b12..00000000000 --- a/cluster/gce/container-linux/health-monitor.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is for master and node instance health monitoring, which is -# packed in kube-manifest tarball. It is executed through a systemd service -# in cluster/gce/gci/.yaml. The env variables come from an env -# file provided by the systemd service. - -set -o nounset -set -o pipefail - -# We simply kill the process when there is a failure. Another systemd service will -# automatically restart the process. -function docker_monitoring { - while [ 1 ]; do - if ! timeout 60 docker ps > /dev/null; then - echo "Docker daemon failed!" - pkill docker - # Wait for a while, as we don't want to kill it again before it is really up. - sleep 30 - else - sleep "${SLEEP_SECONDS}" - fi - done -} - -function kubelet_monitoring { - echo "Wait for 2 minutes for kubelet to be fuctional" - # TODO(andyzheng0831): replace it with a more reliable method if possible. - sleep 120 - local -r max_seconds=10 - local output="" - while [ 1 ]; do - if ! output=$(curl --insecure -m "${max_seconds}" -f -s -S https://127.0.0.1:${KUBELET_PORT:-10250}/healthz 2>&1); then - # Print the response and/or errors. - echo $output - echo "Kubelet is unhealthy!" - pkill kubelet - # Wait for a while, as we don't want to kill it again before it is really up. - sleep 60 - else - sleep "${SLEEP_SECONDS}" - fi - done -} - - -############## Main Function ################ -if [[ "$#" -ne 1 ]]; then - echo "Usage: health-monitor.sh " - exit 1 -fi - -KUBE_ENV="/home/kubernetes/kube-env" -if [[ ! -e "${KUBE_ENV}" ]]; then - echo "The ${KUBE_ENV} file does not exist!! Terminate health monitoring" - exit 1 -fi - -SLEEP_SECONDS=10 -component=$1 -echo "Start kubernetes health monitoring for ${component}" -source "${KUBE_ENV}" -if [[ "${component}" == "docker" ]]; then - docker_monitoring -elif [[ "${component}" == "kubelet" ]]; then - kubelet_monitoring -else - echo "Health monitoring for component "${component}" is not supported!" -fi diff --git a/cluster/gce/container-linux/helper.sh b/cluster/gce/container-linux/helper.sh deleted file mode 100755 index ddd45791266..00000000000 --- a/cluster/gce/container-linux/helper.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A library of helper functions and constants for the Container Linux distro. - -# This file intentionally left blank diff --git a/cluster/gce/container-linux/master-helper.sh b/cluster/gce/container-linux/master-helper.sh deleted file mode 100755 index 3cd3ee3a3e5..00000000000 --- a/cluster/gce/container-linux/master-helper.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A library of helper functions and constant for the Container Linux distro. -source "${KUBE_ROOT}/cluster/gce/container-linux/helper.sh" - -# create-master-instance creates the master instance. If called with -# an argument, the argument is used as the name to a reserved IP -# address for the master. (In the case of upgrade/repair, we re-use -# the same IP.) -# -# It requires a whole slew of assumed variables, partially due to to -# the call to write-master-env. Listing them would be rather -# futile. Instead, we list the required calls to ensure any additional -# -# variables are set: -# ensure-temp-dir -# detect-project -# get-bearer-token -function create-master-instance { - local address="" - [[ -n ${1:-} ]] && address="${1}" - - write-master-env - create-master-instance-internal "${MASTER_NAME}" "${address}" -} - -function replicate-master-instance() { - local existing_master_zone="${1}" - local existing_master_name="${2}" - local existing_master_replicas="${3}" - - local kube_env="$(get-metadata "${existing_master_zone}" "${existing_master_name}" kube-env)" - # Substitute INITIAL_ETCD_CLUSTER to enable etcd clustering. - kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER")" - kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER: '${existing_master_replicas},${REPLICA_NAME}'")" - ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" | sed "s/^.*: '//" | sed "s/'$//")" - ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" | sed "s/^.*: '//" | sed "s/'$//")" - - create-etcd-certs "${REPLICA_NAME}" "${ETCD_CA_CERT}" "${ETCD_CA_KEY}" - - kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")" - kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")" - kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_CERT")" - kube_env="$(echo -e "${kube_env}\nETCD_PEER_CERT: '${ETCD_PEER_CERT_BASE64}'")" - - echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml - get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt" - - create-master-instance-internal "${REPLICA_NAME}" -} - - -function create-master-instance-internal() { - local gcloud="gcloud" - local retries=5 - if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then - gcloud="gcloud beta" - fi - - local -r master_name="${1}" - local -r address="${2:-}" - - local preemptible_master="" - if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then - preemptible_master="--preemptible --maintenance-policy TERMINATE" - fi - - local network=$(make-gcloud-network-argument \ - "${NETWORK_PROJECT}" "${REGION}" "${NETWORK}" "${SUBNETWORK:-}" \ - "${address:-}" "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SIZE:-}") - - local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml" - metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/container-linux/master.yaml" - metadata="${metadata},configure-sh=${KUBE_ROOT}/cluster/gce/container-linux/configure.sh" - metadata="${metadata},cluster-name=${KUBE_TEMP}/cluster-name.txt" - - local disk="name=${master_name}-pd" - disk="${disk},device-name=master-pd" - disk="${disk},mode=rw" - disk="${disk},boot=no" - disk="${disk},auto-delete=no" - - for attempt in $(seq 1 ${retries}); do - if result=$(${gcloud} compute instances create "${master_name}" \ - --project "${PROJECT}" \ - --zone "${ZONE}" \ - --machine-type "${MASTER_SIZE}" \ - --image-project="${MASTER_IMAGE_PROJECT}" \ - --image "${MASTER_IMAGE}" \ - --tags "${MASTER_TAG}" \ - --scopes "storage-ro,compute-rw,monitoring,logging-write" \ - --metadata-from-file "${metadata}" \ - --disk "${disk}" \ - --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \ - ${MASTER_MIN_CPU_ARCHITECTURE:+"--min-cpu-platform=${MASTER_MIN_CPU_ARCHITECTURE}"} \ - ${preemptible_master} \ - ${network} 2>&1); then - echo "${result}" >&2 - return 0 - else - echo "${result}" >&2 - if [[ ! "${result}" =~ "try again later" ]]; then - echo "Failed to create master instance due to non-retryable error" >&2 - return 1 - fi - sleep 10 - fi - done - - echo "Failed to create master instance despite ${retries} attempts" >&2 - return 1 -} - -function get-metadata() { - local zone="${1}" - local name="${2}" - local key="${3}" - - local metadata_url="http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}" - - gcloud compute ssh "${name}" \ - --project "${PROJECT}" \ - --zone "${zone}" \ - --command "curl '${metadata_url}' -H 'Metadata-Flavor: Google'" 2>/dev/null -} diff --git a/cluster/gce/container-linux/master.yaml b/cluster/gce/container-linux/master.yaml deleted file mode 100644 index 444d3042739..00000000000 --- a/cluster/gce/container-linux/master.yaml +++ /dev/null @@ -1,57 +0,0 @@ -#cloud-config - -coreos: - update: - reboot-strategy: off - units: - - name: locksmithd.service - mask: true - - name: kube-master-installation.service - command: start - content: | - [Unit] - Description=Download and install k8s binaries and configurations - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /opt/kubernetes/bin - # Use --retry-connrefused opt only if it's supported by curl. - ExecStartPre=/bin/bash -c 'OPT=""; if curl --help | grep -q -- "--retry-connrefused"; then OPT="--retry-connrefused"; fi; /usr/bin/curl --fail --retry 5 --retry-delay 3 $OPT --silent --show-error -H "X-Google-Metadata-Request: True" -o /opt/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh' - ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure.sh - ExecStart=/opt/kubernetes/bin/configure.sh - - [Install] - WantedBy=kubernetes.target - - name: kube-master-configuration.service - command: start - content: | - [Unit] - Description=Configure kubernetes master - After=kube-master-installation.service - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure-helper.sh - ExecStart=/opt/kubernetes/bin/configure-helper.sh - - [Install] - WantedBy=kubernetes.target - - name: kubernetes.target - enable: true - command: start - content: | - [Unit] - Description=Kubernetes - - [Install] - WantedBy=multi-user.target - - name: docker.service - drop-ins: - - name: "use-cgroupfs-driver.conf" - # This is required for setting cgroup parent in the current ~1.4 per-pod cgroup impl - content: | - [Service] - Environment="DOCKER_CGROUPS=--exec-opt native.cgroupdriver=" diff --git a/cluster/gce/container-linux/node-helper.sh b/cluster/gce/container-linux/node-helper.sh deleted file mode 100755 index c2432c5b0ea..00000000000 --- a/cluster/gce/container-linux/node-helper.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A library of helper functions and constant for the Container Linux distro. -source "${KUBE_ROOT}/cluster/gce/container-linux/helper.sh" - -function get-node-instance-metadata { - local metadata="" - metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml," - metadata+="user-data=${KUBE_ROOT}/cluster/gce/container-linux/node.yaml," - metadata+="configure-sh=${KUBE_ROOT}/cluster/gce/container-linux/configure.sh," - metadata+="cluster-name=${KUBE_TEMP}/cluster-name.txt" - echo "${metadata}" -} - -# $1: template name (required). -function create-node-instance-template { - local template_name="$1" - - create-node-template "$template_name" "${scope_flags[*]}" "$(get-node-instance-metadata)" - # TODO(euank): We should include update-strategy here. We should also switch to ignition -} diff --git a/cluster/gce/container-linux/node.yaml b/cluster/gce/container-linux/node.yaml deleted file mode 100644 index 9886679cd78..00000000000 --- a/cluster/gce/container-linux/node.yaml +++ /dev/null @@ -1,57 +0,0 @@ -#cloud-config - -coreos: - update: - reboot-strategy: off - units: - - name: locksmithd.service - mask: true - - name: kube-node-installation.service - command: start - content: | - [Unit] - Description=Download and install k8s binaries and configurations - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /opt/kubernetes/bin - # Use --retry-connrefused opt only if it's supported by curl. - ExecStartPre=/bin/bash -c 'OPT=""; if curl --help | grep -q -- "--retry-connrefused"; then OPT="--retry-connrefused"; fi; /usr/bin/curl --fail --retry 5 --retry-delay 3 $OPT --silent --show-error -H "X-Google-Metadata-Request: True" -o /opt/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh' - ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure.sh - ExecStart=/opt/kubernetes/bin/configure.sh - - [Install] - WantedBy=kubernetes.target - - name: kube-node-configuration.service - command: start - content: | - [Unit] - Description=Configure kubernetes master - After=kube-node-installation.service - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure-helper.sh - ExecStart=/opt/kubernetes/bin/configure-helper.sh - - [Install] - WantedBy=kubernetes.target - - name: kubernetes.target - enable: true - command: start - content: | - [Unit] - Description=Kubernetes - - [Install] - WantedBy=multi-user.target - - name: docker.service - drop-ins: - - name: "use-cgroupfs-driver.conf" - # This is required for setting cgroup parent in the current ~1.4 per-pod cgroup impl - content: | - [Service] - Environment="DOCKER_CGROUPS=--exec-opt native.cgroupdriver=" diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 46178dba693..5c4f48d12cd 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -25,14 +25,14 @@ source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/common.sh" source "${KUBE_ROOT}/hack/lib/util.sh" -if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "container-linux" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then +if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh" else echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2 exit 1 fi -if [[ "${MASTER_OS_DISTRIBUTION}" == "container-linux" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then +if [[ "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh" else echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2