mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-26 13:07:07 +00:00
Revert "Refactored kubemark into cloud-provider independent code and GCE specific code"
This commit is contained in:
Wojciech Tyczynski
GitHub
parent
bcc1dc5c15
commit
fbd5c7c380
@ -1,6 +1,6 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
# Copyright 2017 The Kubernetes Authors.
|
# Copyright 2015 The Kubernetes Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
@ -15,6 +15,5 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
|
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
|
||||||
source ${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh
|
source ${KUBE_ROOT}/cluster/gce/util.sh
|
||||||
source ${KUBE_ROOT}/cluster/${CLOUD_PROVIDER}/util.sh
|
source ${KUBE_ROOT}/cluster/kubemark/config-default.sh
|
||||||
source ${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh
|
|
||||||
|
|||||||
@ -1,17 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Copyright 2017 The Kubernetes Authors.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
CLOUD_PROVIDER="${CLOUD_PROVIDER:-gce}"
|
|
||||||
71
test/kubemark/common.sh
Normal file
71
test/kubemark/common.sh
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Copyright 2015 The Kubernetes Authors.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
source "${KUBE_ROOT}/cluster/kubemark/config-default.sh"
|
||||||
|
source "${KUBE_ROOT}/cluster/kubemark/util.sh"
|
||||||
|
source "${KUBE_ROOT}/cluster/lib/util.sh"
|
||||||
|
|
||||||
|
# hack/lib/init.sh will ovewrite ETCD_VERSION if this is unset
|
||||||
|
# what what is default in hack/lib/etcd.sh
|
||||||
|
# To avoid it, if it is empty, we set it to 'avoid-overwrite' and
|
||||||
|
# clean it after that.
|
||||||
|
if [ -z "${ETCD_VERSION:-}" ]; then
|
||||||
|
ETCD_VERSION="avoid-overwrite"
|
||||||
|
fi
|
||||||
|
source "${KUBE_ROOT}/hack/lib/init.sh"
|
||||||
|
if [ "${ETCD_VERSION:-}" == "avoid-overwrite" ]; then
|
||||||
|
ETCD_VERSION=""
|
||||||
|
fi
|
||||||
|
|
||||||
|
detect-project &> /dev/null
|
||||||
|
export PROJECT
|
||||||
|
find-release-tars
|
||||||
|
|
||||||
|
MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master"
|
||||||
|
MASTER_TAG="kubemark-master"
|
||||||
|
EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store"
|
||||||
|
|
||||||
|
RETRIES=3
|
||||||
|
|
||||||
|
export KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
|
||||||
|
export KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
|
||||||
|
export RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
|
||||||
|
|
||||||
|
# Runs gcloud compute command with the given parameters. Up to $RETRIES will be made
|
||||||
|
# to execute the command.
|
||||||
|
# arguments:
|
||||||
|
# $@: all stuff that goes after 'gcloud compute '
|
||||||
|
function run-gcloud-compute-with-retries {
|
||||||
|
echo "" > /tmp/gcloud_retries
|
||||||
|
for attempt in $(seq 1 ${RETRIES}); do
|
||||||
|
if ! gcloud compute $@ &> /tmp/gcloud_retries; then
|
||||||
|
if [[ $(grep -c "already exists" /tmp/gcloud_retries) -gt 0 ]]; then
|
||||||
|
if [[ "${attempt}" == 1 ]]; then
|
||||||
|
echo -e "${color_red} Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo -e "${color_yellow}Attempt $(($attempt+1)) failed to $1 $2 $3. Retrying.${color_norm}" >& 2
|
||||||
|
sleep $(($attempt * 5))
|
||||||
|
else
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo -e "${color_red} Failed to $1 $2 $3.${color_norm}" >& 2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
@ -14,7 +14,9 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# This script assumes that kubectl binary is present in PATH.
|
curl https://sdk.cloud.google.com 2> /dev/null | bash
|
||||||
|
sudo gcloud components update kubectl -q
|
||||||
|
sudo ln -s /usr/local/share/google/google-cloud-sdk/bin/kubectl /bin/
|
||||||
kubectl config set-cluster hollow-cluster --server=http://localhost:8080 --insecure-skip-tls-verify=true
|
kubectl config set-cluster hollow-cluster --server=http://localhost:8080 --insecure-skip-tls-verify=true
|
||||||
kubectl config set-credentials $(whoami)
|
kubectl config set-credentials $(whoami)
|
||||||
kubectl config set-context hollow-context --cluster=hollow-cluster --user=$(whoami)
|
kubectl config set-context hollow-context --cluster=hollow-cluster --user=$(whoami)
|
||||||
|
|||||||
@ -1,140 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Copyright 2017 The Kubernetes Authors.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
source "${KUBE_ROOT}/cluster/gce/util.sh"
|
|
||||||
|
|
||||||
detect-project &> /dev/null
|
|
||||||
export PROJECT
|
|
||||||
|
|
||||||
RETRIES=3
|
|
||||||
# Runs gcloud compute command with the given parameters. Up to $RETRIES will be made
|
|
||||||
# to execute the command.
|
|
||||||
# arguments:
|
|
||||||
# $@: all stuff that goes after 'gcloud compute'
|
|
||||||
function run-gcloud-compute-with-retries {
|
|
||||||
for attempt in $(seq 1 ${RETRIES}); do
|
|
||||||
local -r gcloud_cmd_hash=`echo "gcloud compute $@" | md5sum | cut -f1 -d" "`
|
|
||||||
local -r gcloud_logfile="/tmp/gcloud_${gcloud_cmd_hash}.log"
|
|
||||||
echo "" > ${gcloud_logfile}
|
|
||||||
if ! gcloud compute "$@" |& tee ${gcloud_logfile}; then
|
|
||||||
if [[ $(grep -c "already exists" ${gcloud_logfile}) -gt 0 ]]; then
|
|
||||||
if [[ "${attempt}" == 1 ]]; then
|
|
||||||
echo -e "${color_red} Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2
|
|
||||||
sleep $(($attempt * 5))
|
|
||||||
else
|
|
||||||
echo -e "${color_green}Succeeded to gcloud compute $1 $2 $3.${color_norm}"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo -e "${color_red} Failed to $1 $2 $3.${color_norm}" >& 2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
function create-master-instance-with-resources {
|
|
||||||
GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}"
|
|
||||||
|
|
||||||
run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \
|
|
||||||
${GCLOUD_COMMON_ARGS} \
|
|
||||||
--type "${MASTER_DISK_TYPE}" \
|
|
||||||
--size "${MASTER_DISK_SIZE}"
|
|
||||||
|
|
||||||
if [ "${EVENT_PD:-false}" == "true" ]; then
|
|
||||||
run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \
|
|
||||||
${GCLOUD_COMMON_ARGS} \
|
|
||||||
--type "${MASTER_DISK_TYPE}" \
|
|
||||||
--size "${MASTER_DISK_SIZE}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \
|
|
||||||
--project "${PROJECT}" \
|
|
||||||
--region "${REGION}" -q
|
|
||||||
|
|
||||||
MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
|
|
||||||
--project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
|
|
||||||
|
|
||||||
run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \
|
|
||||||
${GCLOUD_COMMON_ARGS} \
|
|
||||||
--address "${MASTER_IP}" \
|
|
||||||
--machine-type "${MASTER_SIZE}" \
|
|
||||||
--image-project="${MASTER_IMAGE_PROJECT}" \
|
|
||||||
--image "${MASTER_IMAGE}" \
|
|
||||||
--tags "${MASTER_TAG}" \
|
|
||||||
--network "${NETWORK}" \
|
|
||||||
--scopes "storage-ro,compute-rw,logging-write" \
|
|
||||||
--boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \
|
|
||||||
--disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no"
|
|
||||||
|
|
||||||
if [ "${EVENT_PD:-false}" == "true" ]; then
|
|
||||||
echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}"
|
|
||||||
run-gcloud-compute-with-retries instances attach-disk "${MASTER_NAME}" \
|
|
||||||
${GCLOUD_COMMON_ARGS} \
|
|
||||||
--disk "${MASTER_NAME}-event-pd" \
|
|
||||||
--device-name="master-event-pd"
|
|
||||||
fi
|
|
||||||
|
|
||||||
run-gcloud-compute-with-retries firewall-rules create "${MASTER_NAME}-https" \
|
|
||||||
--project "${PROJECT}" \
|
|
||||||
--network "${NETWORK}" \
|
|
||||||
--source-ranges "0.0.0.0/0" \
|
|
||||||
--target-tags "${MASTER_TAG}" \
|
|
||||||
--allow "tcp:443"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Command to be executed is '$1'.
|
|
||||||
# No. of retries is '$2' (if provided) or 1 (default).
|
|
||||||
function execute-cmd-on-master-with-retries() {
|
|
||||||
RETRIES="${2:-1}" run-gcloud-compute-with-retries ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" --command="$1"
|
|
||||||
}
|
|
||||||
|
|
||||||
function copy-files() {
|
|
||||||
run-gcloud-compute-with-retries copy-files --zone="${ZONE}" --project="${PROJECT}" $@
|
|
||||||
}
|
|
||||||
|
|
||||||
function delete-master-instance-and-resources {
|
|
||||||
GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE} --quiet"
|
|
||||||
|
|
||||||
gcloud compute instances delete "${MASTER_NAME}" \
|
|
||||||
${GCLOUD_COMMON_ARGS} || true
|
|
||||||
|
|
||||||
gcloud compute disks delete "${MASTER_NAME}-pd" \
|
|
||||||
${GCLOUD_COMMON_ARGS} || true
|
|
||||||
|
|
||||||
gcloud compute disks delete "${MASTER_NAME}-event-pd" \
|
|
||||||
${GCLOUD_COMMON_ARGS} &> /dev/null || true
|
|
||||||
|
|
||||||
gcloud compute addresses delete "${MASTER_NAME}-ip" \
|
|
||||||
--project "${PROJECT}" \
|
|
||||||
--region "${REGION}" \
|
|
||||||
--quiet || true
|
|
||||||
|
|
||||||
gcloud compute firewall-rules delete "${MASTER_NAME}-https" \
|
|
||||||
--project "${PROJECT}" \
|
|
||||||
--quiet || true
|
|
||||||
|
|
||||||
if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then
|
|
||||||
gcloud compute instances delete "${EVENT_STORE_NAME}" \
|
|
||||||
${GCLOUD_COMMON_ARGS} || true
|
|
||||||
|
|
||||||
gcloud compute disks delete "${EVENT_STORE_NAME}-pd" \
|
|
||||||
${GCLOUD_COMMON_ARGS} || true
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
@ -66,7 +66,7 @@
|
|||||||
"/eventer"
|
"/eventer"
|
||||||
],
|
],
|
||||||
"args": [
|
"args": [
|
||||||
"--source=kubernetes:https://{{MASTER_IP}}:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/heapster.kubeconfig"
|
"--source=kubernetes:https://104.197.233.84:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/heapster.kubeconfig"
|
||||||
],
|
],
|
||||||
"volumeMounts": [
|
"volumeMounts": [
|
||||||
{
|
{
|
||||||
|
|||||||
@ -80,7 +80,7 @@
|
|||||||
"command": [
|
"command": [
|
||||||
"/bin/sh",
|
"/bin/sh",
|
||||||
"-c",
|
"-c",
|
||||||
"./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kubelet_$(NODE_NAME).log 2>&1"
|
"./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kubelet_$(MY_POD_NAME).log 2>&1"
|
||||||
],
|
],
|
||||||
"volumeMounts": [
|
"volumeMounts": [
|
||||||
{
|
{
|
||||||
@ -129,7 +129,7 @@
|
|||||||
"command": [
|
"command": [
|
||||||
"/bin/sh",
|
"/bin/sh",
|
||||||
"-c",
|
"-c",
|
||||||
"./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kubeproxy_$(NODE_NAME).log 2>&1"
|
"./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kube_proxy_$(MY_POD_NAME).log 2>&1"
|
||||||
],
|
],
|
||||||
"volumeMounts": [
|
"volumeMounts": [
|
||||||
{
|
{
|
||||||
|
|||||||
@ -14,18 +14,17 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
|
|
||||||
|
|
||||||
source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
|
|
||||||
source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
|
|
||||||
source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
|
|
||||||
|
|
||||||
export KUBERNETES_PROVIDER="kubemark"
|
export KUBERNETES_PROVIDER="kubemark"
|
||||||
export KUBE_CONFIG_FILE="config-default.sh"
|
export KUBE_CONFIG_FILE="config-default.sh"
|
||||||
|
|
||||||
|
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
|
||||||
|
|
||||||
# We need an absolute path to KUBE_ROOT
|
# We need an absolute path to KUBE_ROOT
|
||||||
ABSOLUTE_ROOT=$(readlink -f ${KUBE_ROOT})
|
ABSOLUTE_ROOT=$(readlink -f ${KUBE_ROOT})
|
||||||
|
|
||||||
|
source ${KUBE_ROOT}/cluster/kubemark/util.sh
|
||||||
|
source ${KUBE_ROOT}/cluster/kubemark/config-default.sh
|
||||||
|
|
||||||
echo "Kubemark master name: ${MASTER_NAME}"
|
echo "Kubemark master name: ${MASTER_NAME}"
|
||||||
|
|
||||||
detect-master
|
detect-master
|
||||||
|
|||||||
@ -1,63 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Copyright 2017 The Kubernetes Authors.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
# This script contains the helper functions that each provider hosting
|
|
||||||
# Kubermark must implement to use test/kubemark/start-kubemark.sh and
|
|
||||||
# test/kubemark/stop-kubemark.sh scripts.
|
|
||||||
|
|
||||||
|
|
||||||
# This function should create a machine instance for the master along
|
|
||||||
# with any/all of the following resources:
|
|
||||||
# - Attach a PD to the master (optionally 1 more for storing events)
|
|
||||||
# - A public IP address for the master ($MASTER_IP)
|
|
||||||
# - A network firewall rule allowing all TCP traffic on port 443 in master
|
|
||||||
# Note: This step is compulsory in order for kubemark to work
|
|
||||||
#
|
|
||||||
# ENV vars that should be defined by the end of this function:
|
|
||||||
# 1. MASTER_IP
|
|
||||||
# 2. MASTER_NAME
|
|
||||||
#
|
|
||||||
# Recommended for this function to include retrying logic for the above
|
|
||||||
# operations in case of failures.
|
|
||||||
function create-master-instance-with-resources {
|
|
||||||
echo "MASTER_IP: $MASTER_IP" 1>&2
|
|
||||||
echo "MASTER_NAME: $MASTER_NAME" 1>&2
|
|
||||||
}
|
|
||||||
|
|
||||||
# This function should execute the command('$1') on the master machine
|
|
||||||
# (possibly through SSH), retrying in case of failure. The allowed number of
|
|
||||||
# retries would be '$2' (if not provided, default to single try).
|
|
||||||
function execute-cmd-on-master-with-retries() {
|
|
||||||
echo "Executing command on the master" 1>&2
|
|
||||||
}
|
|
||||||
|
|
||||||
# This function should act as an scp for the kubemark cluster, which copies
|
|
||||||
# the files given by the first n-1 arguments to the remote location given
|
|
||||||
# by the n^th argument.
|
|
||||||
#
|
|
||||||
# Recommended for this function to include retrying logic in case of failures.
|
|
||||||
function copy-files() {
|
|
||||||
echo "Copying files" 1>&2
|
|
||||||
}
|
|
||||||
|
|
||||||
# This function should delete the master instance along with all the
|
|
||||||
# resources that have been allocated inside the function
|
|
||||||
# 'create-master-instance-with-resources' above.
|
|
||||||
#
|
|
||||||
# Recommended for this function to include retrying logic in case of failures.
|
|
||||||
function delete-master-instance-and-resources {
|
|
||||||
echo "Deleting master instance and its allocated resources" 1>&2
|
|
||||||
}
|
|
||||||
@ -14,34 +14,15 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Script that creates a Kubemark cluster for any given cloud provider.
|
# Script that creates a Kubemark cluster with Master running on GCE.
|
||||||
|
|
||||||
|
# Hack to make it work for OS X. Ugh...
|
||||||
TMP_ROOT="$(dirname "${BASH_SOURCE}")/../.."
|
TMP_ROOT="$(dirname "${BASH_SOURCE}")/../.."
|
||||||
KUBE_ROOT=$(readlink -e ${TMP_ROOT} 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' ${TMP_ROOT})
|
KUBE_ROOT=$(readlink -e ${TMP_ROOT} 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' ${TMP_ROOT})
|
||||||
KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
|
|
||||||
KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
|
|
||||||
RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
|
|
||||||
|
|
||||||
source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh"
|
source "${KUBE_ROOT}/test/kubemark/common.sh"
|
||||||
source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
|
|
||||||
source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
|
|
||||||
source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
|
|
||||||
|
|
||||||
# hack/lib/init.sh will ovewrite ETCD_VERSION if this is unset
|
function writeEnvironmentFile() {
|
||||||
# to what is default in hack/lib/etcd.sh
|
|
||||||
# To avoid it, if it is empty, we set it to 'avoid-overwrite' and
|
|
||||||
# clean it after that.
|
|
||||||
if [ -z "${ETCD_VERSION:-}" ]; then
|
|
||||||
ETCD_VERSION="avoid-overwrite"
|
|
||||||
fi
|
|
||||||
source "${KUBE_ROOT}/hack/lib/init.sh"
|
|
||||||
if [ "${ETCD_VERSION:-}" == "avoid-overwrite" ]; then
|
|
||||||
ETCD_VERSION=""
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Write all environment variables that we need to pass to the kubemark master,
|
|
||||||
# locally to the file ${RESOURCE_DIRECTORY}/kubemark-master-env.sh.
|
|
||||||
function create-master-environment-file {
|
|
||||||
cat > "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" <<EOF
|
cat > "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" <<EOF
|
||||||
# Generic variables.
|
# Generic variables.
|
||||||
INSTANCE_PREFIX="${INSTANCE_PREFIX:-}"
|
INSTANCE_PREFIX="${INSTANCE_PREFIX:-}"
|
||||||
@ -66,33 +47,74 @@ STORAGE_BACKEND="${STORAGE_BACKEND:-}"
|
|||||||
NUM_NODES="${NUM_NODES:-}"
|
NUM_NODES="${NUM_NODES:-}"
|
||||||
CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota}"
|
CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota}"
|
||||||
EOF
|
EOF
|
||||||
echo "Created the environment file for master."
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Generate certs/keys for CA, master, kubelet and kubecfg, and tokens for kubelet
|
writeEnvironmentFile
|
||||||
# and kubeproxy.
|
|
||||||
function generate-pki-config {
|
|
||||||
ensure-temp-dir
|
|
||||||
gen-kube-bearertoken
|
|
||||||
gen-kube-basicauth
|
|
||||||
create-certs ${MASTER_IP}
|
|
||||||
KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
|
||||||
KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
|
||||||
NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
|
||||||
HEAPSTER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
|
||||||
echo "Generated PKI authentication data for kubemark."
|
|
||||||
}
|
|
||||||
|
|
||||||
# Wait for the master to be reachable for executing commands on it. We do this by
|
GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}"
|
||||||
# trying to run the bash noop(:) on the master, with 10 retries.
|
|
||||||
function wait-for-master-reachability {
|
|
||||||
execute-cmd-on-master-with-retries ":" 10
|
|
||||||
echo "Checked master reachability for remote command execution."
|
|
||||||
}
|
|
||||||
|
|
||||||
# Write all the relevant certs/keys/tokens to the master.
|
run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \
|
||||||
function write-pki-config-to-master {
|
${GCLOUD_COMMON_ARGS} \
|
||||||
PKI_SETUP_CMD="sudo mkdir /home/kubernetes -p && sudo mkdir /etc/srv/kubernetes -p && \
|
--type "${MASTER_DISK_TYPE}" \
|
||||||
|
--size "${MASTER_DISK_SIZE}"
|
||||||
|
|
||||||
|
if [ "${EVENT_PD:-false}" == "true" ]; then
|
||||||
|
run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \
|
||||||
|
${GCLOUD_COMMON_ARGS} \
|
||||||
|
--type "${MASTER_DISK_TYPE}" \
|
||||||
|
--size "${MASTER_DISK_SIZE}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \
|
||||||
|
--project "${PROJECT}" \
|
||||||
|
--region "${REGION}" -q
|
||||||
|
|
||||||
|
MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
|
||||||
|
--project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
|
||||||
|
|
||||||
|
run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \
|
||||||
|
${GCLOUD_COMMON_ARGS} \
|
||||||
|
--address "${MASTER_IP}" \
|
||||||
|
--machine-type "${MASTER_SIZE}" \
|
||||||
|
--image-project="${MASTER_IMAGE_PROJECT}" \
|
||||||
|
--image "${MASTER_IMAGE}" \
|
||||||
|
--tags "${MASTER_TAG}" \
|
||||||
|
--network "${NETWORK}" \
|
||||||
|
--scopes "storage-ro,compute-rw,logging-write" \
|
||||||
|
--boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \
|
||||||
|
--disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no"
|
||||||
|
|
||||||
|
if [ "${EVENT_PD:-false}" == "true" ]; then
|
||||||
|
echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}"
|
||||||
|
run-gcloud-compute-with-retries instances attach-disk "${MASTER_NAME}" \
|
||||||
|
${GCLOUD_COMMON_ARGS} \
|
||||||
|
--disk "${MASTER_NAME}-event-pd" \
|
||||||
|
--device-name="master-event-pd"
|
||||||
|
fi
|
||||||
|
|
||||||
|
run-gcloud-compute-with-retries firewall-rules create "${INSTANCE_PREFIX}-kubemark-master-https" \
|
||||||
|
--project "${PROJECT}" \
|
||||||
|
--network "${NETWORK}" \
|
||||||
|
--source-ranges "0.0.0.0/0" \
|
||||||
|
--target-tags "${MASTER_TAG}" \
|
||||||
|
--allow "tcp:443"
|
||||||
|
|
||||||
|
ensure-temp-dir
|
||||||
|
gen-kube-bearertoken
|
||||||
|
create-certs ${MASTER_IP}
|
||||||
|
KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||||
|
KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||||
|
HEAPSTER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||||
|
NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||||
|
|
||||||
|
until gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" --command="ls" &> /dev/null; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
password=$(python -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))')
|
||||||
|
|
||||||
|
run-gcloud-compute-with-retries ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \
|
||||||
|
--command="sudo mkdir /home/kubernetes -p && sudo mkdir /etc/srv/kubernetes -p && \
|
||||||
sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/ca.crt\" && \
|
sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/ca.crt\" && \
|
||||||
sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/server.cert\" && \
|
sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/server.cert\" && \
|
||||||
sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 --decode > /etc/srv/kubernetes/server.key\" && \
|
sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 --decode > /etc/srv/kubernetes/server.key\" && \
|
||||||
@ -101,16 +123,11 @@ function write-pki-config-to-master {
|
|||||||
sudo bash -c \"echo \"${KUBE_BEARER_TOKEN},admin,admin\" > /etc/srv/kubernetes/known_tokens.csv\" && \
|
sudo bash -c \"echo \"${KUBE_BEARER_TOKEN},admin,admin\" > /etc/srv/kubernetes/known_tokens.csv\" && \
|
||||||
sudo bash -c \"echo \"${KUBELET_TOKEN},system:node:node-name,uid:kubelet,system:nodes\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
|
sudo bash -c \"echo \"${KUBELET_TOKEN},system:node:node-name,uid:kubelet,system:nodes\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
|
||||||
sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},system:kube-proxy,uid:kube_proxy\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
|
sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},system:kube-proxy,uid:kube_proxy\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
|
||||||
sudo bash -c \"echo \"${NODE_PROBLEM_DETECTOR_TOKEN},system:node-problem-detector,uid:system:node-problem-detector\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
|
|
||||||
sudo bash -c \"echo \"${HEAPSTER_TOKEN},system:heapster,uid:heapster\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
|
sudo bash -c \"echo \"${HEAPSTER_TOKEN},system:heapster,uid:heapster\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
|
||||||
sudo bash -c \"echo ${KUBE_PASSWORD},admin,admin > /etc/srv/kubernetes/basic_auth.csv\""
|
sudo bash -c \"echo \"${NODE_PROBLEM_DETECTOR_TOKEN},system:node-problem-detector,uid:system:node-problem-detector\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
|
||||||
execute-cmd-on-master-with-retries "${PKI_SETUP_CMD}" 3
|
sudo bash -c \"echo ${password},admin,admin > /etc/srv/kubernetes/basic_auth.csv\""
|
||||||
echo "Wrote PKI certs, keys, tokens and admin password to master."
|
|
||||||
}
|
|
||||||
|
|
||||||
# Copy all the necessary resource files (scripts/configs/manifests) to the master.
|
run-gcloud-compute-with-retries copy-files --zone="${ZONE}" --project="${PROJECT}" \
|
||||||
function copy-resource-files-to-master {
|
|
||||||
copy-files \
|
|
||||||
"${SERVER_BINARY_TAR}" \
|
"${SERVER_BINARY_TAR}" \
|
||||||
"${RESOURCE_DIRECTORY}/kubemark-master-env.sh" \
|
"${RESOURCE_DIRECTORY}/kubemark-master-env.sh" \
|
||||||
"${RESOURCE_DIRECTORY}/start-kubemark-master.sh" \
|
"${RESOURCE_DIRECTORY}/start-kubemark-master.sh" \
|
||||||
@ -123,24 +140,120 @@ function copy-resource-files-to-master {
|
|||||||
"${RESOURCE_DIRECTORY}/manifests/kube-addon-manager.yaml" \
|
"${RESOURCE_DIRECTORY}/manifests/kube-addon-manager.yaml" \
|
||||||
"${RESOURCE_DIRECTORY}/manifests/addons/kubemark-rbac-bindings" \
|
"${RESOURCE_DIRECTORY}/manifests/addons/kubemark-rbac-bindings" \
|
||||||
"kubernetes@${MASTER_NAME}":/home/kubernetes/
|
"kubernetes@${MASTER_NAME}":/home/kubernetes/
|
||||||
echo "Copied server binary, master startup scripts, configs and resource manifests to master."
|
|
||||||
}
|
|
||||||
|
|
||||||
# Make startup scripts executable and run start-kubemark-master.sh.
|
gcloud compute ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" \
|
||||||
function start-master-components {
|
--command="sudo chmod a+x /home/kubernetes/configure-kubectl.sh && \
|
||||||
echo ""
|
|
||||||
MASTER_STARTUP_CMD="sudo chmod a+x /home/kubernetes/configure-kubectl.sh && \
|
|
||||||
sudo chmod a+x /home/kubernetes/start-kubemark-master.sh && \
|
sudo chmod a+x /home/kubernetes/start-kubemark-master.sh && \
|
||||||
sudo bash /home/kubernetes/start-kubemark-master.sh"
|
sudo bash /home/kubernetes/start-kubemark-master.sh"
|
||||||
execute-cmd-on-master-with-retries "${MASTER_STARTUP_CMD}"
|
|
||||||
echo "The master has started and is now live."
|
|
||||||
}
|
|
||||||
|
|
||||||
# Write kubeconfig to ${RESOURCE_DIRECTORY}/kubeconfig.kubemark in order to
|
# Setup the docker image for kubemark hollow-node.
|
||||||
# use kubectl locally.
|
MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
|
||||||
function write-local-kubeconfig {
|
KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)"
|
||||||
LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
|
if [[ -z "${KUBEMARK_BIN}" ]]; then
|
||||||
cat > "${LOCAL_KUBECONFIG}" << EOF
|
echo 'Cannot find cmd/kubemark binary'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Copying kubemark to ${MAKE_DIR}"
|
||||||
|
cp "${KUBEMARK_BIN}" "${MAKE_DIR}"
|
||||||
|
CURR_DIR=`pwd`
|
||||||
|
cd "${MAKE_DIR}"
|
||||||
|
RETRIES=3
|
||||||
|
for attempt in $(seq 1 ${RETRIES}); do
|
||||||
|
if ! make; then
|
||||||
|
if [[ $((attempt)) -eq "${RETRIES}" ]]; then
|
||||||
|
echo "${color_red}Make failed. Exiting.${color_norm}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo -e "${color_yellow}Make attempt $(($attempt)) failed. Retrying.${color_norm}" >& 2
|
||||||
|
sleep $(($attempt * 5))
|
||||||
|
else
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
rm kubemark
|
||||||
|
cd $CURR_DIR
|
||||||
|
|
||||||
|
# Create kubeconfig for Kubelet.
|
||||||
|
KUBELET_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
|
||||||
|
kind: Config
|
||||||
|
users:
|
||||||
|
- name: kubelet
|
||||||
|
user:
|
||||||
|
client-certificate-data: "${KUBELET_CERT_BASE64}"
|
||||||
|
client-key-data: "${KUBELET_KEY_BASE64}"
|
||||||
|
clusters:
|
||||||
|
- name: kubemark
|
||||||
|
cluster:
|
||||||
|
certificate-authority-data: "${CA_CERT_BASE64}"
|
||||||
|
server: https://${MASTER_IP}
|
||||||
|
contexts:
|
||||||
|
- context:
|
||||||
|
cluster: kubemark
|
||||||
|
user: kubelet
|
||||||
|
name: kubemark-context
|
||||||
|
current-context: kubemark-context")
|
||||||
|
|
||||||
|
# Create kubeconfig for Kubeproxy.
|
||||||
|
KUBEPROXY_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
|
||||||
|
kind: Config
|
||||||
|
users:
|
||||||
|
- name: kube-proxy
|
||||||
|
user:
|
||||||
|
token: ${KUBE_PROXY_TOKEN}
|
||||||
|
clusters:
|
||||||
|
- name: kubemark
|
||||||
|
cluster:
|
||||||
|
insecure-skip-tls-verify: true
|
||||||
|
server: https://${MASTER_IP}
|
||||||
|
contexts:
|
||||||
|
- context:
|
||||||
|
cluster: kubemark
|
||||||
|
user: kube-proxy
|
||||||
|
name: kubemark-context
|
||||||
|
current-context: kubemark-context")
|
||||||
|
|
||||||
|
# Create kubeconfig for Heapster.
|
||||||
|
HEAPSTER_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
|
||||||
|
kind: Config
|
||||||
|
users:
|
||||||
|
- name: heapster
|
||||||
|
user:
|
||||||
|
token: ${HEAPSTER_TOKEN}
|
||||||
|
clusters:
|
||||||
|
- name: kubemark
|
||||||
|
cluster:
|
||||||
|
insecure-skip-tls-verify: true
|
||||||
|
server: https://${MASTER_IP}
|
||||||
|
contexts:
|
||||||
|
- context:
|
||||||
|
cluster: kubemark
|
||||||
|
user: heapster
|
||||||
|
name: kubemark-context
|
||||||
|
current-context: kubemark-context")
|
||||||
|
|
||||||
|
# Create kubeconfig for NodeProblemDetector.
|
||||||
|
NPD_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
|
||||||
|
kind: Config
|
||||||
|
users:
|
||||||
|
- name: node-problem-detector
|
||||||
|
user:
|
||||||
|
token: ${NODE_PROBLEM_DETECTOR_TOKEN}
|
||||||
|
clusters:
|
||||||
|
- name: kubemark
|
||||||
|
cluster:
|
||||||
|
insecure-skip-tls-verify: true
|
||||||
|
server: https://${MASTER_IP}
|
||||||
|
contexts:
|
||||||
|
- context:
|
||||||
|
cluster: kubemark
|
||||||
|
user: node-problem-detector
|
||||||
|
name: kubemark-npd-context
|
||||||
|
current-context: kubemark-npd-context")
|
||||||
|
|
||||||
|
# Create kubeconfig for local kubectl.
|
||||||
|
LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
|
||||||
|
cat > "${LOCAL_KUBECONFIG}" << EOF
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Config
|
kind: Config
|
||||||
users:
|
users:
|
||||||
@ -162,217 +275,70 @@ contexts:
|
|||||||
name: kubemark-context
|
name: kubemark-context
|
||||||
current-context: kubemark-context
|
current-context: kubemark-context
|
||||||
EOF
|
EOF
|
||||||
echo "Kubeconfig file for kubemark master written to ${LOCAL_KUBECONFIG}."
|
|
||||||
}
|
|
||||||
|
|
||||||
# Finds the right kubemark binary for 'linux/amd64' platform and uses it to
|
sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json"
|
||||||
# create a docker image for hollow-node and upload it to the appropriate
|
sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
||||||
# docker container registry for the cloud provider.
|
sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
||||||
# TODO(shyamjvs): Make the image upload URL and makefile variable w.r.t. provider.
|
|
||||||
function create-and-upload-hollow-node-image {
|
mkdir "${RESOURCE_DIRECTORY}/addons" || true
|
||||||
MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
|
|
||||||
KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)"
|
sed "s/{{MASTER_IP}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json"
|
||||||
if [[ -z "${KUBEMARK_BIN}" ]]; then
|
metrics_mem_per_node=4
|
||||||
echo 'Cannot find cmd/kubemark binary'
|
metrics_mem=$((200 + ${metrics_mem_per_node}*${NUM_NODES:-10}))
|
||||||
|
sed -i'' -e "s/{{METRICS_MEM}}/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
|
||||||
|
eventer_mem_per_node=500
|
||||||
|
eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES:-10}))
|
||||||
|
sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
|
||||||
|
|
||||||
|
# Create kubemark namespace.
|
||||||
|
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json"
|
||||||
|
# Create configmap for configuring hollow- kubelet, proxy and npd.
|
||||||
|
"${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \
|
||||||
|
--from-literal=content.type="${TEST_CLUSTER_API_CONTENT_TYPE}" \
|
||||||
|
--from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json"
|
||||||
|
# Create secret for passing kubeconfigs to kubelet, kubeproxy and npd.
|
||||||
|
"${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \
|
||||||
|
--from-literal=kubelet.kubeconfig="${KUBELET_KUBECONFIG_CONTENTS}" \
|
||||||
|
--from-literal=kubeproxy.kubeconfig="${KUBEPROXY_KUBECONFIG_CONTENTS}" \
|
||||||
|
--from-literal=heapster.kubeconfig="${HEAPSTER_KUBECONFIG_CONTENTS}" \
|
||||||
|
--from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}"
|
||||||
|
# Create addon pods.
|
||||||
|
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
|
||||||
|
# Create the replication controller for hollow-nodes.
|
||||||
|
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark"
|
||||||
|
|
||||||
|
echo "Waiting for all HollowNodes to become Running..."
|
||||||
|
start=$(date +%s)
|
||||||
|
nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
|
||||||
|
ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1))
|
||||||
|
|
||||||
|
until [[ "${ready}" -ge "${NUM_NODES}" ]]; do
|
||||||
|
echo -n .
|
||||||
|
sleep 1
|
||||||
|
now=$(date +%s)
|
||||||
|
# Fail it if it already took more than 30 minutes.
|
||||||
|
if [ $((now - start)) -gt 1800 ]; then
|
||||||
|
echo ""
|
||||||
|
echo "Timeout waiting for all HollowNodes to become Running"
|
||||||
|
# Try listing nodes again - if it fails it means that API server is not responding
|
||||||
|
if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then
|
||||||
|
echo "Found only ${ready} ready Nodes while waiting for ${NUM_NODES}."
|
||||||
|
else
|
||||||
|
echo "Got error while trying to list Nodes. Probably API server is down."
|
||||||
|
fi
|
||||||
|
pods=$("${KUBECTL}" get pods --namespace=kubemark) || true
|
||||||
|
running=$(($(echo "${pods}" | grep "Running" | wc -l)))
|
||||||
|
echo "${running} HollowNode pods are reported as 'Running'"
|
||||||
|
not_running=$(($(echo "${pods}" | grep -v "Running" | wc -l) - 1))
|
||||||
|
echo "${not_running} HollowNode pods are reported as NOT 'Running'"
|
||||||
|
echo $(echo "${pods}" | grep -v "Running")
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
|
||||||
echo "Copying kubemark binary to ${MAKE_DIR}"
|
|
||||||
cp "${KUBEMARK_BIN}" "${MAKE_DIR}"
|
|
||||||
CURR_DIR=`pwd`
|
|
||||||
cd "${MAKE_DIR}"
|
|
||||||
RETRIES=3
|
|
||||||
for attempt in $(seq 1 ${RETRIES}); do
|
|
||||||
if ! make; then
|
|
||||||
if [[ $((attempt)) -eq "${RETRIES}" ]]; then
|
|
||||||
echo "${color_red}Make failed. Exiting.${color_norm}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo -e "${color_yellow}Make attempt $(($attempt)) failed. Retrying.${color_norm}" >& 2
|
|
||||||
sleep $(($attempt * 5))
|
|
||||||
else
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
rm kubemark
|
|
||||||
cd $CURR_DIR
|
|
||||||
echo "Created and uploaded the kubemark hollow-node image to docker registry."
|
|
||||||
}
|
|
||||||
|
|
||||||
# Generate secret and configMap for the hollow-node pods to work, prepare
|
|
||||||
# manifests of the hollow-node and heapster replication controllers from
|
|
||||||
# templates, and finally create these resources through kubectl.
|
|
||||||
function create-kube-hollow-node-resources {
|
|
||||||
# Create kubeconfig for Kubelet.
|
|
||||||
KUBELET_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
|
|
||||||
kind: Config
|
|
||||||
users:
|
|
||||||
- name: kubelet
|
|
||||||
user:
|
|
||||||
client-certificate-data: "${KUBELET_CERT_BASE64}"
|
|
||||||
client-key-data: "${KUBELET_KEY_BASE64}"
|
|
||||||
clusters:
|
|
||||||
- name: kubemark
|
|
||||||
cluster:
|
|
||||||
certificate-authority-data: "${CA_CERT_BASE64}"
|
|
||||||
server: https://${MASTER_IP}
|
|
||||||
contexts:
|
|
||||||
- context:
|
|
||||||
cluster: kubemark
|
|
||||||
user: kubelet
|
|
||||||
name: kubemark-context
|
|
||||||
current-context: kubemark-context")
|
|
||||||
|
|
||||||
# Create kubeconfig for Kubeproxy.
|
|
||||||
KUBEPROXY_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
|
|
||||||
kind: Config
|
|
||||||
users:
|
|
||||||
- name: kube-proxy
|
|
||||||
user:
|
|
||||||
token: ${KUBE_PROXY_TOKEN}
|
|
||||||
clusters:
|
|
||||||
- name: kubemark
|
|
||||||
cluster:
|
|
||||||
insecure-skip-tls-verify: true
|
|
||||||
server: https://${MASTER_IP}
|
|
||||||
contexts:
|
|
||||||
- context:
|
|
||||||
cluster: kubemark
|
|
||||||
user: kube-proxy
|
|
||||||
name: kubemark-context
|
|
||||||
current-context: kubemark-context")
|
|
||||||
|
|
||||||
# Create kubeconfig for NodeProblemDetector.
|
|
||||||
NPD_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
|
|
||||||
kind: Config
|
|
||||||
users:
|
|
||||||
- name: node-problem-detector
|
|
||||||
user:
|
|
||||||
token: ${NODE_PROBLEM_DETECTOR_TOKEN}
|
|
||||||
clusters:
|
|
||||||
- name: kubemark
|
|
||||||
cluster:
|
|
||||||
insecure-skip-tls-verify: true
|
|
||||||
server: https://${MASTER_IP}
|
|
||||||
contexts:
|
|
||||||
- context:
|
|
||||||
cluster: kubemark
|
|
||||||
user: node-problem-detector
|
|
||||||
name: kubemark-context
|
|
||||||
current-context: kubemark-context")
|
|
||||||
|
|
||||||
# Create kubeconfig for Heapster.
|
|
||||||
HEAPSTER_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
|
|
||||||
kind: Config
|
|
||||||
users:
|
|
||||||
- name: heapster
|
|
||||||
user:
|
|
||||||
token: ${HEAPSTER_TOKEN}
|
|
||||||
clusters:
|
|
||||||
- name: kubemark
|
|
||||||
cluster:
|
|
||||||
insecure-skip-tls-verify: true
|
|
||||||
server: https://${MASTER_IP}
|
|
||||||
contexts:
|
|
||||||
- context:
|
|
||||||
cluster: kubemark
|
|
||||||
user: heapster
|
|
||||||
name: kubemark-context
|
|
||||||
current-context: kubemark-context")
|
|
||||||
|
|
||||||
# Create kubemark namespace.
|
|
||||||
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json"
|
|
||||||
|
|
||||||
# Create configmap for configuring hollow- kubelet, proxy and npd.
|
|
||||||
"${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \
|
|
||||||
--from-literal=content.type="${TEST_CLUSTER_API_CONTENT_TYPE}" \
|
|
||||||
--from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json"
|
|
||||||
|
|
||||||
# Create secret for passing kubeconfigs to kubelet, kubeproxy and npd.
|
|
||||||
"${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \
|
|
||||||
--from-literal=kubelet.kubeconfig="${KUBELET_KUBECONFIG_CONTENTS}" \
|
|
||||||
--from-literal=kubeproxy.kubeconfig="${KUBEPROXY_KUBECONFIG_CONTENTS}" \
|
|
||||||
--from-literal=heapster.kubeconfig="${HEAPSTER_KUBECONFIG_CONTENTS}" \
|
|
||||||
--from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}"
|
|
||||||
|
|
||||||
# Create addon pods.
|
|
||||||
# TODO(shyamjvs): Make path to docker image variable in heapster_template.json.
|
|
||||||
mkdir -p "${RESOURCE_DIRECTORY}/addons"
|
|
||||||
sed "s/{{MASTER_IP}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json"
|
|
||||||
metrics_mem_per_node=4
|
|
||||||
metrics_mem=$((200 + ${metrics_mem_per_node}*${NUM_NODES:-10}))
|
|
||||||
sed -i'' -e "s/{{METRICS_MEM}}/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
|
|
||||||
eventer_mem_per_node=500
|
|
||||||
eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES:-10}))
|
|
||||||
sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
|
|
||||||
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
|
|
||||||
|
|
||||||
# Create the replication controller for hollow-nodes.
|
|
||||||
# TODO(shyamjvs): Make path to docker image variable in hollow-node_template.json.
|
|
||||||
sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json"
|
|
||||||
sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
|
||||||
sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
|
||||||
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark"
|
|
||||||
|
|
||||||
echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."
|
|
||||||
}
|
|
||||||
|
|
||||||
# Wait until all hollow-nodes are running or there is a timeout.
|
|
||||||
function wait-for-hollow-nodes-to-run-or-timeout {
|
|
||||||
echo -n "Waiting for all hollow-nodes to become Running"
|
|
||||||
start=$(date +%s)
|
|
||||||
nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node) || true
|
|
||||||
ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1))
|
ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1))
|
||||||
|
done
|
||||||
until [[ "${ready}" -ge "${NUM_NODES}" ]]; do
|
|
||||||
echo -n "."
|
|
||||||
sleep 1
|
|
||||||
now=$(date +%s)
|
|
||||||
# Fail it if it already took more than 30 minutes.
|
|
||||||
if [ $((now - start)) -gt 1800 ]; then
|
|
||||||
echo ""
|
|
||||||
echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}"
|
|
||||||
# Try listing nodes again - if it fails it means that API server is not responding
|
|
||||||
if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then
|
|
||||||
echo "Found only ${ready} ready hollow-nodes while waiting for ${NUM_NODES}."
|
|
||||||
else
|
|
||||||
echo "Got error while trying to list hollow-nodes. Probably API server is down."
|
|
||||||
fi
|
|
||||||
pods=$("${KUBECTL}" get pods --namespace=kubemark) || true
|
|
||||||
running=$(($(echo "${pods}" | grep "Running" | wc -l)))
|
|
||||||
echo "${running} hollow-nodes are reported as 'Running'"
|
|
||||||
not_running=$(($(echo "${pods}" | grep -v "Running" | wc -l) - 1))
|
|
||||||
echo "${not_running} hollow-nodes are reported as NOT 'Running'"
|
|
||||||
echo $(echo "${pods}" | grep -v "Running")
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node) || true
|
|
||||||
ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1))
|
|
||||||
done
|
|
||||||
echo -e "${color_green} Done!${color_norm}"
|
|
||||||
}
|
|
||||||
|
|
||||||
############################### Main Function ########################################
|
|
||||||
# Setup for master.
|
|
||||||
echo -e "${color_yellow}STARTING SETUP FOR MASTER${color_norm}"
|
|
||||||
find-release-tars
|
|
||||||
create-master-environment-file
|
|
||||||
create-master-instance-with-resources
|
|
||||||
generate-pki-config
|
|
||||||
wait-for-master-reachability
|
|
||||||
write-pki-config-to-master
|
|
||||||
copy-resource-files-to-master
|
|
||||||
start-master-components
|
|
||||||
|
|
||||||
# Setup for hollow-nodes.
|
|
||||||
echo ""
|
echo ""
|
||||||
echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}"
|
|
||||||
write-local-kubeconfig
|
|
||||||
create-and-upload-hollow-node-image
|
|
||||||
create-kube-hollow-node-resources
|
|
||||||
wait-for-hollow-nodes-to-run-or-timeout
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "Master IP: ${MASTER_IP}"
|
echo "Master IP: ${MASTER_IP}"
|
||||||
echo "Password to kubemark master: ${KUBE_PASSWORD}"
|
echo "Password to kubemark master: ${password}"
|
||||||
echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}"
|
echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}"
|
||||||
|
|||||||
@ -14,25 +14,47 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Script that destroys Kubemark cluster and deletes all master resources.
|
# Script that destroys Kubemark clusters and deletes all GCE resources created for Master
|
||||||
|
|
||||||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
|
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
|
||||||
|
|
||||||
source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh"
|
source "${KUBE_ROOT}/test/kubemark/common.sh"
|
||||||
source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
|
|
||||||
source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
|
|
||||||
source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
|
|
||||||
|
|
||||||
KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
|
|
||||||
RESOURCE_DIRECTORY="${KUBE_ROOT}/test/kubemark/resources"
|
|
||||||
|
|
||||||
|
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/hollow-kubelet.json" &> /dev/null || true
|
||||||
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/addons" &> /dev/null || true
|
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/addons" &> /dev/null || true
|
||||||
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/hollow-node.json" &> /dev/null || true
|
|
||||||
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" &> /dev/null || true
|
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" &> /dev/null || true
|
||||||
|
rm -rf "${RESOURCE_DIRECTORY}/addons"
|
||||||
|
|
||||||
rm -rf "${RESOURCE_DIRECTORY}/addons" \
|
GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE} --quiet"
|
||||||
"${RESOURCE_DIRECTORY}/kubeconfig.kubemark" \
|
|
||||||
|
gcloud compute instances delete "${MASTER_NAME}" \
|
||||||
|
${GCLOUD_COMMON_ARGS} || true
|
||||||
|
|
||||||
|
gcloud compute disks delete "${MASTER_NAME}-pd" \
|
||||||
|
${GCLOUD_COMMON_ARGS} || true
|
||||||
|
|
||||||
|
gcloud compute disks delete "${MASTER_NAME}-event-pd" \
|
||||||
|
${GCLOUD_COMMON_ARGS} &> /dev/null || true
|
||||||
|
|
||||||
|
gcloud compute addresses delete "${MASTER_NAME}-ip" \
|
||||||
|
--project "${PROJECT}" \
|
||||||
|
--region "${REGION}" \
|
||||||
|
--quiet || true
|
||||||
|
|
||||||
|
gcloud compute firewall-rules delete "${INSTANCE_PREFIX}-kubemark-master-https" \
|
||||||
|
--project "${PROJECT}" \
|
||||||
|
--quiet || true
|
||||||
|
|
||||||
|
if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then
|
||||||
|
gcloud compute instances delete "${EVENT_STORE_NAME}" \
|
||||||
|
${GCLOUD_COMMON_ARGS} || true
|
||||||
|
|
||||||
|
gcloud compute disks delete "${EVENT_STORE_NAME}-pd" \
|
||||||
|
${GCLOUD_COMMON_ARGS} || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -rf "${RESOURCE_DIRECTORY}/addons" "${RESOURCE_DIRECTORY}/kubeconfig.kubemark" &> /dev/null || true
|
||||||
|
rm "${RESOURCE_DIRECTORY}/ca.crt" \
|
||||||
|
"${RESOURCE_DIRECTORY}/kubecfg.crt" \
|
||||||
|
"${RESOURCE_DIRECTORY}/kubecfg.key" \
|
||||||
"${RESOURCE_DIRECTORY}/hollow-node.json" \
|
"${RESOURCE_DIRECTORY}/hollow-node.json" \
|
||||||
"${RESOURCE_DIRECTORY}/kubemark-master-env.sh" &> /dev/null || true
|
"${RESOURCE_DIRECTORY}/kubemark-master-env.sh" &> /dev/null || true
|
||||||
|
|
||||||
delete-master-instance-and-resources
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user