From 82e1d208f612d7d49f451ed3c9978342fcfcea64 Mon Sep 17 00:00:00 2001 From: Ryan Hallisey Date: Mon, 6 Mar 2017 16:08:28 -0500 Subject: [PATCH] Launch kubemark with an existing Kubemark Master In order to expand the use of kubemark, allow developers to use kubemark with a pre-existing Kubemark master. --- .../kubemark/pre-existing/config-default.sh | 39 ++++++++++++ cluster/pre-existing/util.sh | 62 +++++++++++++++++++ test/kubemark/common/util.sh | 43 +++++++++++++ test/kubemark/gce/util.sh | 29 ++------- test/kubemark/pre-existing/util.sh | 27 ++++++++ 5 files changed, 176 insertions(+), 24 deletions(-) create mode 100644 cluster/kubemark/pre-existing/config-default.sh create mode 100644 cluster/pre-existing/util.sh create mode 100644 test/kubemark/common/util.sh create mode 100644 test/kubemark/pre-existing/util.sh diff --git a/cluster/kubemark/pre-existing/config-default.sh b/cluster/kubemark/pre-existing/config-default.sh new file mode 100644 index 00000000000..d6e4ab51871 --- /dev/null +++ b/cluster/kubemark/pre-existing/config-default.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for landing a Kubemark cluster on a pre-existing Kubernetes +# cluster. + +# Pre-existing provider expects a MASTER_IP. +# If you need to specify a port that's not the default (443), add it to MASTER_IP. +# +# Example: Connect to the Master on the secure port 6443 +# MASTER_IP=192.168.122.5:6443 +# +MASTER_IP="${MASTER_IP:-}" + +# The container registry and project given to the kubemark container: +# $CONTAINER_REGISTRY/$PROJECT/kubemark +# +CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-}" +PROJECT="${PROJECT:-}" + +NUM_NODES="${NUM_NODES:-1}" + +TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}" +KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-}" +KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-}" +MASTER_NAME="${MASTER_NAME:-}" +USE_REAL_PROXIER="${USE_REAL_PROXIER:-true}" diff --git a/cluster/pre-existing/util.sh b/cluster/pre-existing/util.sh new file mode 100644 index 00000000000..8443653c5a2 --- /dev/null +++ b/cluster/pre-existing/util.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions for landing kubemark containers on a +# pre-existing Kubernetes master. See test/kubemark/pre-existing/README.md +# for me details on using a pre-existing provider. + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +source "${KUBE_ROOT}/cluster/common.sh" +source "${KUBE_ROOT}/hack/lib/util.sh" + +function detect-project() { + if [[ -z "${MASTER_IP:-}" ]]; then + echo "Set 'MASTER_IP' to the instance assigned to be the Kubernetes master" 1>&2 + exit 1 + fi + + if [[ -z "${PROJECT:-}" ]]; then + echo "Set 'PROJECT' to the name of the container project: $CONTAINER_REGISTRY/$PROJECT/kubemark" >&2 + exit 1 + fi + + if [[ -z "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then + cluster_range=$(echo "${MASTER_IP}" | awk -F '.' '{printf("%d.%d.%d.0", $1, $2, $3)}') + SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-$cluster_range/16}" + fi +} + +function create-certs { + rm /tmp/kubeconfig + + execute-cmd-on-pre-existing-master-with-retries "sudo cat /etc/kubernetes/admin.conf" > /tmp/kubeconfig + CA_CERT_BASE64=$(cat /tmp/kubeconfig | grep certificate-authority | awk '{print $2}' | head -n 1) + KUBELET_CERT_BASE64=$(cat /tmp/kubeconfig | grep client-certificate-data | awk '{print $2}' | head -n 1) + KUBELET_KEY_BASE64=$(cat /tmp/kubeconfig | grep client-key-data | awk '{print $2}' | head -n 1) + + # Local kubeconfig.kubemark vars + KUBECFG_CERT_BASE64="${KUBELET_CERT_BASE64}" + KUBECFG_KEY_BASE64="${KUBELET_KEY_BASE64}" + + # The pre-existing Kubernetes master already has these setup + # Set these vars but don't use them + CA_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + MASTER_CERT_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + MASTER_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + KUBEAPISERVER_CERT_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + KUBEAPISERVER_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) +} diff --git a/test/kubemark/common/util.sh b/test/kubemark/common/util.sh new file mode 100644 index 00000000000..1f4ccbc0aa5 --- /dev/null +++ b/test/kubemark/common/util.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Running cmd $RETRIES times in case of failures. +function run-cmd-with-retries { + RETRIES="${RETRIES:-3}" + for attempt in $(seq 1 ${RETRIES}); do + exec 5>&1 # Duplicate &1 to &5 for use below. + # We don't use 'local' to declare result as then ret_val always gets value 0. + # We use tee to output to &5 (redirected to stdout) while also storing it in the variable. + result=$("$@" 2>&1 | tee >(cat - >&5)) || local ret_val="$?" + if [[ "${ret_val:-0}" -ne "0" ]]; then + if [[ $(echo "${result}" | grep -c "already exists") -gt 0 ]]; then + if [[ "${attempt}" == 1 ]]; then + echo -e "${color_red}Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2 + exit 1 + fi + echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}" + return 0 + fi + echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2 + sleep $(($attempt * 5)) + else + echo -e "${color_green}Succeeded to $1 $2 $3.${color_norm}" + return 0 + fi + done + echo -e "${color_red}Failed to $1 $2 $3.${color_norm}" >& 2 + exit 1 +} diff --git a/test/kubemark/gce/util.sh b/test/kubemark/gce/util.sh index 149958eec71..e86b6c870ac 100644 --- a/test/kubemark/gce/util.sh +++ b/test/kubemark/gce/util.sh @@ -14,34 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../.. + +source "${KUBE_ROOT}/test/kubemark/common/util.sh" + # Wrapper for gcloud compute, running it $RETRIES times in case of failures. # Args: # $@: all stuff that goes after 'gcloud compute' function run-gcloud-compute-with-retries { - RETRIES="${RETRIES:-3}" - for attempt in $(seq 1 ${RETRIES}); do - exec 5>&1 # Duplicate &1 to &5 for use below. - # We don't use 'local' to declare gcloud_result as then ret_val always gets value 0. - # We use tee to output to &5 (redirected to stdout) while also storing it in the variable. - gcloud_result=$(gcloud compute "$@" 2>&1 | tee >(cat - >&5)) || local ret_val="$?" - if [[ "${ret_val:-0}" -ne "0" ]]; then - if [[ $(echo "${gcloud_result}" | grep -c "already exists") -gt 0 ]]; then - if [[ "${attempt}" == 1 ]]; then - echo -e "${color_red}Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2 - exit 1 - fi - echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}" - return 0 - fi - echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2 - sleep $(($attempt * 5)) - else - echo -e "${color_green}Succeeded to gcloud compute $1 $2 $3.${color_norm}" - return 0 - fi - done - echo -e "${color_red}Failed to $1 $2 $3.${color_norm}" >& 2 - exit 1 + run-cmd-with-retries gcloud compute "$@" } function create-master-instance-with-resources { diff --git a/test/kubemark/pre-existing/util.sh b/test/kubemark/pre-existing/util.sh new file mode 100644 index 00000000000..107c921267d --- /dev/null +++ b/test/kubemark/pre-existing/util.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../.. + +source "${KUBE_ROOT}/test/kubemark/common/util.sh" + +# Leave the skeleton definition of execute-cmd-on-master-with-retries +# so only the pre-existing provider functions will target this. +function execute-cmd-on-pre-existing-master-with-retries() { + IP_WITHOUT_PORT=$(echo "${MASTER_IP}" | cut -f 1 -d ':') || "${MASTER_IP}" + + RETRIES="${2:-1}" run-cmd-with-retries ssh kubernetes@"${IP_WITHOUT_PORT}" $1 +}