From 034412aff1cb932f0e10267fd052655cd2456d84 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 25 Mar 2015 06:23:22 -0700 Subject: [PATCH] Support multiple k8s clusters --- cluster/aws/config-default.sh | 1 + cluster/aws/config-test.sh | 1 + cluster/aws/util.sh | 58 ++++++++++++++++++++--------------- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/cluster/aws/config-default.sh b/cluster/aws/config-default.sh index 4789633b1b0..1f17b707e5f 100644 --- a/cluster/aws/config-default.sh +++ b/cluster/aws/config-default.sh @@ -27,6 +27,7 @@ NUM_MINIONS=${NUM_MINIONS:-4} AWS_S3_REGION=${AWS_S3_REGION:-us-east-1} INSTANCE_PREFIX="${KUBE_AWS_INSTANCE_PREFIX:-kubernetes}" +CLUSTER_ID=${INSTANCE_PREFIX} AWS_SSH_KEY=${AWS_SSH_KEY:-$HOME/.ssh/kube_aws_rsa} IAM_PROFILE_MASTER="kubernetes-master" IAM_PROFILE_MINION="kubernetes-minion" diff --git a/cluster/aws/config-test.sh b/cluster/aws/config-test.sh index 552178c52c1..1f60315ae69 100755 --- a/cluster/aws/config-test.sh +++ b/cluster/aws/config-test.sh @@ -23,6 +23,7 @@ NUM_MINIONS=${NUM_MINIONS:-2} AWS_S3_REGION=${AWS_S3_REGION:-us-east-1} INSTANCE_PREFIX="${KUBE_AWS_INSTANCE_PREFIX:-e2e-test-${USER}}" +CLUSTER_ID=${INSTANCE_PREFIX} AWS_SSH_KEY=${AWS_SSH_KEY:-$HOME/.ssh/kube_aws_rsa} IAM_PROFILE_MASTER="kubernetes-master" IAM_PROFILE_MINION="kubernetes-minion" diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 38ce605d21b..579be1474e4 100644 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -36,12 +36,12 @@ function json_val { # TODO (ayurchuk) Refactor the get_* functions to use filters # TODO (bburns) Parameterize this for multiple cluster per project -function get_instance_ids { - python -c "import json,sys; lst = [str(instance['InstanceId']) for reservation in json.load(sys.stdin)['Reservations'] for instance in reservation['Instances'] for tag in instance.get('Tags', []) if tag['Value'].startswith('${MASTER_TAG}') or tag['Value'].startswith('${MINION_TAG}')]; print ' '.join(lst)" -} function get_vpc_id { - python -c 'import json,sys; lst = [str(vpc["VpcId"]) for vpc in json.load(sys.stdin)["Vpcs"] for tag in vpc.get("Tags", []) if tag["Value"] == "kubernetes-vpc"]; print "".join(lst)' + $AWS_CMD --output text describe-vpcs \ + --filters Name=tag:Name,Values=kubernetes-vpc \ + Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ + --query Vpcs[].VpcId } function get_subnet_id { @@ -69,7 +69,9 @@ function expect_instance_states { function get_instance_public_ip { local tagName=$1 $AWS_CMD --output text describe-instances \ - --filters Name=tag:Name,Values=${tagName} Name=instance-state-name,Values=running \ + --filters Name=tag:Name,Values=${tagName} \ + Name=instance-state-name,Values=running \ + Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ --query Reservations[].Instances[].NetworkInterfaces[0].Association.PublicIp } @@ -371,7 +373,7 @@ function kube-up { $AWS_CMD import-key-pair --key-name kubernetes --public-key-material "file://$AWS_SSH_KEY.pub" > $LOG 2>&1 || true - VPC_ID=$($AWS_CMD describe-vpcs | get_vpc_id) + VPC_ID=$(get_vpc_id) if [[ -z "$VPC_ID" ]]; then echo "Creating vpc." @@ -379,6 +381,7 @@ function kube-up { $AWS_CMD modify-vpc-attribute --vpc-id $VPC_ID --enable-dns-support '{"Value": true}' > $LOG $AWS_CMD modify-vpc-attribute --vpc-id $VPC_ID --enable-dns-hostnames '{"Value": true}' > $LOG add-tag $VPC_ID Name kubernetes-vpc + add-tag $VPC_ID KubernetesCluster ${CLUSTER_ID} fi echo "Using VPC $VPC_ID" @@ -467,6 +470,7 @@ function kube-up { --user-data file://${KUBE_TEMP}/master-start.sh | json_val '["Instances"][0]["InstanceId"]') add-tag $master_id Name $MASTER_NAME add-tag $master_id Role $MASTER_TAG + add-tag $master_id KubernetesCluster ${CLUSTER_ID} echo "Waiting for master to be ready" @@ -548,6 +552,7 @@ function kube-up { add-tag $minion_id Name ${MINION_NAMES[$i]} add-tag $minion_id Role $MINION_TAG + add-tag $minion_id KubernetesCluster ${CLUSTER_ID} MINION_IDS[$i]=$minion_id done @@ -700,25 +705,7 @@ EOF } function kube-down { - instance_ids=$($AWS_CMD describe-instances | get_instance_ids) - if [[ -n ${instance_ids} ]]; then - $AWS_CMD terminate-instances --instance-ids $instance_ids > $LOG - echo "Waiting for instances deleted" - while true; do - instance_states=$($AWS_CMD describe-instances --instance-ids $instance_ids | expect_instance_states terminated) - if [[ "$instance_states" == "" ]]; then - echo "All instances terminated" - break - else - echo "Instances not yet terminated: $instance_states" - echo "Sleeping for 3 seconds..." - sleep 3 - fi - done - fi - - echo "Deleting VPC" - vpc_id=$($AWS_CMD describe-vpcs | get_vpc_id) + vpc_id=$(get_vpc_id) if [[ -n "${vpc_id}" ]]; then local elb_ids=$(get_elbs_in_vpc ${vpc_id}) if [[ -n ${elb_ids} ]]; then @@ -741,6 +728,27 @@ function kube-down { done fi + echo "Deleting instances in VPC: ${vpc_id}" + instance_ids=$($AWS_CMD --output text describe-instances \ + --filters Name=vpc-id,Values=${vpc_id} \ + Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ + --query Reservations[].Instances[].InstanceId) + if [[ -n ${instance_ids} ]]; then + $AWS_CMD terminate-instances --instance-ids $instance_ids > $LOG + echo "Waiting for instances to be deleted" + while true; do + instance_states=$($AWS_CMD describe-instances --instance-ids $instance_ids | expect_instance_states terminated) + if [[ "$instance_states" == "" ]]; then + echo "All instances deleted" + break + else + echo "Instances not yet deleted: $instance_states" + echo "Sleeping for 3 seconds..." + sleep 3 + fi + done + fi + echo "Deleting VPC: ${vpc_id}" default_sg_id=$($AWS_CMD --output text describe-security-groups \ --filters Name=vpc-id,Values=$vpc_id Name=group-name,Values=default \