diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 53093d55f9d..38b22f4c7ad 100755 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -64,9 +64,9 @@ esac AWS_REGION=${ZONE%?} export AWS_DEFAULT_REGION=${AWS_REGION} -AWS_CMD="aws --output json ec2" -AWS_ELB_CMD="aws --output json elb" -AWS_ASG_CMD="aws --output json autoscaling" +export AWS_DEFAULT_OUTPUT=text +AWS_CMD="aws ec2" +AWS_ASG_CMD="aws autoscaling" VPC_CIDR_BASE=172.20 MASTER_IP_SUFFIX=.9 @@ -88,15 +88,10 @@ BLOCK_DEVICE_MAPPINGS_BASE="{\"DeviceName\": \"/dev/sdc\",\"VirtualName\":\"ephe MASTER_BLOCK_DEVICE_MAPPINGS="[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":${MASTER_ROOT_DISK_SIZE},\"VolumeType\":\"${MASTER_ROOT_DISK_TYPE}\"}}, ${BLOCK_DEVICE_MAPPINGS_BASE}]" MINION_BLOCK_DEVICE_MAPPINGS="[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":${MINION_ROOT_DISK_SIZE},\"VolumeType\":\"${MINION_ROOT_DISK_TYPE}\"}}, ${BLOCK_DEVICE_MAPPINGS_BASE}]" -function json_val { - python -c 'import json,sys;obj=json.load(sys.stdin);print obj'$1'' -} - -# TODO (ayurchuk) Refactor the get_* functions to use filters # TODO (bburns) Parameterize this for multiple cluster per project function get_vpc_id { - $AWS_CMD --output text describe-vpcs \ + $AWS_CMD describe-vpcs \ --filters Name=tag:Name,Values=kubernetes-vpc \ Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ --query Vpcs[].VpcId @@ -105,7 +100,7 @@ function get_vpc_id { function get_subnet_id { local vpc_id=$1 local az=$2 - $AWS_CMD --output text describe-subnets \ + $AWS_CMD describe-subnets \ --filters Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ Name=availabilityZone,Values=${az} \ Name=vpc-id,Values=${vpc_id} \ @@ -114,24 +109,20 @@ function get_subnet_id { function get_igw_id { local vpc_id=$1 - $AWS_CMD --output text describe-internet-gateways \ + $AWS_CMD describe-internet-gateways \ --filters Name=attachment.vpc-id,Values=${vpc_id} \ --query InternetGateways[].InternetGatewayId } function get_elbs_in_vpc { - # ELB doesn't seem to be on the same platform as the rest of AWS; doesn't support filtering - $AWS_ELB_CMD describe-load-balancers | \ + # ELB doesn't seem to be on the same platform as the rest of AWS; doesn't support filtering + aws elb --output json describe-load-balancers | \ python -c "import json,sys; lst = [str(lb['LoadBalancerName']) for lb in json.load(sys.stdin)['LoadBalancerDescriptions'] if lb['VPCId'] == '$1']; print '\n'.join(lst)" } -function expect_instance_states { - python -c "import json,sys; lst = [str(instance['InstanceId']) for reservation in json.load(sys.stdin)['Reservations'] for instance in reservation['Instances'] if instance['State']['Name'] != '$1']; print ' '.join(lst)" -} - function get_instanceid_from_name { local tagName=$1 - $AWS_CMD --output text describe-instances \ + $AWS_CMD describe-instances \ --filters Name=tag:Name,Values=${tagName} \ Name=instance-state-name,Values=running \ Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ @@ -140,14 +131,14 @@ function get_instanceid_from_name { function get_instance_public_ip { local instance_id=$1 - $AWS_CMD --output text describe-instances \ + $AWS_CMD describe-instances \ --instance-ids ${instance_id} \ --query Reservations[].Instances[].NetworkInterfaces[0].Association.PublicIp } function get_instance_private_ip { local instance_id=$1 - $AWS_CMD --output text describe-instances \ + $AWS_CMD describe-instances \ --instance-ids ${instance_id} \ --query Reservations[].Instances[].NetworkInterfaces[0].PrivateIpAddress } @@ -155,7 +146,7 @@ function get_instance_private_ip { # Gets a security group id, by name ($1) function get_security_group_id { local name=$1 - $AWS_CMD --output text describe-security-groups \ + $AWS_CMD describe-security-groups \ --filters Name=vpc-id,Values=${VPC_ID} \ Name=group-name,Values=${name} \ Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ @@ -185,7 +176,7 @@ function detect-master () { function query-running-minions () { local query=$1 - $AWS_CMD --output text describe-instances \ + $AWS_CMD describe-instances \ --filters Name=instance-state-name,Values=running \ Name=vpc-id,Values=${VPC_ID} \ Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ @@ -391,7 +382,7 @@ function create-security-group { local sgid=$(get_security_group_id "${name}") if [[ -z "$sgid" ]]; then echo "Creating security group ${name}." - sgid=$($AWS_CMD create-security-group --group-name "${name}" --description "${description}" --vpc-id "${VPC_ID}" --query GroupId --output text) + sgid=$($AWS_CMD create-security-group --group-name "${name}" --description "${description}" --vpc-id "${VPC_ID}" --query GroupId) add-tag $sgid KubernetesCluster ${CLUSTER_ID} fi } @@ -421,7 +412,7 @@ function authorize-security-group-ingress { function find-master-pd { local name=${MASTER_NAME}-pd if [[ -z "${MASTER_DISK_ID}" ]]; then - MASTER_DISK_ID=`$AWS_CMD --output text describe-volumes \ + MASTER_DISK_ID=`$AWS_CMD describe-volumes \ --filters Name=availability-zone,Values=${ZONE} \ Name=tag:Name,Values=${name} \ Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ @@ -438,7 +429,7 @@ function ensure-master-pd { if [[ -z "${MASTER_DISK_ID}" ]]; then echo "Creating master disk: size ${MASTER_DISK_SIZE}GB, type ${MASTER_DISK_TYPE}" - MASTER_DISK_ID=`$AWS_CMD create-volume --availability-zone ${ZONE} --volume-type ${MASTER_DISK_TYPE} --size ${MASTER_DISK_SIZE} --query VolumeId --output text` + MASTER_DISK_ID=`$AWS_CMD create-volume --availability-zone ${ZONE} --volume-type ${MASTER_DISK_TYPE} --size ${MASTER_DISK_SIZE} --query VolumeId` add-tag ${MASTER_DISK_ID} Name ${name} add-tag ${MASTER_DISK_ID} KubernetesCluster ${CLUSTER_ID} fi @@ -456,12 +447,12 @@ function create-dhcp-option-set () { OPTION_SET_DOMAIN="${AWS_REGION}.compute.internal" esac - DHCP_OPTION_SET_ID=$($AWS_CMD create-dhcp-options --dhcp-configuration Key=domain-name,Values=${OPTION_SET_DOMAIN} Key=domain-name-servers,Values=AmazonProvidedDNS | json_val '["DhcpOptions"]["DhcpOptionsId"]') + DHCP_OPTION_SET_ID=$($AWS_CMD create-dhcp-options --dhcp-configuration Key=domain-name,Values=${OPTION_SET_DOMAIN} Key=domain-name-servers,Values=AmazonProvidedDNS --query DhcpOptions.DhcpOptionsId) add-tag ${DHCP_OPTION_SET_ID} Name kubernetes-dhcp-option-set add-tag ${DHCP_OPTION_SET_ID} KubernetesCluster ${CLUSTER_ID} - $AWS_CMD associate-dhcp-options --dhcp-options-id ${DHCP_OPTION_SET_ID} --vpc-id ${VPC_ID} + $AWS_CMD associate-dhcp-options --dhcp-options-id ${DHCP_OPTION_SET_ID} --vpc-id ${VPC_ID} > $LOG echo "Using DHCP option set ${DHCP_OPTION_SET_ID}" } @@ -561,7 +552,7 @@ function upload-server-tars() { done fi - local s3_bucket_location=$(aws --output text s3api get-bucket-location --bucket ${AWS_S3_BUCKET}) + local s3_bucket_location=$(aws s3api get-bucket-location --bucket ${AWS_S3_BUCKET}) local s3_url_base=https://s3-${s3_bucket_location}.amazonaws.com if [[ "${s3_bucket_location}" == "None" ]]; then # "US Classic" does not follow the pattern @@ -641,15 +632,17 @@ function ensure-iam-profiles { } } -# Wait for instance to be in running state -function wait-for-instance-running { +# Wait for instance to be in specified state +function wait-for-instance-state { instance_id=$1 + state=$2 + while true; do - instance_state=$($AWS_CMD describe-instances --instance-ids ${instance_id} | expect_instance_states running) - if [[ "$instance_state" == "" ]]; then + instance_state=$($AWS_CMD describe-instances --instance-ids ${instance_id} --query Reservations[].Instances[].State.Name) + if [[ "$instance_state" == "${state}" ]]; then break else - echo "Waiting for instance ${instance_id} to spawn" + echo "Waiting for instance ${instance_id} to be ${state} (currently ${instance_state})" echo "Sleeping for 3 seconds..." sleep 3 fi @@ -659,22 +652,16 @@ function wait-for-instance-running { # Allocates new Elastic IP from Amazon # Output: allocated IP address function allocate-elastic-ip { - $AWS_CMD allocate-address --domain vpc --output text | cut -f3 + $AWS_CMD allocate-address --domain vpc --query PublicIp } function assign-ip-to-instance { local ip_address=$1 local instance_id=$2 - local fallback_ip=$3 - local elastic_ip_allocation_id=$($AWS_CMD describe-addresses --public-ips $ip_address --output text | cut -f2) - local association_result=$($AWS_CMD associate-address --instance-id ${master_instance_id} --allocation-id ${elastic_ip_allocation_id} > /dev/null && echo "success" || echo "failure") - - if [[ $association_result = "success" ]]; then - echo "${ip_address}" - else - echo "${fallback_ip}" - fi + local elastic_ip_allocation_id=$($AWS_CMD describe-addresses --public-ips $ip_address --query Addresses[].AllocationId) + echo "Attaching IP ${ip_address} to instance ${instance_id}" + $AWS_CMD associate-address --instance-id ${instance_id} --allocation-id ${elastic_ip_allocation_id} > $LOG } # If MASTER_RESERVED_IP looks like IP address, will try to assign it to master instance @@ -732,7 +719,7 @@ function kube-up { fi if [[ -z "$VPC_ID" ]]; then echo "Creating vpc." - VPC_ID=$($AWS_CMD create-vpc --cidr-block ${VPC_CIDR} | json_val '["Vpc"]["VpcId"]') + VPC_ID=$($AWS_CMD create-vpc --cidr-block ${VPC_CIDR} --query Vpc.VpcId) $AWS_CMD modify-vpc-attribute --vpc-id $VPC_ID --enable-dns-support '{"Value": true}' > $LOG $AWS_CMD modify-vpc-attribute --vpc-id $VPC_ID --enable-dns-hostnames '{"Value": true}' > $LOG add-tag $VPC_ID Name kubernetes-vpc @@ -749,12 +736,12 @@ function kube-up { if [[ -z "$SUBNET_ID" ]]; then echo "Creating subnet." - SUBNET_ID=$($AWS_CMD create-subnet --cidr-block ${SUBNET_CIDR} --vpc-id $VPC_ID --availability-zone ${ZONE} | json_val '["Subnet"]["SubnetId"]') + SUBNET_ID=$($AWS_CMD create-subnet --cidr-block ${SUBNET_CIDR} --vpc-id $VPC_ID --availability-zone ${ZONE} --query Subnet.SubnetId) add-tag $SUBNET_ID KubernetesCluster ${CLUSTER_ID} else - EXISTING_CIDR=$($AWS_CMD describe-subnets --subnet-ids ${SUBNET_ID} --query Subnets[].CidrBlock --output text) + EXISTING_CIDR=$($AWS_CMD describe-subnets --subnet-ids ${SUBNET_ID} --query Subnets[].CidrBlock) echo "Using existing subnet with CIDR $EXISTING_CIDR" - VPC_CIDR=$($AWS_CMD describe-vpcs --vpc-ids ${VPC_ID} --query Vpcs[].CidrBlock --output text) + VPC_CIDR=$($AWS_CMD describe-vpcs --vpc-ids ${VPC_ID} --query Vpcs[].CidrBlock) echo "VPC CIDR is $VPC_CIDR" VPC_CIDR_BASE=${VPC_CIDR%.*.*} MASTER_INTERNAL_IP=${VPC_CIDR_BASE}.0${MASTER_IP_SUFFIX} @@ -766,20 +753,20 @@ function kube-up { IGW_ID=$(get_igw_id $VPC_ID) if [[ -z "$IGW_ID" ]]; then echo "Creating Internet Gateway." - IGW_ID=$($AWS_CMD create-internet-gateway | json_val '["InternetGateway"]["InternetGatewayId"]') + IGW_ID=$($AWS_CMD create-internet-gateway --query InternetGateway.InternetGatewayId) $AWS_CMD attach-internet-gateway --internet-gateway-id $IGW_ID --vpc-id $VPC_ID > $LOG fi echo "Using Internet Gateway $IGW_ID" echo "Associating route table." - ROUTE_TABLE_ID=$($AWS_CMD --output text describe-route-tables \ + ROUTE_TABLE_ID=$($AWS_CMD describe-route-tables \ --filters Name=vpc-id,Values=${VPC_ID} \ Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ --query RouteTables[].RouteTableId) if [[ -z "${ROUTE_TABLE_ID}" ]]; then echo "Creating route table" - ROUTE_TABLE_ID=$($AWS_CMD --output text create-route-table \ + ROUTE_TABLE_ID=$($AWS_CMD create-route-table \ --vpc-id=${VPC_ID} \ --query RouteTable.RouteTableId) add-tag ${ROUTE_TABLE_ID} KubernetesCluster ${CLUSTER_ID} @@ -920,7 +907,8 @@ function start-master() { --security-group-ids ${MASTER_SG_ID} \ --associate-public-ip-address \ --block-device-mappings "${MASTER_BLOCK_DEVICE_MAPPINGS}" \ - --user-data file://${KUBE_TEMP}/master-start.sh | json_val '["Instances"][0]["InstanceId"]') + --user-data file://${KUBE_TEMP}/master-start.sh \ + --query Instances[].InstanceId) add-tag $master_id Name $MASTER_NAME add-tag $master_id Role $MASTER_TAG add-tag $master_id KubernetesCluster ${CLUSTER_ID} @@ -941,7 +929,7 @@ function start-master() { fi else # We are not able to add an elastic ip, a route or volume to the instance until that instance is in "running" state. - wait-for-instance-running $master_id + wait-for-instance-state ${master_id} "running" KUBE_MASTER=${MASTER_NAME} KUBE_MASTER_IP=$(assign-elastic-ip $ip $master_id) @@ -1185,7 +1173,7 @@ function kube-down { if [[ -n "${elb_ids}" ]]; then echo "Deleting ELBs in: ${vpc_id}" for elb_id in ${elb_ids}; do - $AWS_ELB_CMD delete-load-balancer --load-balancer-name=${elb_id} + aws elb delete-load-balancer --load-balancer-name=${elb_id} >$LOG done echo "Waiting for ELBs to be deleted" @@ -1203,21 +1191,21 @@ function kube-down { fi echo "Deleting instances in VPC: ${vpc_id}" - instance_ids=$($AWS_CMD --output text describe-instances \ + instance_ids=$($AWS_CMD describe-instances \ --filters Name=vpc-id,Values=${vpc_id} \ Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ --query Reservations[].Instances[].InstanceId) if [[ -n "${instance_ids}" ]]; then - asg_groups=$($AWS_CMD --output text describe-instances \ + asg_groups=$($AWS_CMD describe-instances \ --query 'Reservations[].Instances[].Tags[?Key==`aws:autoscaling:groupName`].Value[]' \ --instance-ids ${instance_ids}) for asg_group in ${asg_groups}; do - if [[ -n $(${AWS_ASG_CMD} --output text describe-auto-scaling-groups --auto-scaling-group-names ${asg_group} --query AutoScalingGroups[].AutoScalingGroupName) ]]; then + if [[ -n $(${AWS_ASG_CMD} describe-auto-scaling-groups --auto-scaling-group-names ${asg_group} --query AutoScalingGroups[].AutoScalingGroupName) ]]; then echo "Deleting auto-scaling group: ${asg_group}" ${AWS_ASG_CMD} delete-auto-scaling-group --force-delete --auto-scaling-group-name ${asg_group} fi - if [[ -n $(${AWS_ASG_CMD} --output text describe-launch-configurations --launch-configuration-names ${asg_group} --query LaunchConfigurations[].LaunchConfigurationName) ]]; then + if [[ -n $(${AWS_ASG_CMD} describe-launch-configurations --launch-configuration-names ${asg_group} --query LaunchConfigurations[].LaunchConfigurationName) ]]; then echo "Deleting auto-scaling launch configuration: ${asg_group}" ${AWS_ASG_CMD} delete-launch-configuration --launch-configuration-name ${asg_group} fi @@ -1225,26 +1213,19 @@ function kube-down { $AWS_CMD terminate-instances --instance-ids ${instance_ids} > $LOG echo "Waiting for instances to be deleted" - while true; do - local instance_states=$($AWS_CMD describe-instances --instance-ids ${instance_ids} | expect_instance_states terminated) - if [[ -z "${instance_states}" ]]; then - echo "All instances deleted" - break - else - echo "Instances not yet deleted: ${instance_states}" - echo "Sleeping for 3 seconds..." - sleep 3 - fi + for instance_id in ${instance_ids}; do + wait-for-instance-state ${instance_id} "terminated" done + echo "All instances deleted" fi - echo "Deleting VPC: ${vpc_id}" - default_sg_id=$($AWS_CMD --output text describe-security-groups \ + echo "Cleaning up resources in VPC: ${vpc_id}" + default_sg_id=$($AWS_CMD describe-security-groups \ --filters Name=vpc-id,Values=${vpc_id} \ Name=group-name,Values=default \ --query SecurityGroups[].GroupId \ | tr "\t" "\n") - sg_ids=$($AWS_CMD --output text describe-security-groups \ + sg_ids=$($AWS_CMD describe-security-groups \ --filters Name=vpc-id,Values=${vpc_id} \ Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ --query SecurityGroups[].GroupId \ @@ -1258,7 +1239,7 @@ function kube-down { fi echo "Cleaning up security group: ${sg_id}" - other_sgids=$(aws ec2 describe-security-groups --group-id "${sg_id}" --query SecurityGroups[].IpPermissions[].UserIdGroupPairs[].GroupId --output text) + other_sgids=$(${AWS_CMD} describe-security-groups --group-id "${sg_id}" --query SecurityGroups[].IpPermissions[].UserIdGroupPairs[].GroupId) for other_sgid in ${other_sgids}; do $AWS_CMD revoke-security-group-ingress --group-id "${sg_id}" --source-group "${other_sgid}" --protocol all > $LOG done @@ -1274,7 +1255,7 @@ function kube-down { $AWS_CMD delete-security-group --group-id ${sg_id} > $LOG done - subnet_ids=$($AWS_CMD --output text describe-subnets \ + subnet_ids=$($AWS_CMD describe-subnets \ --filters Name=vpc-id,Values=${vpc_id} \ Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ --query Subnets[].SubnetId \ @@ -1283,7 +1264,7 @@ function kube-down { $AWS_CMD delete-subnet --subnet-id ${subnet_id} > $LOG done - igw_ids=$($AWS_CMD --output text describe-internet-gateways \ + igw_ids=$($AWS_CMD describe-internet-gateways \ --filters Name=attachment.vpc-id,Values=${vpc_id} \ --query InternetGateways[].InternetGatewayId \ | tr "\t" "\n") @@ -1292,7 +1273,7 @@ function kube-down { $AWS_CMD delete-internet-gateway --internet-gateway-id $igw_id > $LOG done - route_table_ids=$($AWS_CMD --output text describe-route-tables \ + route_table_ids=$($AWS_CMD describe-route-tables \ --filters Name=vpc-id,Values=$vpc_id \ Name=route.destination-cidr-block,Values=0.0.0.0/0 \ --query RouteTables[].RouteTableId \ @@ -1300,7 +1281,7 @@ function kube-down { for route_table_id in ${route_table_ids}; do $AWS_CMD delete-route --route-table-id $route_table_id --destination-cidr-block 0.0.0.0/0 > $LOG done - route_table_ids=$($AWS_CMD --output text describe-route-tables \ + route_table_ids=$($AWS_CMD describe-route-tables \ --filters Name=vpc-id,Values=$vpc_id \ Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \ --query RouteTables[].RouteTableId \