From 661d6bde6fb2c59393d4742d23c71c67f3b5b5fc Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Tue, 11 Oct 2016 16:50:30 -0700 Subject: [PATCH] Delete all firewall rules and network on GCE/GKE test cluster teardown --- cluster/gce/config-default.sh | 1 + cluster/gce/config-test.sh | 1 + cluster/gce/util.sh | 65 +++++++++++++++++++---------------- cluster/gke/config-default.sh | 2 ++ cluster/gke/config-test.sh | 1 + cluster/gke/util.sh | 22 +++++++++--- 6 files changed, 59 insertions(+), 33 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 728e20fea71..f3385382c02 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -35,6 +35,7 @@ REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-true} PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false} PREEMPTIBLE_MASTER=${PREEMPTIBLE_MASTER:-false} KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true} +KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false} MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 1e32dac84fd..0a0fae84ebb 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -36,6 +36,7 @@ KUBE_APISERVER_REQUEST_TIMEOUT=300 PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false} PREEMPTIBLE_MASTER=${PREEMPTIBLE_MASTER:-false} KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true} +KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-true} MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index f7ea12130bc..0edf9f9c095 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -696,6 +696,27 @@ function create-network() { fi } +function delete-firewall-rules() { + for fw in $@; do + if [[ -n $(gcloud compute firewall-rules --project "${PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then + gcloud compute firewall-rules delete --project "${PROJECT}" --quiet "${fw}" & + fi + done + kube::util::wait-for-jobs || { + echo -e "${color_red}Failed to delete firewall rules.${color_norm}" >&2 + } +} + +function delete-network() { + if [[ -n $(gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then + if ! gcloud compute networks delete --project "${PROJECT}" --quiet "${NETWORK}"; then + echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:" + gcloud compute firewall-rules --project "${PROJECT}" list --filter="network=${NETWORK}" + return 1 + fi + fi +} + # Assumes: # NUM_NODES # Sets: @@ -1273,13 +1294,8 @@ function kube-down() { # If there are no more remaining master replicas, we should delete all remaining network resources. if [[ "${REMAINING_MASTER_COUNT}" == "0" ]]; then - # Delete firewall rule for the master. - if gcloud compute firewall-rules describe --project "${PROJECT}" "${MASTER_NAME}-https" &>/dev/null; then - gcloud compute firewall-rules delete \ - --project "${PROJECT}" \ - --quiet \ - "${MASTER_NAME}-https" - fi + # Delete firewall rule for the master, etcd servers, and nodes. + delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all" # Delete the master's reserved IP if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then gcloud compute addresses delete \ @@ -1288,20 +1304,6 @@ function kube-down() { --quiet \ "${MASTER_NAME}-ip" fi - # Delete firewall rule for minions. - if gcloud compute firewall-rules describe --project "${PROJECT}" "${NODE_TAG}-all" &>/dev/null; then - gcloud compute firewall-rules delete \ - --project "${PROJECT}" \ - --quiet \ - "${NODE_TAG}-all" - fi - # Delete firewall rule for etcd servers. - if gcloud compute firewall-rules --project "${PROJECT}" describe "${MASTER_NAME}-etcd" &>/dev/null; then - gcloud compute firewall-rules delete \ - --project "${PROJECT}" \ - --quiet \ - "${MASTER_NAME}-etcd" - fi fi if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then @@ -1353,6 +1355,16 @@ function kube-down() { "${INSTANCE_PREFIX}"-influxdb-pd fi + # Delete all remaining firewall rules and network. + delete-firewall-rules \ + "${NETWORK}-default-internal-master" \ + "${NETWORK}-default-internal-node" \ + "${NETWORK}-default-ssh" \ + "${NETWORK}-default-internal" # Pre-1.5 clusters + if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then + delete-network + fi + # If there are no more remaining master replicas, we should update kubeconfig. if [[ "${REMAINING_MASTER_COUNT}" == "0" ]]; then export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}" @@ -1721,14 +1733,9 @@ function test-setup() { function test-teardown() { detect-project echo "Shutting down test cluster in background." - gcloud compute firewall-rules delete \ - --project "${PROJECT}" \ - --quiet \ - "${NODE_TAG}-${INSTANCE_PREFIX}-http-alt" || true - gcloud compute firewall-rules delete \ - --project "${PROJECT}" \ - --quiet \ - "${NODE_TAG}-${INSTANCE_PREFIX}-nodeports" || true + delete-firewall-rules \ + "${NODE_TAG}-${INSTANCE_PREFIX}-http-alt" \ + "${NODE_TAG}-${INSTANCE_PREFIX}-nodeports" if [[ ${MULTIZONE:-} == "true" ]]; then local zones=( ${E2E_ZONES} ) # tear them down in reverse order, finally tearing down the master too. diff --git a/cluster/gke/config-default.sh b/cluster/gke/config-default.sh index 03aa6f66e12..46697fa67e4 100644 --- a/cluster/gke/config-default.sh +++ b/cluster/gke/config-default.sh @@ -40,3 +40,5 @@ ENABLE_L7_LOADBALANCING="${KUBE_ENABLE_L7_LOADBALANCING:-glbc}" # google - Heapster, Google Cloud Monitoring, and Google Cloud Logging # standalone - Heapster only. Metrics available via Heapster REST API. ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-standalone}" + +KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false} diff --git a/cluster/gke/config-test.sh b/cluster/gke/config-test.sh index 9070f44c5d0..fea85e3f068 100644 --- a/cluster/gke/config-test.sh +++ b/cluster/gke/config-test.sh @@ -20,6 +20,7 @@ NETWORK=${KUBE_GKE_NETWORK:-e2e} NODE_TAG="k8s-${CLUSTER_NAME}-node" IMAGE_TYPE="${KUBE_GKE_IMAGE_TYPE:-container_vm}" +KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-true} # For ease of maintenance, extract any pieces that do not vary between default # and test in a common config. diff --git a/cluster/gke/util.sh b/cluster/gke/util.sh index 7544eb50d71..3b3e4245d88 100755 --- a/cluster/gke/util.sh +++ b/cluster/gke/util.sh @@ -370,13 +370,27 @@ function test-teardown() { # instances, but we can safely delete the cluster before the firewall. # # NOTE: Keep in sync with names above in test-setup. - "${GCLOUD}" compute firewall-rules delete "${CLUSTER_NAME}-http-alt" \ - --project="${PROJECT}" & - "${GCLOUD}" compute firewall-rules delete "${CLUSTER_NAME}-nodeports" \ - --project="${PROJECT}" & + for fw in "${CLUSTER_NAME}-http-alt" "${CLUSTER_NAME}-nodeports" "${FIREWALL_SSH}"; do + if [[ -n $("${GCLOUD}" compute firewall-rules --project "${PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then + "${GCLOUD}" compute firewall-rules delete "${fw}" --project="${PROJECT}" --quiet & + fi + done # Wait for firewall rule teardown. kube::util::wait-for-jobs || true + + # It's unfortunate that the $FIREWALL_SSH rule and network are created in + # kube-up, but we can only really delete them in test-teardown. So much for + # symmetry. + if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then + if [[ -n $("${GCLOUD}" compute networks --project "${PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then + if ! "${GCLOUD}" compute networks delete --project "${PROJECT}" --quiet "${NETWORK}"; then + echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:" + "${GCLOUD}" compute firewall-rules --project "${PROJECT}" list --filter="network=${NETWORK}" + return 1 + fi + fi + fi } # Actually take down the cluster. This is called from test-teardown.