Merge pull request #34577 from ixdy/cleanup-network

Automatic merge from submit-queue

Delete all firewall rules (and optionally network) on GCE/GKE cluster teardown

Not entirely ready for review yet; I want to see what Jenkins thinks of this.

cluster/gce/config-default.sh

@@ -35,6 +35,7 @@ REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-true}
 PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false}
 PREEMPTIBLE_MASTER=${PREEMPTIBLE_MASTER:-false}
 KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true}
+KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false}
 
 MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
 NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}

cluster/gce/config-test.sh

@@ -36,6 +36,7 @@ KUBE_APISERVER_REQUEST_TIMEOUT=300
 PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false}
 PREEMPTIBLE_MASTER=${PREEMPTIBLE_MASTER:-false}
 KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true}
+KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-true}
 
 MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
 NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}

cluster/gce/util.sh

@@ -696,6 +696,27 @@ function create-network() {
   fi
 }
 
+function delete-firewall-rules() {
+  for fw in $@; do
+    if [[ -n $(gcloud compute firewall-rules --project "${PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then
+      gcloud compute firewall-rules delete --project "${PROJECT}" --quiet "${fw}" &
+    fi
+  done
+  kube::util::wait-for-jobs || {
+    echo -e "${color_red}Failed to delete firewall rules.${color_norm}" >&2
+  }
+}
+
+function delete-network() {
+  if [[ -n $(gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then
+    if ! gcloud compute networks delete --project "${PROJECT}" --quiet "${NETWORK}"; then
+      echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:"
+      gcloud compute firewall-rules --project "${PROJECT}" list --filter="network=${NETWORK}"
+      return 1
+    fi
+  fi
+}
+
 # Assumes:
 #   NUM_NODES
 # Sets:
@@ -1273,13 +1294,8 @@ function kube-down() {
 
   # If there are no more remaining master replicas, we should delete all remaining network resources.
   if [[ "${REMAINING_MASTER_COUNT}" == "0" ]]; then
-    # Delete firewall rule for the master.
+    # Delete firewall rule for the master, etcd servers, and nodes.
-    if gcloud compute firewall-rules describe --project "${PROJECT}" "${MASTER_NAME}-https" &>/dev/null; then
+    delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all"
-      gcloud compute firewall-rules delete  \
-        --project "${PROJECT}" \
-        --quiet \
-        "${MASTER_NAME}-https"
-    fi
     # Delete the master's reserved IP
     if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
       gcloud compute addresses delete \
@@ -1288,20 +1304,6 @@ function kube-down() {
         --quiet \
         "${MASTER_NAME}-ip"
     fi
-    # Delete firewall rule for minions.
-    if gcloud compute firewall-rules describe --project "${PROJECT}" "${NODE_TAG}-all" &>/dev/null; then
-      gcloud compute firewall-rules delete  \
-        --project "${PROJECT}" \
-        --quiet \
-        "${NODE_TAG}-all"
-    fi
-    # Delete firewall rule for etcd servers.
-    if gcloud compute firewall-rules --project "${PROJECT}" describe "${MASTER_NAME}-etcd" &>/dev/null; then
-      gcloud compute firewall-rules delete  \
-        --project "${PROJECT}" \
-        --quiet \
-        "${MASTER_NAME}-etcd"
-    fi
   fi
 
   if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then
@@ -1353,6 +1355,16 @@ function kube-down() {
       "${INSTANCE_PREFIX}"-influxdb-pd
   fi
 
+  # Delete all remaining firewall rules and network.
+  delete-firewall-rules \
+    "${NETWORK}-default-internal-master" \
+    "${NETWORK}-default-internal-node" \
+    "${NETWORK}-default-ssh" \
+    "${NETWORK}-default-internal"  # Pre-1.5 clusters
+  if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then
+    delete-network
+  fi
+
   # If there are no more remaining master replicas, we should update kubeconfig.
   if [[ "${REMAINING_MASTER_COUNT}" == "0" ]]; then
     export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
@@ -1373,7 +1385,7 @@ function kube-down() {
 function get-replica-name() {
   echo $(gcloud compute instances list \
     --project "${PROJECT}" \
-    --zone "${ZONE}" \
+    --zones "${ZONE}" \
     --regexp "$(get-replica-name-regexp)" \
     --format "value(name)" | head -n1)
 }
@@ -1721,14 +1733,9 @@ function test-setup() {
 function test-teardown() {
   detect-project
   echo "Shutting down test cluster in background."
-  gcloud compute firewall-rules delete  \
+  delete-firewall-rules \
-    --project "${PROJECT}" \
+    "${NODE_TAG}-${INSTANCE_PREFIX}-http-alt" \
-    --quiet \
+    "${NODE_TAG}-${INSTANCE_PREFIX}-nodeports"
-    "${NODE_TAG}-${INSTANCE_PREFIX}-http-alt" || true
-  gcloud compute firewall-rules delete  \
-    --project "${PROJECT}" \
-    --quiet \
-    "${NODE_TAG}-${INSTANCE_PREFIX}-nodeports" || true
   if [[ ${MULTIZONE:-} == "true" ]]; then
       local zones=( ${E2E_ZONES} )
       # tear them down in reverse order, finally tearing down the master too.

cluster/gke/config-default.sh

@@ -40,3 +40,5 @@ ENABLE_L7_LOADBALANCING="${KUBE_ENABLE_L7_LOADBALANCING:-glbc}"
 #   google   - Heapster, Google Cloud Monitoring, and Google Cloud Logging
 #   standalone     - Heapster only. Metrics available via Heapster REST API.
 ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-standalone}"
+
+KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false}

cluster/gke/config-test.sh

@@ -20,6 +20,7 @@ NETWORK=${KUBE_GKE_NETWORK:-e2e}
 NODE_TAG="k8s-${CLUSTER_NAME}-node"
 IMAGE_TYPE="${KUBE_GKE_IMAGE_TYPE:-container_vm}"
 
+KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-true}
 
 # For ease of maintenance, extract any pieces that do not vary between default
 # and test in a common config.

cluster/gke/util.sh

@@ -370,13 +370,27 @@ function test-teardown() {
   # instances, but we can safely delete the cluster before the firewall.
   #
   # NOTE: Keep in sync with names above in test-setup.
-  "${GCLOUD}" compute firewall-rules delete "${CLUSTER_NAME}-http-alt" \
+  for fw in "${CLUSTER_NAME}-http-alt" "${CLUSTER_NAME}-nodeports" "${FIREWALL_SSH}"; do
-    --project="${PROJECT}" &
+    if [[ -n $("${GCLOUD}" compute firewall-rules --project "${PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then
-  "${GCLOUD}" compute firewall-rules delete "${CLUSTER_NAME}-nodeports" \
+      "${GCLOUD}" compute firewall-rules delete "${fw}" --project="${PROJECT}" --quiet &
-    --project="${PROJECT}" &
+    fi
+  done
 
   # Wait for firewall rule teardown.
   kube::util::wait-for-jobs || true
+
+  # It's unfortunate that the $FIREWALL_SSH rule and network are created in
+  # kube-up, but we can only really delete them in test-teardown. So much for
+  # symmetry.
+  if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then
+    if [[ -n $("${GCLOUD}" compute networks --project "${PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then
+      if ! "${GCLOUD}" compute networks delete --project "${PROJECT}" --quiet "${NETWORK}"; then
+        echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:"
+        "${GCLOUD}" compute firewall-rules --project "${PROJECT}" list --filter="network=${NETWORK}"
+        return 1
+      fi
+    fi
+  fi
 }
 
 # Actually take down the cluster. This is called from test-teardown.