diff --git a/cluster/gce/list-resources.sh b/cluster/gce/list-resources.sh new file mode 100755 index 00000000000..9c0d9c0420b --- /dev/null +++ b/cluster/gce/list-resources.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Calls gcloud to print out a variety of Google Cloud Platform resources used by +# Kubernetes. Can be run before/after test runs and compared to track leaking +# resources. + +# PROJECT must be set in the environment. +# If ZONE, KUBE_GCE_INSTANCE_PREFIX, CLUSTER_NAME, KUBE_GCE_NETWORK, or +# KUBE_GKE_NETWORK is set, they will be used to filter the results. + +set -o errexit +set -o nounset +set -o pipefail + +ZONE=${ZONE:-} +REGION=${ZONE%-*} +INSTANCE_PREFIX=${KUBE_GCE_INSTANCE_PREFIX:-${CLUSTER_NAME:-}} +NETWORK=${KUBE_GCE_NETWORK:-${KUBE_GKE_NETWORK:-}} + +# Usage: gcloud-compute-list +# GREP_REGEX is applied to the output of gcloud if set +GREP_REGEX="" +function gcloud-compute-list() { + local -r resource=$1 + echo -e "\n\n[ ${resource} ]" + gcloud compute ${resource} list --project=${PROJECT} ${@:2} | grep "${GREP_REGEX}" +} + +echo "Project: ${PROJECT}" +echo "Region: ${REGION}" +echo "Zone: ${ZONE}" +echo "Instance prefix: ${INSTANCE_PREFIX:-}" +echo "Network: ${NETWORK:-}" + +# List resources related to instances, filtering by the instance prefix if +# provided. +gcloud-compute-list instance-templates --regexp="${INSTANCE_PREFIX}.*" +gcloud-compute-list instance-groups ${ZONE:+"--zone=${ZONE}"} --regexp="${INSTANCE_PREFIX}.*" +gcloud-compute-list instances ${ZONE:+"--zone=${ZONE}"} --regexp="${INSTANCE_PREFIX}.*" + +# List disk resources, filterying by instance prefix if provided. +gcloud-compute-list disks ${ZONE:+"--zone=${ZONE}"} --regexp="${INSTANCE_PREFIX}.*" + +# List network resources. We include names starting with "a", corresponding to +# those that Kubernetes creates. +gcloud-compute-list addresses ${REGION:+"--region=${REGION}"} --regexp="a.*|${INSTANCE_PREFIX}.*" +# Match either the header or a line with the specified e2e network. +# This assumes that the network name is the second field in the output. +GREP_REGEX="^NAME\|^[^\s]\+\s\+\(default\|${NETWORK}\)\s" +gcloud-compute-list routes --regexp="default.*|${INSTANCE_PREFIX}.*" +gcloud-compute-list firewall-rules --regexp="default.*|k8s-fw.*|${INSTANCE_PREFIX}.*" +GREP_REGEX="" +gcloud-compute-list forwarding-rules ${REGION:+"--region=${REGION}"} +gcloud-compute-list target-pools ${REGION:+"--region=${REGION}"} diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index bd3b0cc2abb..ff7c2da6bf0 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -952,6 +952,9 @@ cd kubernetes ARTIFACTS=${WORKSPACE}/_artifacts mkdir -p ${ARTIFACTS} export E2E_REPORT_DIR=${ARTIFACTS} +declare -r gcp_resources_before="${ARTIFACTS}/gcp-resources-before.txt" +declare -r gcp_resources_cluster_up="${ARTIFACTS}/gcp-resources-cluster-up.txt" +declare -r gcp_resources_after="${ARTIFACTS}/gcp-resources-after.txt" ### Pre Set Up ### # Install gcloud from a custom path if provided. Used to test GKE with gcloud @@ -971,8 +974,14 @@ fi ### Set up ### if [[ "${E2E_UP,,}" == "true" ]]; then go run ./hack/e2e.go ${E2E_OPT} -v --down + if [[ ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ]]; then + ./cluster/gce/list-resources.sh > "${gcp_resources_before}" + fi go run ./hack/e2e.go ${E2E_OPT} -v --up go run ./hack/e2e.go -v --ctl="version --match-server-version=false" + if [[ ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ]]; then + ./cluster/gce/list-resources.sh > "${gcp_resources_cluster_up}" + fi fi ### Run tests ### @@ -1005,14 +1014,6 @@ if [[ "${USE_KUBEMARK:-}" == "true" ]]; then unset MASTER_SIZE_BKP fi -# TODO(zml): We have a bunch of legacy Jenkins configs that are -# expecting junit*.xml to be in ${WORKSPACE} root and it's Friday -# afternoon, so just put the junit report where it's expected. -# If link already exists, non-zero return code should not cause build to fail. -for junit in ${ARTIFACTS}/junit*.xml; do - ln -s -f ${junit} ${WORKSPACE} || true -done - ### Clean up ### if [[ "${E2E_DOWN,,}" == "true" ]]; then # Sleep before deleting the cluster to give the controller manager time to @@ -1023,4 +1024,11 @@ if [[ "${E2E_DOWN,,}" == "true" ]]; then # for the wait between attempts. sleep 30 go run ./hack/e2e.go ${E2E_OPT} -v --down + if [[ ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ]]; then + ./cluster/gce/list-resources.sh > "${gcp_resources_after}" + fi +fi + +if [[ -f "${gcp_resources_before}" && -f "${gcp_resources_after}" ]]; then + diff -sw -U0 -F'^\[.*\]$' "${gcp_resources_before}" "${gcp_resources_after}" || true fi