From dc070254704d4aafad0cf7a04c84c90a8bc896f4 Mon Sep 17 00:00:00 2001
From: Mike Spreitzer <mspreitz@us.ibm.com>
Date: Fri, 5 Nov 2021 19:58:48 +0000
Subject: [PATCH] Add periodic etcd scraping to integration tests

.. to help understand where and why things go bad.
---
 hack/lib/etcd.sh                    | 28 ++++++++++++++++++++++++++++
 hack/make-rules/test-integration.sh |  3 +++
 2 files changed, 31 insertions(+)

diff --git a/hack/lib/etcd.sh b/hack/lib/etcd.sh
index e7a62cab15d..84518695f1c 100755
--- a/hack/lib/etcd.sh
+++ b/hack/lib/etcd.sh
@@ -91,7 +91,35 @@ kube::etcd::start() {
   curl -fs -X POST "${KUBE_INTEGRATION_ETCD_URL}/v3/kv/put" -d '{"key": "X3Rlc3Q=", "value": ""}'
 }
 
+kube::etcd::start_scraping() {
+  if [[ -d "${ARTIFACTS:-}" ]]; then
+    ETCD_SCRAPE_DIR="${ARTIFACTS}/etcd-scrapes"
+  else
+    ETCD_SCRAPE_DIR=$(mktemp -d -t test-etcd-scrapes.XXXXXX)
+  fi
+  kube::log::info "Periodically scraping etcd to ${ETCD_SCRAPE_DIR} ."
+  mkdir -p "${ETCD_SCRAPE_DIR}"
+  (
+    while sleep 30; do
+      kube::etcd::scrape
+    done
+  ) &
+  ETCD_SCRAPE_PID=$!
+}
+
+kube::etcd::scrape() {
+    curl -s -S "${KUBE_INTEGRATION_ETCD_URL}/metrics" > "${ETCD_SCRAPE_DIR}/next" && mv "${ETCD_SCRAPE_DIR}/next" "${ETCD_SCRAPE_DIR}/$(date +%H%M%S).scrape"
+}
+
+
 kube::etcd::stop() {
+  if [[ -n "${ETCD_SCRAPE_PID:-}" ]]; then
+    kill "${ETCD_SCRAPE_PID}" &>/dev/null || :
+    wait "${ETCD_SCRAPE_PID}" &>/dev/null || :
+    kube::etcd::scrape || :
+    # shellcheck disable=SC2015
+    tar czf "${ETCD_SCRAPE_DIR}/scrapes.tgz" "${ETCD_SCRAPE_DIR}"/*.scrape && rm "${ETCD_SCRAPE_DIR}"/*.scrape || :
+  fi
   if [[ -n "${ETCD_PID-}" ]]; then
     kill "${ETCD_PID}" &>/dev/null || :
     wait "${ETCD_PID}" &>/dev/null || :
diff --git a/hack/make-rules/test-integration.sh b/hack/make-rules/test-integration.sh
index 9707fefd37d..68789119280 100755
--- a/hack/make-rules/test-integration.sh
+++ b/hack/make-rules/test-integration.sh
@@ -64,6 +64,9 @@ runTests() {
   kube::log::status "Starting etcd instance"
   CLEANUP_REQUIRED=1
   kube::etcd::start
+  # shellcheck disable=SC2034
+  local ETCD_SCRAPE_PID # Set in kube::etcd::start_scraping, used in cleanup
+  kube::etcd::start_scraping
   kube::log::status "Running integration test cases"
 
   make -C "${KUBE_ROOT}" test \