From 8faa88626d4eb2c30286d9e5b7801edbe54a6a12 Mon Sep 17 00:00:00 2001
From: Daniel Smith <dbsmith@google.com>
Date: Fri, 3 Jun 2016 15:13:26 -0700
Subject: [PATCH] Revamp doc generation

Auto generated docs are **NO LONGER CHECKED IN**, only placeholders.

To generate them, e.g. before exporting docs, run hack/generate-docs.sh.

hack/verify-generated-docs.sh ensures that generated docs are merely the
placeholder text.

hack/update-generated-docs.sh puts the placeholder text in the proper
places.

The old munge behavior is moved into hack/{update|verify}-munge-docs.sh.
---
 build/common.sh                    |  1 -
 hack/autogenerated_placeholder.txt |  3 ++
 hack/generate-docs.sh              | 52 ++++++++++++++++++++++
 hack/lib/util.sh                   | 45 +++++++++++--------
 hack/update-generated-docs.sh      | 46 +++++--------------
 hack/update-munge-docs.sh          | 64 +++++++++++++++++++++++++++
 hack/verify-generated-docs.sh      | 64 ++++++++-------------------
 hack/verify-munge-docs.sh          | 71 ++++++++++++++++++++++++++++++
 8 files changed, 244 insertions(+), 102 deletions(-)
 create mode 100644 hack/autogenerated_placeholder.txt
 create mode 100755 hack/generate-docs.sh
 create mode 100755 hack/update-munge-docs.sh
 create mode 100755 hack/verify-munge-docs.sh

diff --git a/build/common.sh b/build/common.sh
index 63b52865b39..46ee1863143 100755
--- a/build/common.sh
+++ b/build/common.sh
@@ -483,7 +483,6 @@ function kube::build::source_targets() {
     third_party
     vendor
     contrib/mesos
-    .generated_docs
   )
   if [ -n "${KUBERNETES_CONTRIB:-}" ]; then
     for contrib in "${KUBERNETES_CONTRIB}"; do
diff --git a/hack/autogenerated_placeholder.txt b/hack/autogenerated_placeholder.txt
new file mode 100644
index 00000000000..b6fd7a0f989
--- /dev/null
+++ b/hack/autogenerated_placeholder.txt
@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.
diff --git a/hack/generate-docs.sh b/hack/generate-docs.sh
new file mode 100755
index 00000000000..19ee1c83854
--- /dev/null
+++ b/hack/generate-docs.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is not intended to be run automatically. It is meant to be run
+# immediately before exporting docs. We do not want to check these documents in
+# by default.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+kube::golang::setup_env
+
+"${KUBE_ROOT}/hack/build-go.sh" \
+    cmd/gendocs \
+    cmd/genkubedocs \
+    cmd/genman \
+    cmd/genyaml \
+    federation/cmd/genfeddocs
+
+kube::util::ensure-temp-dir
+
+kube::util::gen-docs "${KUBE_TEMP}"
+
+# remove all of the old docs
+kube::util::remove-gen-docs
+
+# Copy fresh docs into the repo.
+# the shopt is so that we get .generated_docs from the glob.
+shopt -s dotglob
+cp -af "${KUBE_TEMP}"/* "${KUBE_ROOT}"
+shopt -u dotglob
+
+echo "Generated docs have been placed in the repository tree. Running hack/update-munge-docs.sh."
+
+"${KUBE_ROOT}/hack/update-munge-docs.sh"
diff --git a/hack/lib/util.sh b/hack/lib/util.sh
index e38c9b3b343..6be0cebc949 100755
--- a/hack/lib/util.sh
+++ b/hack/lib/util.sh
@@ -208,27 +208,34 @@ kube::util::gen-docs() {
   touch .generated_docs
   find . -type f | cut -sd / -f 2- | LC_ALL=C sort > .generated_docs
   popd > /dev/null
+}
 
-  while read file; do
-    # Copy out of KUBE_ROOT if we didn't really change anything
-    if [[ -e "${dest}/${file}" && -e "${KUBE_ROOT}/${file}" ]]; then
-      # Filter all munges from original content.
-      original=$(cat "${KUBE_ROOT}/${file}" | sed '/^<!-- BEGIN MUNGE:.*/,/^<!-- END MUNGE:.*/d')
-      generated=$(cat "${dest}/${file}")
-
-      # Filter out meaningless lines with timestamps
-      original=$(echo "${original}" | grep -v "Auto generated by spf13/cobra" || :)
-      generated=$(echo "${generated}" | grep -v "Auto generated by spf13/cobra" || :)
-
-      # By now, the contents should be normalized and stripped of any
-      # auto-managed content.  We also ignore whitespace here because of
-      # markdown strictness fixups later in the pipeline.
-      if diff -Bw >/dev/null <(echo "${original}") <(echo "${generated}"); then
-        # actual contents same, overwrite generated with original.
-        cp "${KUBE_ROOT}/${file}" "${dest}/${file}"
+# Puts a placeholder for every generated doc. This makes the link checker work.
+kube::util::set-placeholder-gen-docs() {
+  local list_file="${KUBE_ROOT}/.generated_docs"
+  if [ -e ${list_file} ]; then
+    # remove all of the old docs; we don't want to check them in.
+    while read file; do
+      if [[ "${list_file}" != "${KUBE_ROOT}/${file}" ]]; then
+        cp "${KUBE_ROOT}/hack/autogenerated_placeholder.txt" "${KUBE_ROOT}/${file}"
       fi
-    fi
-  done <"${KUBE_ROOT}/.generated_docs"
+    done <"${list_file}"
+    # The .generated_docs file lists itself, so we don't need to explicitly
+    # delete it.
+  fi
+}
+
+# Removes previously generated docs-- we don't want to check them in. $KUBE_ROOT
+# must be set.
+kube::util::remove-gen-docs() {
+  if [ -e "${KUBE_ROOT}/.generated_docs" ]; then
+    # remove all of the old docs; we don't want to check them in.
+    while read file; do
+      rm "${KUBE_ROOT}/${file}" 2>/dev/null || true
+    done <"${KUBE_ROOT}/.generated_docs"
+    # The .generated_docs file lists itself, so we don't need to explicitly
+    # delete it.
+  fi
 }
 
 # Takes a path $1 to traverse for md files to append the ga-beacon tracking
diff --git a/hack/update-generated-docs.sh b/hack/update-generated-docs.sh
index f97dcd781be..7720a042c31 100755
--- a/hack/update-generated-docs.sh
+++ b/hack/update-generated-docs.sh
@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This file is not intended to be run automatically. It is meant to be run
+# immediately before exporting docs. We do not want to check these documents in
+# by default.
+
 set -o errexit
 set -o nounset
 set -o pipefail
@@ -28,7 +32,6 @@ kube::golang::setup_env
     cmd/genkubedocs \
     cmd/genman \
     cmd/genyaml \
-    cmd/mungedocs \
     federation/cmd/genfeddocs
 
 kube::util::ensure-temp-dir
@@ -36,46 +39,17 @@ kube::util::ensure-temp-dir
 kube::util::gen-docs "${KUBE_TEMP}"
 
 # remove all of the old docs
-while read file; do
-  rm "${KUBE_ROOT}/${file}" 2>/dev/null || true
-done <"${KUBE_ROOT}/.generated_docs"
+kube::util::remove-gen-docs
 
+# Copy fresh docs into the repo.
 # the shopt is so that we get .generated_docs from the glob.
 shopt -s dotglob
 cp -af "${KUBE_TEMP}"/* "${KUBE_ROOT}"
 shopt -u dotglob
 
-kube::util::gen-analytics "${KUBE_ROOT}"
+# Replace with placeholder docs
+kube::util::set-placeholder-gen-docs
 
-mungedocs=$(kube::util::find-binary "mungedocs")
-"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${KUBE_ROOT}/docs/" && ret=0 || ret=$?
-if [[ $ret -eq 1 ]]; then
-  echo "${KUBE_ROOT}/docs/ requires manual changes.  See preceding errors."
-  exit 1
-elif [[ $ret -gt 1 ]]; then
-  echo "Error running mungedocs."
-  exit 1
-fi
+echo "Generated docs have been placed in the repository tree. Running hack/update-munge-docs.sh."
 
-"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${KUBE_ROOT}/examples/" && ret=0 || ret=$?
-if [[ $ret -eq 1 ]]; then
-  echo "${KUBE_ROOT}/examples/ requires manual changes.  See preceding errors."
-  exit 1
-elif [[ $ret -gt 1 ]]; then
-  echo "Error running mungedocs."
-  exit 1
-fi
-
-"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" \
-               "--skip-munges=unversioned-warning,analytics" \
-               "--norecurse" \
-               "--root-dir=${KUBE_ROOT}/" && ret=0 || ret=$?
-if [[ $ret -eq 1 ]]; then
-  echo "${KUBE_ROOT}/ requires manual changes.  See preceding errors."
-  exit 1
-elif [[ $ret -gt 1 ]]; then
-  echo "Error running mungedocs."
-  exit 1
-fi
-
-# ex: ts=2 sw=2 et filetype=sh
+"${KUBE_ROOT}/hack/update-munge-docs.sh"
diff --git a/hack/update-munge-docs.sh b/hack/update-munge-docs.sh
new file mode 100755
index 00000000000..8a259a77b13
--- /dev/null
+++ b/hack/update-munge-docs.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+kube::golang::setup_env
+
+"${KUBE_ROOT}/hack/build-go.sh" \
+    cmd/mungedocs
+
+kube::util::ensure-temp-dir
+
+kube::util::gen-analytics "${KUBE_ROOT}"
+
+mungedocs=$(kube::util::find-binary "mungedocs")
+"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${KUBE_ROOT}/docs/" && ret=0 || ret=$?
+if [[ $ret -eq 1 ]]; then
+  echo "${KUBE_ROOT}/docs/ requires manual changes.  See preceding errors."
+  exit 1
+elif [[ $ret -gt 1 ]]; then
+  echo "Error running mungedocs."
+  exit 1
+fi
+
+"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${KUBE_ROOT}/examples/" && ret=0 || ret=$?
+if [[ $ret -eq 1 ]]; then
+  echo "${KUBE_ROOT}/examples/ requires manual changes.  See preceding errors."
+  exit 1
+elif [[ $ret -gt 1 ]]; then
+  echo "Error running mungedocs."
+  exit 1
+fi
+
+"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" \
+               "--skip-munges=unversioned-warning,analytics" \
+               "--norecurse" \
+               "--root-dir=${KUBE_ROOT}/" && ret=0 || ret=$?
+if [[ $ret -eq 1 ]]; then
+  echo "${KUBE_ROOT}/ requires manual changes.  See preceding errors."
+  exit 1
+elif [[ $ret -gt 1 ]]; then
+  echo "Error running mungedocs."
+  exit 1
+fi
+
+# ex: ts=2 sw=2 et filetype=sh
diff --git a/hack/verify-generated-docs.sh b/hack/verify-generated-docs.sh
index e4b38b3fafd..99d29752039 100755
--- a/hack/verify-generated-docs.sh
+++ b/hack/verify-generated-docs.sh
@@ -28,57 +28,29 @@ kube::golang::setup_env
     cmd/genkubedocs \
     cmd/genman \
     cmd/genyaml \
-    cmd/mungedocs
-
-# Find binary
-mungedocs=$(kube::util::find-binary "mungedocs")
-
-DOCROOT="${KUBE_ROOT}/docs/"
-EXAMPLEROOT="${KUBE_ROOT}/examples/"
-
-# mungedocs --verify can (and should) be run on the real docs, otherwise their
-# links will be distorted. --verify means that it will not make changes.
-# --verbose gives us output we can use for a diff.
-"${mungedocs}" "--verify=true" "--verbose=true" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${DOCROOT}" && ret=0 || ret=$?
-if [[ $ret -eq 1 ]]; then
-  echo "${DOCROOT} is out of date. Please run hack/update-generated-docs.sh"
-  exit 1
-fi
-if [[ $ret -gt 1 ]]; then
-  echo "Error running mungedocs"
-  exit 1
-fi
-
-"${mungedocs}" "--verify=true" "--verbose=true" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${EXAMPLEROOT}" && ret=0 || ret=$?
-if [[ $ret -eq 1 ]]; then
-  echo "${EXAMPLEROOT} is out of date. Please run hack/update-generated-docs.sh"
-  exit 1
-fi
-if [[ $ret -gt 1 ]]; then
-  echo "Error running mungedocs"
-  exit 1
-fi
+    federation/cmd/genfeddocs
 
 kube::util::ensure-temp-dir
 
 kube::util::gen-docs "${KUBE_TEMP}"
-diff -Naup "${KUBE_TEMP}/.generated_docs" "${KUBE_ROOT}/.generated_docs" || ret=1 || true
-while read file; do
-  diff -Naup "${KUBE_TEMP}/${file}" "${KUBE_ROOT}/${file}" || ret=1 || true
-done <"${KUBE_TEMP}/.generated_docs"
 
-needsanalytics=($(kube::util::gen-analytics "${KUBE_ROOT}" 1))
-if [[ ${#needsanalytics[@]} -ne 0 ]]; then
-  echo -e "Some md files are missing ga-beacon analytics link:"
-  printf '%s\n' "${needsanalytics[@]}"
-  ret=1
-fi
-if [[ $ret -eq 0 ]]
-then
-  echo "Generated docs are up to date."
-else
-  echo "Generated docs are out of date. Please run hack/update-generated-docs.sh"
+# Verify the list matches the expected list (diff should be empty)
+if [[ "$(diff ${KUBE_ROOT}/.generated_docs ${KUBE_TEMP}/.generated_docs)" != "" ]]; then
+  echo "List of generated docs doesn't match a freshly built list. Please run hack/update-generated-docs.sh"
   exit 1
 fi
 
-# ex: ts=2 sw=2 et filetype=sh
+# Verify the files in the repo all contain the boilerplate instead of the actual
+# content.
+while read file; do
+  # Ignore .generated_docs-- it should not have the boilerplate!
+  [[ "${file}" == ".generated_docs" ]] && continue
+
+  # Search for "hack/generate-docs.sh" as a proxy for the boilerplate content,
+  # since the munger adds a bunch of other stuff.
+  if [[ "$(grep "hack/generate-docs.sh" "${KUBE_ROOT}/${file}")" == "" ]]; then
+    echo "${file} doesn't seem to have the correct boilerplate content for an autogenerated file."
+    echo "Please run hack/update-generated-docs.sh"
+    exit 1
+  fi
+done <"${KUBE_ROOT}/.generated_docs"
diff --git a/hack/verify-munge-docs.sh b/hack/verify-munge-docs.sh
new file mode 100755
index 00000000000..6162b5329bb
--- /dev/null
+++ b/hack/verify-munge-docs.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+kube::golang::setup_env
+
+"${KUBE_ROOT}/hack/build-go.sh" \
+    cmd/mungedocs
+
+# Find binary
+mungedocs=$(kube::util::find-binary "mungedocs")
+
+DOCROOT="${KUBE_ROOT}/docs/"
+EXAMPLEROOT="${KUBE_ROOT}/examples/"
+
+# mungedocs --verify can (and should) be run on the real docs, otherwise their
+# links will be distorted. --verify means that it will not make changes.
+# --verbose gives us output we can use for a diff.
+"${mungedocs}" "--verify=true" "--verbose=true" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${DOCROOT}" && ret=0 || ret=$?
+if [[ $ret -eq 1 ]]; then
+  echo "${DOCROOT} is out of date. Please run hack/update-munge-docs.sh"
+  exit 1
+fi
+if [[ $ret -gt 1 ]]; then
+  echo "Error running mungedocs"
+  exit 1
+fi
+
+"${mungedocs}" "--verify=true" "--verbose=true" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${EXAMPLEROOT}" && ret=0 || ret=$?
+if [[ $ret -eq 1 ]]; then
+  echo "${EXAMPLEROOT} is out of date. Please run hack/update-munge-docs.sh"
+  exit 1
+fi
+if [[ $ret -gt 1 ]]; then
+  echo "Error running mungedocs"
+  exit 1
+fi
+
+needsanalytics=($(kube::util::gen-analytics "${KUBE_ROOT}" 1))
+if [[ ${#needsanalytics[@]} -ne 0 ]]; then
+  echo -e "Some md files are missing ga-beacon analytics link:"
+  printf '%s\n' "${needsanalytics[@]}"
+  ret=1
+fi
+if [[ $ret -eq 0 ]]; then
+  echo "Docs are properly munged."
+else
+  echo "Docs need munging. Please run hack/update-munge-docs.sh"
+  exit 1
+fi
+
+# ex: ts=2 sw=2 et filetype=sh