Rework doc generation to simplify and centralize

Just do all doc generation in the hack::util::gen-docs instead of spread
around. We also only track the generated docs in a single file for the
whole tree.

.generated_docs

@@ -0,0 +1,64 @@
+.generated_docs
+contrib/completions/bash/kubectl
+docs/man/man1/kubectl-annotate.1
+docs/man/man1/kubectl-api-versions.1
+docs/man/man1/kubectl-attach.1
+docs/man/man1/kubectl-cluster-info.1
+docs/man/man1/kubectl-config-set-cluster.1
+docs/man/man1/kubectl-config-set-context.1
+docs/man/man1/kubectl-config-set-credentials.1
+docs/man/man1/kubectl-config-set.1
+docs/man/man1/kubectl-config-unset.1
+docs/man/man1/kubectl-config-use-context.1
+docs/man/man1/kubectl-config-view.1
+docs/man/man1/kubectl-config.1
+docs/man/man1/kubectl-create.1
+docs/man/man1/kubectl-delete.1
+docs/man/man1/kubectl-describe.1
+docs/man/man1/kubectl-exec.1
+docs/man/man1/kubectl-expose.1
+docs/man/man1/kubectl-get.1
+docs/man/man1/kubectl-label.1
+docs/man/man1/kubectl-logs.1
+docs/man/man1/kubectl-namespace.1
+docs/man/man1/kubectl-patch.1
+docs/man/man1/kubectl-port-forward.1
+docs/man/man1/kubectl-proxy.1
+docs/man/man1/kubectl-replace.1
+docs/man/man1/kubectl-rolling-update.1
+docs/man/man1/kubectl-run.1
+docs/man/man1/kubectl-scale.1
+docs/man/man1/kubectl-stop.1
+docs/man/man1/kubectl-version.1
+docs/man/man1/kubectl.1
+docs/user-guide/kubectl/kubectl.md
+docs/user-guide/kubectl/kubectl_annotate.md
+docs/user-guide/kubectl/kubectl_api-versions.md
+docs/user-guide/kubectl/kubectl_attach.md
+docs/user-guide/kubectl/kubectl_cluster-info.md
+docs/user-guide/kubectl/kubectl_config.md
+docs/user-guide/kubectl/kubectl_config_set-cluster.md
+docs/user-guide/kubectl/kubectl_config_set-context.md
+docs/user-guide/kubectl/kubectl_config_set-credentials.md
+docs/user-guide/kubectl/kubectl_config_set.md
+docs/user-guide/kubectl/kubectl_config_unset.md
+docs/user-guide/kubectl/kubectl_config_use-context.md
+docs/user-guide/kubectl/kubectl_config_view.md
+docs/user-guide/kubectl/kubectl_create.md
+docs/user-guide/kubectl/kubectl_delete.md
+docs/user-guide/kubectl/kubectl_describe.md
+docs/user-guide/kubectl/kubectl_exec.md
+docs/user-guide/kubectl/kubectl_expose.md
+docs/user-guide/kubectl/kubectl_get.md
+docs/user-guide/kubectl/kubectl_label.md
+docs/user-guide/kubectl/kubectl_logs.md
+docs/user-guide/kubectl/kubectl_namespace.md
+docs/user-guide/kubectl/kubectl_patch.md
+docs/user-guide/kubectl/kubectl_port-forward.md
+docs/user-guide/kubectl/kubectl_proxy.md
+docs/user-guide/kubectl/kubectl_replace.md
+docs/user-guide/kubectl/kubectl_rolling-update.md
+docs/user-guide/kubectl/kubectl_run.md
+docs/user-guide/kubectl/kubectl_scale.md
+docs/user-guide/kubectl/kubectl_stop.md
+docs/user-guide/kubectl/kubectl_version.md

contrib/completions/bash/.files_generated

@@ -1 +0,0 @@
-kubectl

docs/man/man1/.files_generated

@@ -1,31 +0,0 @@
-kubectl-annotate.1
-kubectl-api-versions.1
-kubectl-attach.1
-kubectl-cluster-info.1
-kubectl-config-set-cluster.1
-kubectl-config-set-context.1
-kubectl-config-set-credentials.1
-kubectl-config-set.1
-kubectl-config-unset.1
-kubectl-config-use-context.1
-kubectl-config-view.1
-kubectl-config.1
-kubectl-create.1
-kubectl-delete.1
-kubectl-describe.1
-kubectl-exec.1
-kubectl-expose.1
-kubectl-get.1
-kubectl-label.1
-kubectl-logs.1
-kubectl-namespace.1
-kubectl-patch.1
-kubectl-port-forward.1
-kubectl-proxy.1
-kubectl-replace.1
-kubectl-rolling-update.1
-kubectl-run.1
-kubectl-scale.1
-kubectl-stop.1
-kubectl-version.1
-kubectl.1

docs/user-guide/kubectl/.files_generated

@@ -1,31 +0,0 @@
-kubectl.md
-kubectl_annotate.md
-kubectl_api-versions.md
-kubectl_attach.md
-kubectl_cluster-info.md
-kubectl_config.md
-kubectl_config_set-cluster.md
-kubectl_config_set-context.md
-kubectl_config_set-credentials.md
-kubectl_config_set.md
-kubectl_config_unset.md
-kubectl_config_use-context.md
-kubectl_config_view.md
-kubectl_create.md
-kubectl_delete.md
-kubectl_describe.md
-kubectl_exec.md
-kubectl_expose.md
-kubectl_get.md
-kubectl_label.md
-kubectl_logs.md
-kubectl_namespace.md
-kubectl_patch.md
-kubectl_port-forward.md
-kubectl_proxy.md
-kubectl_replace.md
-kubectl_rolling-update.md
-kubectl_run.md
-kubectl_scale.md
-kubectl_stop.md
-kubectl_version.md

hack/after-build/update-generated-docs.sh

@@ -23,17 +23,23 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 
 kube::golang::setup_env
 
-# Find binary
+kube::util::ensure-temp-dir
-gendocs=$(kube::util::find-binary "gendocs")
+
-genman=$(kube::util::find-binary "genman")
+kube::util::gen-docs "${KUBE_TEMP}"
-genbashcomp=$(kube::util::find-binary "genbashcomp")
+
-mungedocs=$(kube::util::find-binary "mungedocs")
+# remove all of the old docs
+while read file; do
+  rm "${KUBE_ROOT}/${file}" 2>/dev/null || true
+done <"${KUBE_ROOT}/.generated_docs"
+
+# the shopt is so that we get .generated_docs from the glob.
+shopt -s dotglob
+cp -af "${KUBE_TEMP}"/* "${KUBE_ROOT}"
+shopt -u dotglob
 
-kube::util::gen-doc "${gendocs}" "${KUBE_ROOT}" "docs/user-guide/kubectl/" '###### Auto generated by spf13/cobra'
-kube::util::gen-doc "${genman}" "${KUBE_ROOT}" "docs/man/man1"
-kube::util::gen-doc "${genbashcomp}" "${KUBE_ROOT}" "contrib/completions/bash/"
 kube::util::gen-analytics "${KUBE_ROOT}"
 
+mungedocs=$(kube::util::find-binary "mungedocs")
 "${mungedocs}" "--root-dir=${KUBE_ROOT}/docs/" && ret=0 || ret=$?
 if [[ $ret -eq 1 ]]; then
   echo "${KUBE_ROOT}/docs/ requires manual changes.  See preceding errors."

hack/after-build/verify-generated-docs.sh

@@ -31,11 +31,6 @@ mungedocs=$(kube::util::find-binary "mungedocs")
 
 DOCROOT="${KUBE_ROOT}/docs/"
 EXAMPLEROOT="${KUBE_ROOT}/examples/"
-TMP_DOCROOT="${KUBE_ROOT}/_tmp/docs/"
-_tmp="${KUBE_ROOT}/_tmp"
-
-mkdir -p "${_tmp}"
-cp -a "${DOCROOT}" "${TMP_DOCROOT}"
 
 # mungedocs --verify can (and should) be run on the real docs, otherwise their
 # links will be distorted. --verify means that it will not make changes.
@@ -59,12 +54,15 @@ if [[ $ret -gt 1 ]]; then
   exit 1
 fi
 
-kube::util::gen-doc "${genman}" "${_tmp}" "docs/man/man1/"
-kube::util::gen-doc "${gendocs}" "${_tmp}" "docs/user-guide/kubectl/" '###### Auto generated by spf13/cobra'
 
-echo "diffing ${DOCROOT} against freshly generated docs"
+kube::util::ensure-temp-dir
-diff -Naupr "${DOCROOT}" "${TMP_DOCROOT}" && ret=0 || ret=$?
+
-rm -rf "${_tmp}"
+kube::util::gen-docs "${KUBE_TEMP}"
+diff -Naup "${KUBE_TEMP}/.generated_docs" "${KUBE_ROOT}/.generated_docs" || ret=1 || true
+while read file; do
+  diff -Naup "${KUBE_TEMP}/${file}" "${KUBE_ROOT}/${file}" || ret=1 || true
+done <"${KUBE_TEMP}/.generated_docs"
+
 needsanalytics=($(kube::util::gen-analytics "${KUBE_ROOT}" 1))
 if [[ ${#needsanalytics[@]} -ne 0 ]]; then
   echo -e "Some md files are missing ga-beacon analytics link:"
@@ -73,26 +71,9 @@ if [[ ${#needsanalytics[@]} -ne 0 ]]; then
 fi
 if [[ $ret -eq 0 ]]
 then
-  echo "${DOCROOT} up to date."
+  echo "Generated docs are up to date."
 else
-  echo "${DOCROOT} is out of date. Please run hack/update-generated-docs.sh"
+  echo "Generated docs are out of date. Please run hack/update-generated-docs.sh"
   exit 1
 fi
-
-COMPROOT="${KUBE_ROOT}/contrib/completions"
-TMP_COMPROOT="${KUBE_ROOT}/contrib/completions_tmp"
-cp -a "${COMPROOT}" "${TMP_COMPROOT}"
-kube::util::gen-doc "${genbashcomp}" "${TMP_COMPROOT}" "bash/"
-diff -Naupr "${COMPROOT}" "${TMP_COMPROOT}" && ret=0 || ret=$?
-rm -rf ${TMP_COMPROOT}
-if [ $ret -eq 0 ]
-then
-	echo "${COMPROOT} up to date."
-else
-	echo "${COMPROOT} is out of date. Please run hack/update-generated-docs.sh"
-	echo "If you did not make a change to kubectl or its dependencies,"
-	echo "run 'make clean' and retry this command."
-	exit 1
-fi
-
 # ex: ts=2 sw=2 et filetype=sh

hack/lib/util.sh

@@ -155,60 +155,49 @@ kube::util::wait-for-jobs() {
   return ${fail}
 }
 
-# Takes a binary to run $1 and then copies the results to $2.
+# Run all known doc generators (today gendocs, genman, and genbashcomp for kubectl)
-# If the generated and original files are the same after filtering lines
+# $1 is the directory to put those generated documents
-# that match $3, copy is skipped.
+kube::util::gen-docs() {
-kube::util::gen-doc() {
+  local dest="$1"
-  local cmd="$1"
+
-  local base_dest="$(kube::util::realpath $2)"
+  # Find binary
-  local relative_doc_dest="$(echo $3 | sed 's/\/$//')"
+  gendocs=$(kube::util::find-binary "gendocs")
-  local dest="${base_dest}/${relative_doc_dest}"
+  genman=$(kube::util::find-binary "genman")
-  local skipprefix="${4:-}"
+  genbashcomp=$(kube::util::find-binary "genbashcomp")
+
+  mkdir -p "${dest}/docs/user-guide/kubectl/"
+  "${gendocs}" "${dest}/docs/user-guide/kubectl/"
+  mkdir -p "${dest}/docs/man/man1/"
+  "${genman}" "${dest}/docs/man/man1/"
+  mkdir -p "${dest}/contrib/completions/bash/"
+  "${genbashcomp}" "${dest}/contrib/completions/bash/"
 
-  # We do this in a tmpdir in case the dest has other non-autogenned files
-  # We don't want to include them in the list of gen'd files
-  local tmpdir="${KUBE_ROOT}/doc_tmp"
-  mkdir -p "${tmpdir}"
-  # generate the new files
-  ${cmd} "${tmpdir}"
   # create the list of generated files
-  ls "${tmpdir}" | LC_ALL=C sort > "${tmpdir}/.files_generated"
+  pushd "${dest}" > /dev/null
+  touch .generated_docs
+  find -type f | cut -sd / -f 2- | LC_ALL=C sort > .generated_docs
+  popd > /dev/null
 
   while read file; do
-    # Don't add analytics link to generated .md files -- that is done (and
+    # Copy out of KUBE_ROOT if we didn't really change anything
-    # checked) by mungedocs.
+    if [[ -e "${dest}/${file}" && -e "${KUBE_ROOT}/${file}" ]]; then
-
-    # Remove all old generated files from the destination
-    if [[ -e "${tmpdir}/${file}" ]]; then
-      local original generated
       # Filter all munges from original content.
-      original=$(cat "${dest}/${file}" | sed '/^<!-- BEGIN MUNGE:.*/,/^<!-- END MUNGE:.*/d')
+      original=$(cat "${KUBE_ROOT}/${file}" | sed '/^<!-- BEGIN MUNGE:.*/,/^<!-- END MUNGE:.*/d')
-      generated=$(cat "${tmpdir}/${file}")
+      generated=$(cat "${dest}/${file}")
-      # If this function was asked to filter lines with a prefix, do so.
+
-      if [[ -n "${skipprefix}" ]]; then
+      # Filter out meaningless lines with timestamps
-        original=$(echo "${original}" | grep -v "^${skipprefix}" || :)
+      original=$(echo "${original}" | grep -v "Auto generated by spf13/cobra" || :)
-        generated=$(echo "${generated}" | grep -v "^${skipprefix}" || :)
+      generated=$(echo "${generated}" | grep -v "Auto generated by spf13/cobra" || :)
-      fi
+
       # By now, the contents should be normalized and stripped of any
       # auto-managed content.  We also ignore whitespace here because of
       # markdown strictness fixups later in the pipeline.
-      if diff -Bw <(echo "${original}") <(echo "${generated}") > /dev/null; then
+      if diff -Bw >/dev/null <(echo "${original}") <(echo "${generated}"); then
         # actual contents same, overwrite generated with original.
-        mv "${dest}/${file}" "${tmpdir}/${file}"
+        cp "${KUBE_ROOT}/${file}" "${dest}/${file}"
       fi
-    else
-      rm "${dest}/${file}" || true
     fi
-  done <"${dest}/.files_generated"
+  done <"${KUBE_ROOT}/.generated_docs"
-
-  # put the new generated file into the destination
-  # the shopt is so that we get .files_generated from the glob.
-  shopt -s dotglob
-  cp -af "${tmpdir}"/* "${dest}"
-  shopt -u dotglob
-
-  #cleanup
-  rm -rf "${tmpdir}"
 }
 
 # Takes a path $1 to traverse for md files to append the ga-beacon tracking