Support for in-place modification of docs

Adds cmd/mungedocs which is framework for processing
all files under docs/ and either verifying that no changes needed or
making in-place changes.

Did not reuse kube::util::gen-docs because that seemed to be
centered around handling added files, and this pass does not
add files.

Planned uses:
 - table of contents automatic updating
 - linkification
 - internal link checker
 - link-path-relativizer or absolutizer
 - example file syncer
 - header inserter.

Just table-of-contents updating in this PR.

Added Table of Contents to docs/networking.md.

Demonstrates use of new TOC generator presubmit.
Other docs will be added in future PRs.
Additional development will be needed to handle some
of the more complex cases.

cmd/mungedocs/mungedocs.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	flag "github.com/spf13/pflag"
+)
+
+var (
+	verify  = flag.Bool("verify", false, "Exit with status 1 if files would have needed changes but do not change.")
+	rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.")
+
+	ErrChangesNeeded = errors.New("mungedocs: changes required")
+)
+
+func visitAndVerify(path string, i os.FileInfo, e error) error {
+	return visitAndChangeOrVerify(path, i, e, false)
+}
+
+func visitAndChange(path string, i os.FileInfo, e error) error {
+	return visitAndChangeOrVerify(path, i, e, true)
+}
+
+// Either change a file or verify that it needs no changes (according to modify argument)
+func visitAndChangeOrVerify(path string, i os.FileInfo, e error, modify bool) error {
+	if !strings.HasSuffix(path, ".md") {
+		return nil
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	before, err := ioutil.ReadAll(file)
+	if err != nil {
+		return err
+	}
+
+	after, err := updateTOC(before)
+	if err != nil {
+		return err
+	}
+	if modify {
+		// Write out new file with any changes.
+		if !bytes.Equal(after, before) {
+			file.Close()
+			ioutil.WriteFile(path, after, 0644)
+		}
+	} else {
+		// Just verify that there are no changes.
+		if !bytes.Equal(after, before) {
+			return ErrChangesNeeded
+		}
+	}
+
+	// TODO(erictune): more types of passes, such as:
+	// Linkify terms
+	// Verify links point to files.
+
+	return nil
+}
+
+func main() {
+	flag.Parse()
+
+	if *rootDir == "" {
+		fmt.Fprintf(os.Stderr, "usage: %s [--verify] --root-dir <docs root>\n", flag.Arg(0))
+		os.Exit(1)
+	}
+
+	// For each markdown file under source docs root, process the doc.
+	// If any error occurs, will exit with failure.
+	// If verify is true, then status is 0 for no changes needed, 1 for changes needed
+	// and >1 for an error during processing.
+	// If verify is false, then status is 0 if changes successfully made or no changes needed,
+	// 1 if changes were needed but require human intervention, and >1 for an unexpected
+	// error during processing.
+	var err error
+	if *verify {
+		err = filepath.Walk(*rootDir, visitAndVerify)
+	} else {
+		err = filepath.Walk(*rootDir, visitAndChange)
+	}
+	if err != nil {
+		if err == ErrChangesNeeded {
+			if *verify {
+				fmt.Fprintf(os.Stderr,
+					"Some changes needed but not made due to --verify=true\n")
+			} else {
+				fmt.Fprintf(os.Stderr,
+					"Some changes needed but human intervention is required\n")
+			}
+			os.Exit(1)
+		}
+		fmt.Fprintf(os.Stderr, "filepath.Walk() returned %v\n", err)
+		os.Exit(2)
+	}
+}

cmd/mungedocs/toc.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"strings"
+)
+
+// inserts/updates a table of contents in markdown file.
+//
+// First, builds a ToC.
+// Then, finds <!-- BEGIN GENERATED TOC --> and <!-- END GENERATED TOC -->, and replaces anything between those with
+// the ToC, thereby updating any previously inserted ToC.
+//
+// TODO(erictune): put this in own package with tests
+func updateTOC(markdown []byte) ([]byte, error) {
+	toc, err := buildTOC(markdown)
+	if err != nil {
+		return nil, err
+	}
+	updatedMarkdown, err := updateMacroBlock(markdown, "<!-- BEGIN GENERATED TOC -->", "<!-- END GENERATED TOC -->", string(toc))
+	if err != nil {
+		return nil, err
+	}
+	return updatedMarkdown, nil
+}
+
+// Replaces the text between matching "beginMark" and "endMark" within "document" with "insertThis".
+//
+// Delimiters should occupy own line.
+// Returns copy of document with modifications.
+func updateMacroBlock(document []byte, beginMark, endMark, insertThis string) ([]byte, error) {
+	var buffer bytes.Buffer
+	lines := strings.Split(string(document), "\n")
+	// Skip trailing empty string from Split-ing
+	if len(lines) > 0 && lines[len(lines)-1] == "" {
+		lines = lines[:len(lines)-1]
+	}
+	betweenBeginAndEnd := false
+	for _, line := range lines {
+		trimmedLine := strings.Trim(line, " \n")
+		if trimmedLine == beginMark {
+			if betweenBeginAndEnd {
+				return nil, fmt.Errorf("found second begin mark while updating macro blocks")
+			}
+			betweenBeginAndEnd = true
+			buffer.WriteString(line)
+			buffer.WriteString("\n")
+		} else if trimmedLine == endMark {
+			if !betweenBeginAndEnd {
+				return nil, fmt.Errorf("found end mark without being mark while updating macro blocks")
+			}
+			buffer.WriteString(insertThis)
+			// Extra newline avoids github markdown bug where comment ends up on same line as last bullet.
+			buffer.WriteString("\n")
+			buffer.WriteString(line)
+			buffer.WriteString("\n")
+			betweenBeginAndEnd = false
+		} else {
+			if !betweenBeginAndEnd {
+				buffer.WriteString(line)
+				buffer.WriteString("\n")
+			}
+		}
+	}
+	if betweenBeginAndEnd {
+		return nil, fmt.Errorf("never found closing end mark while updating macro blocks")
+	}
+	return buffer.Bytes(), nil
+}
+
+// builds table of contents for markdown file
+//
+// First scans for all section headers (lines that begin with "#" but not within code quotes)
+// and builds a table of contents from those.  Assumes bookmarks for those will be
+// like #each-word-in-heading-in-lowercases-with-dashes-instead-of-spaces.
+// builds the ToC.
+func buildTOC(markdown []byte) ([]byte, error) {
+	var buffer bytes.Buffer
+	scanner := bufio.NewScanner(bytes.NewReader(markdown))
+	for scanner.Scan() {
+		line := scanner.Text()
+		noSharps := strings.TrimLeft(line, "#")
+		numSharps := len(line) - len(noSharps)
+		heading := strings.Trim(noSharps, " \n")
+		if numSharps > 0 {
+			indent := strings.Repeat("  ", numSharps-1)
+			bookmark := strings.Replace(strings.ToLower(heading), " ", "-", -1)
+			tocLine := fmt.Sprintf("%s- [%s](#%s)\n", indent, heading, bookmark)
+			buffer.WriteString(tocLine)
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return []byte{}, err
+	}
+
+	return buffer.Bytes(), nil
+}

cmd/mungedocs/toc_test.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_updateMacroBlock(t *testing.T) {
+	var cases = []struct {
+		in  string
+		out string
+	}{
+		{"", ""},
+		{"Lorem ipsum\ndolor sit amet\n",
+			"Lorem ipsum\ndolor sit amet\n"},
+		{"Lorem ipsum \n BEGIN\ndolor\nEND\nsit amet\n",
+			"Lorem ipsum \n BEGIN\nfoo\n\nEND\nsit amet\n"},
+	}
+	for _, c := range cases {
+		actual, err := updateMacroBlock([]byte(c.in), "BEGIN", "END", "foo\n")
+		assert.NoError(t, err)
+		if c.out != string(actual) {
+			t.Errorf("Expected '%v' but got '%v'", c.out, string(actual))
+		}
+	}
+}
+
+func Test_updateMacroBlock_errors(t *testing.T) {
+	var cases = []struct {
+		in string
+	}{
+		{"BEGIN\n"},
+		{"blah\nBEGIN\nblah"},
+		{"END\n"},
+		{"blah\nEND\nblah\n"},
+		{"END\nBEGIN"},
+		{"BEGIN\nEND\nEND"},
+		{"BEGIN\nBEGIN\nEND"},
+		{"BEGIN\nBEGIN\nEND\nEND"},
+	}
+	for _, c := range cases {
+		_, err := updateMacroBlock([]byte(c.in), "BEGIN", "END", "foo")
+		assert.Error(t, err)
+	}
+}
+
+func Test_buildTOC(t *testing.T) {
+	var cases = []struct {
+		in  string
+		out string
+	}{
+		{"", ""},
+		{"Lorem ipsum\ndolor sit amet\n", ""},
+		{"# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n",
+			"- [Title](#title)\n  - [Section Heading](#section-heading)\n"},
+	}
+	for _, c := range cases {
+		actual, err := buildTOC([]byte(c.in))
+		assert.NoError(t, err)
+		if c.out != string(actual) {
+			t.Errorf("Expected TOC '%v' but got '%v'", c.out, string(actual))
+		}
+	}
+}
+
+func Test_updateTOC(t *testing.T) {
+	var cases = []struct {
+		in  string
+		out string
+	}{
+		{"", ""},
+		{"Lorem ipsum\ndolor sit amet\n",
+			"Lorem ipsum\ndolor sit amet\n"},
+		{"# Title\nLorem ipsum \n**table of contents**\n<!-- BEGIN GENERATED TOC -->\nold cruft\n<!-- END GENERATED TOC -->\n## Section Heading\ndolor sit amet\n",
+			"# Title\nLorem ipsum \n**table of contents**\n<!-- BEGIN GENERATED TOC -->\n- [Title](#title)\n  - [Section Heading](#section-heading)\n\n<!-- END GENERATED TOC -->\n## Section Heading\ndolor sit amet\n"},
+	}
+	for _, c := range cases {
+		actual, err := updateTOC([]byte(c.in))
+		assert.NoError(t, err)
+		if c.out != string(actual) {
+			t.Errorf("Expected TOC '%v' but got '%v'", c.out, string(actual))
+		}
+	}
+}

docs/networking.md

@@ -1,4 +1,20 @@
 # Networking in Kubernetes
+**Table of Contents**
+<!-- BEGIN GENERATED TOC -->
+- [Networking in Kubernetes](#networking-in-kubernetes)
+  - [Summary](#summary)
+  - [Docker model](#docker-model)
+  - [Kubernetes model](#kubernetes-model)
+  - [How to achieve this](#how-to-achieve-this)
+    - [Google Compute Engine (GCE)](#google-compute-engine-(gce))
+    - [L2 networks and linux bridging](#l2-networks-and-linux-bridging)
+    - [Flannel](#flannel)
+    - [OpenVSwitch](#openvswitch)
+    - [Weave](#weave)
+    - [Calico](#calico)
+  - [Other reading](#other-reading)
+
+<!-- END GENERATED TOC -->
 
 ## Summary
 

hack/lib/golang.sh

@@ -64,6 +64,7 @@ kube::golang::test_targets() {
     cmd/integration
     cmd/gendocs
     cmd/genman
+    cmd/mungedocs
     cmd/genbashcomp
     cmd/genconversion
     cmd/gendeepcopy

hack/run-gendocs.sh

@@ -22,23 +22,34 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
 kube::golang::setup_env
-"${KUBE_ROOT}/hack/build-go.sh" cmd/gendocs cmd/genman cmd/genbashcomp
+"${KUBE_ROOT}/hack/build-go.sh" cmd/gendocs cmd/genman cmd/genbashcomp cmd/mungedocs
 
 # Find binary
 gendocs=$(kube::util::find-binary "gendocs")
 genman=$(kube::util::find-binary "genman")
 genbashcomp=$(kube::util::find-binary "genbashcomp")
+mungedocs=$(kube::util::find-binary "mungedocs")
 
-if [[ ! -x "$gendocs" || ! -x "$genman" || ! -x "$genbashcomp" ]]; then
+if [[ ! -x "$gendocs" || ! -x "$genman" || ! -x "$genbashcomp" || ! -x "$mungedocs" ]]; then
   {
-    echo "It looks as if you don't have a compiled gendocs, genman, or genbashcomp binary"
+    echo "It looks as if you don't have a compiled gendocs, genman, genbashcomp or mungedocs binary"
     echo
     echo "If you are running from a clone of the git repo, please run"
-    echo "'./hack/build-go.sh cmd/gendocs cmd/genman cmd/genbashcomp'."
+    echo "'./hack/build-go.sh cmd/gendocs cmd/genman cmd/genbashcomp cmd/mungedocs'."
   } >&2
   exit 1
 fi
 
+"${mungedocs}" "--root-dir=${KUBE_ROOT}/docs/" 
+ret=$?
+if [[ $ret -eq 1 ]]; then
+  echo "${KUBE_ROOT}/docs/ requires manual changes.  See proceeding errors."
+  exit 1
+elif [[ $ret -eq 2 ]]; then
+  echo "Error running mungedocs."
+  exit 1
+fi
+
 kube::util::gen-doc "${gendocs}" "${KUBE_ROOT}/docs/" '###### Auto generated by spf13/cobra'
 kube::util::gen-doc "${genman}" "${KUBE_ROOT}/docs/man/man1"
 kube::util::gen-doc "${genbashcomp}" "${KUBE_ROOT}/contrib/completions/bash/"

hack/verify-gendocs.sh

@@ -23,16 +23,18 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 
 kube::golang::setup_env
 
+# Find binary
 gendocs=$(kube::util::find-binary "gendocs")
 genman=$(kube::util::find-binary "genman")
 genbashcomp=$(kube::util::find-binary "genbashcomp")
+mungedocs=$(kube::util::find-binary "mungedocs")
 
-if [[ ! -x "$gendocs" || ! -x "$genman" || ! -x "$genbashcomp" ]]; then
+if [[ ! -x "$gendocs" || ! -x "$genman" || ! -x "$genbashcomp" || ! -x "$mungedocs" ]]; then
   {
-    echo "It looks as if you don't have a compiled gendocs, genman, or genbashcomp binary"
+    echo "It looks as if you don't have a compiled gendocs, genman, genbashcomp or mungedocs binary"
     echo
     echo "If you are running from a clone of the git repo, please run"
-    echo "'./hack/build-go.sh cmd/gendocs cmd/genman cmd/genbashcomp'."
+    echo "'./hack/build-go.sh cmd/gendocs cmd/genman cmd/genbashcomp cmd/mungedocs'."
   } >&2
   exit 1
 fi
@@ -44,6 +46,17 @@ _tmp="${KUBE_ROOT}/_tmp"
 mkdir -p "${_tmp}"
 cp -a "${DOCROOT}" "${TMP_DOCROOT}"
 
+"${mungedocs}" "--verify=true" "--root-dir=${TMP_DOCROOT}"
+ret=$?
+if [[ $ret -eq 1 ]]; then
+  echo "${DOCROOT} is out of date. Please run hack/run-gendocs.sh"
+  exit 1
+fi
+if [[ $ret -eq 2 ]]; then
+  echo "Error running mungedocs"
+  exit 1
+fi
+
 kube::util::gen-doc "${genman}" "${TMP_DOCROOT}/man/man1/"
 kube::util::gen-doc "${gendocs}" "${TMP_DOCROOT}"
 echo "diffing ${DOCROOT} against freshly generated docs"