diff --git a/cmd/mungedocs/mungedocs.go b/cmd/mungedocs/mungedocs.go new file mode 100644 index 00000000000..ef0e0943a4f --- /dev/null +++ b/cmd/mungedocs/mungedocs.go @@ -0,0 +1,121 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + flag "github.com/spf13/pflag" +) + +var ( + verify = flag.Bool("verify", false, "Exit with status 1 if files would have needed changes but do not change.") + rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.") + + ErrChangesNeeded = errors.New("mungedocs: changes required") +) + +func visitAndVerify(path string, i os.FileInfo, e error) error { + return visitAndChangeOrVerify(path, i, e, false) +} + +func visitAndChange(path string, i os.FileInfo, e error) error { + return visitAndChangeOrVerify(path, i, e, true) +} + +// Either change a file or verify that it needs no changes (according to modify argument) +func visitAndChangeOrVerify(path string, i os.FileInfo, e error, modify bool) error { + if !strings.HasSuffix(path, ".md") { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + before, err := ioutil.ReadAll(file) + if err != nil { + return err + } + + after, err := updateTOC(before) + if err != nil { + return err + } + if modify { + // Write out new file with any changes. + if !bytes.Equal(after, before) { + file.Close() + ioutil.WriteFile(path, after, 0644) + } + } else { + // Just verify that there are no changes. + if !bytes.Equal(after, before) { + return ErrChangesNeeded + } + } + + // TODO(erictune): more types of passes, such as: + // Linkify terms + // Verify links point to files. + + return nil +} + +func main() { + flag.Parse() + + if *rootDir == "" { + fmt.Fprintf(os.Stderr, "usage: %s [--verify] --root-dir \n", flag.Arg(0)) + os.Exit(1) + } + + // For each markdown file under source docs root, process the doc. + // If any error occurs, will exit with failure. + // If verify is true, then status is 0 for no changes needed, 1 for changes needed + // and >1 for an error during processing. + // If verify is false, then status is 0 if changes successfully made or no changes needed, + // 1 if changes were needed but require human intervention, and >1 for an unexpected + // error during processing. + var err error + if *verify { + err = filepath.Walk(*rootDir, visitAndVerify) + } else { + err = filepath.Walk(*rootDir, visitAndChange) + } + if err != nil { + if err == ErrChangesNeeded { + if *verify { + fmt.Fprintf(os.Stderr, + "Some changes needed but not made due to --verify=true\n") + } else { + fmt.Fprintf(os.Stderr, + "Some changes needed but human intervention is required\n") + } + os.Exit(1) + } + fmt.Fprintf(os.Stderr, "filepath.Walk() returned %v\n", err) + os.Exit(2) + } +} diff --git a/cmd/mungedocs/toc.go b/cmd/mungedocs/toc.go new file mode 100644 index 00000000000..c5de6f98bb1 --- /dev/null +++ b/cmd/mungedocs/toc.go @@ -0,0 +1,115 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bufio" + "bytes" + "fmt" + "strings" +) + +// inserts/updates a table of contents in markdown file. +// +// First, builds a ToC. +// Then, finds and , and replaces anything between those with +// the ToC, thereby updating any previously inserted ToC. +// +// TODO(erictune): put this in own package with tests +func updateTOC(markdown []byte) ([]byte, error) { + toc, err := buildTOC(markdown) + if err != nil { + return nil, err + } + updatedMarkdown, err := updateMacroBlock(markdown, "", "", string(toc)) + if err != nil { + return nil, err + } + return updatedMarkdown, nil +} + +// Replaces the text between matching "beginMark" and "endMark" within "document" with "insertThis". +// +// Delimiters should occupy own line. +// Returns copy of document with modifications. +func updateMacroBlock(document []byte, beginMark, endMark, insertThis string) ([]byte, error) { + var buffer bytes.Buffer + lines := strings.Split(string(document), "\n") + // Skip trailing empty string from Split-ing + if len(lines) > 0 && lines[len(lines)-1] == "" { + lines = lines[:len(lines)-1] + } + betweenBeginAndEnd := false + for _, line := range lines { + trimmedLine := strings.Trim(line, " \n") + if trimmedLine == beginMark { + if betweenBeginAndEnd { + return nil, fmt.Errorf("found second begin mark while updating macro blocks") + } + betweenBeginAndEnd = true + buffer.WriteString(line) + buffer.WriteString("\n") + } else if trimmedLine == endMark { + if !betweenBeginAndEnd { + return nil, fmt.Errorf("found end mark without being mark while updating macro blocks") + } + buffer.WriteString(insertThis) + // Extra newline avoids github markdown bug where comment ends up on same line as last bullet. + buffer.WriteString("\n") + buffer.WriteString(line) + buffer.WriteString("\n") + betweenBeginAndEnd = false + } else { + if !betweenBeginAndEnd { + buffer.WriteString(line) + buffer.WriteString("\n") + } + } + } + if betweenBeginAndEnd { + return nil, fmt.Errorf("never found closing end mark while updating macro blocks") + } + return buffer.Bytes(), nil +} + +// builds table of contents for markdown file +// +// First scans for all section headers (lines that begin with "#" but not within code quotes) +// and builds a table of contents from those. Assumes bookmarks for those will be +// like #each-word-in-heading-in-lowercases-with-dashes-instead-of-spaces. +// builds the ToC. +func buildTOC(markdown []byte) ([]byte, error) { + var buffer bytes.Buffer + scanner := bufio.NewScanner(bytes.NewReader(markdown)) + for scanner.Scan() { + line := scanner.Text() + noSharps := strings.TrimLeft(line, "#") + numSharps := len(line) - len(noSharps) + heading := strings.Trim(noSharps, " \n") + if numSharps > 0 { + indent := strings.Repeat(" ", numSharps-1) + bookmark := strings.Replace(strings.ToLower(heading), " ", "-", -1) + tocLine := fmt.Sprintf("%s- [%s](#%s)\n", indent, heading, bookmark) + buffer.WriteString(tocLine) + } + } + if err := scanner.Err(); err != nil { + return []byte{}, err + } + + return buffer.Bytes(), nil +} diff --git a/cmd/mungedocs/toc_test.go b/cmd/mungedocs/toc_test.go new file mode 100644 index 00000000000..e7d502b5088 --- /dev/null +++ b/cmd/mungedocs/toc_test.go @@ -0,0 +1,101 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_updateMacroBlock(t *testing.T) { + var cases = []struct { + in string + out string + }{ + {"", ""}, + {"Lorem ipsum\ndolor sit amet\n", + "Lorem ipsum\ndolor sit amet\n"}, + {"Lorem ipsum \n BEGIN\ndolor\nEND\nsit amet\n", + "Lorem ipsum \n BEGIN\nfoo\n\nEND\nsit amet\n"}, + } + for _, c := range cases { + actual, err := updateMacroBlock([]byte(c.in), "BEGIN", "END", "foo\n") + assert.NoError(t, err) + if c.out != string(actual) { + t.Errorf("Expected '%v' but got '%v'", c.out, string(actual)) + } + } +} + +func Test_updateMacroBlock_errors(t *testing.T) { + var cases = []struct { + in string + }{ + {"BEGIN\n"}, + {"blah\nBEGIN\nblah"}, + {"END\n"}, + {"blah\nEND\nblah\n"}, + {"END\nBEGIN"}, + {"BEGIN\nEND\nEND"}, + {"BEGIN\nBEGIN\nEND"}, + {"BEGIN\nBEGIN\nEND\nEND"}, + } + for _, c := range cases { + _, err := updateMacroBlock([]byte(c.in), "BEGIN", "END", "foo") + assert.Error(t, err) + } +} + +func Test_buildTOC(t *testing.T) { + var cases = []struct { + in string + out string + }{ + {"", ""}, + {"Lorem ipsum\ndolor sit amet\n", ""}, + {"# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n", + "- [Title](#title)\n - [Section Heading](#section-heading)\n"}, + } + for _, c := range cases { + actual, err := buildTOC([]byte(c.in)) + assert.NoError(t, err) + if c.out != string(actual) { + t.Errorf("Expected TOC '%v' but got '%v'", c.out, string(actual)) + } + } +} + +func Test_updateTOC(t *testing.T) { + var cases = []struct { + in string + out string + }{ + {"", ""}, + {"Lorem ipsum\ndolor sit amet\n", + "Lorem ipsum\ndolor sit amet\n"}, + {"# Title\nLorem ipsum \n**table of contents**\n\nold cruft\n\n## Section Heading\ndolor sit amet\n", + "# Title\nLorem ipsum \n**table of contents**\n\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n\n## Section Heading\ndolor sit amet\n"}, + } + for _, c := range cases { + actual, err := updateTOC([]byte(c.in)) + assert.NoError(t, err) + if c.out != string(actual) { + t.Errorf("Expected TOC '%v' but got '%v'", c.out, string(actual)) + } + } +} diff --git a/docs/networking.md b/docs/networking.md index e5e872a59f6..608cd820fd6 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -1,4 +1,20 @@ # Networking in Kubernetes +**Table of Contents** + +- [Networking in Kubernetes](#networking-in-kubernetes) + - [Summary](#summary) + - [Docker model](#docker-model) + - [Kubernetes model](#kubernetes-model) + - [How to achieve this](#how-to-achieve-this) + - [Google Compute Engine (GCE)](#google-compute-engine-(gce)) + - [L2 networks and linux bridging](#l2-networks-and-linux-bridging) + - [Flannel](#flannel) + - [OpenVSwitch](#openvswitch) + - [Weave](#weave) + - [Calico](#calico) + - [Other reading](#other-reading) + + Kubernetes approaches networking somewhat differently than Docker does by default. There are 4 distinct networking problems to solve: diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index e2c83a9db79..94787459ec9 100644 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -64,6 +64,7 @@ kube::golang::test_targets() { cmd/integration cmd/gendocs cmd/genman + cmd/mungedocs cmd/genbashcomp cmd/genconversion cmd/gendeepcopy diff --git a/hack/run-gendocs.sh b/hack/run-gendocs.sh index 7757a3bd362..51d1b85fb93 100755 --- a/hack/run-gendocs.sh +++ b/hack/run-gendocs.sh @@ -22,23 +22,34 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" kube::golang::setup_env -"${KUBE_ROOT}/hack/build-go.sh" cmd/gendocs cmd/genman cmd/genbashcomp +"${KUBE_ROOT}/hack/build-go.sh" cmd/gendocs cmd/genman cmd/genbashcomp cmd/mungedocs # Find binary gendocs=$(kube::util::find-binary "gendocs") genman=$(kube::util::find-binary "genman") genbashcomp=$(kube::util::find-binary "genbashcomp") +mungedocs=$(kube::util::find-binary "mungedocs") -if [[ ! -x "$gendocs" || ! -x "$genman" || ! -x "$genbashcomp" ]]; then +if [[ ! -x "$gendocs" || ! -x "$genman" || ! -x "$genbashcomp" || ! -x "$mungedocs" ]]; then { - echo "It looks as if you don't have a compiled gendocs, genman, or genbashcomp binary" + echo "It looks as if you don't have a compiled gendocs, genman, genbashcomp or mungedocs binary" echo echo "If you are running from a clone of the git repo, please run" - echo "'./hack/build-go.sh cmd/gendocs cmd/genman cmd/genbashcomp'." + echo "'./hack/build-go.sh cmd/gendocs cmd/genman cmd/genbashcomp cmd/mungedocs'." } >&2 exit 1 fi +"${mungedocs}" "--root-dir=${KUBE_ROOT}/docs/" +ret=$? +if [[ $ret -eq 1 ]]; then + echo "${KUBE_ROOT}/docs/ requires manual changes. See proceeding errors." + exit 1 +elif [[ $ret -eq 2 ]]; then + echo "Error running mungedocs." + exit 1 +fi + kube::util::gen-doc "${gendocs}" "${KUBE_ROOT}/docs/" '###### Auto generated by spf13/cobra' kube::util::gen-doc "${genman}" "${KUBE_ROOT}/docs/man/man1" kube::util::gen-doc "${genbashcomp}" "${KUBE_ROOT}/contrib/completions/bash/" diff --git a/hack/verify-gendocs.sh b/hack/verify-gendocs.sh index a8db1f2ed3a..331a25edf3e 100755 --- a/hack/verify-gendocs.sh +++ b/hack/verify-gendocs.sh @@ -23,16 +23,18 @@ source "${KUBE_ROOT}/hack/lib/init.sh" kube::golang::setup_env +# Find binary gendocs=$(kube::util::find-binary "gendocs") genman=$(kube::util::find-binary "genman") genbashcomp=$(kube::util::find-binary "genbashcomp") +mungedocs=$(kube::util::find-binary "mungedocs") -if [[ ! -x "$gendocs" || ! -x "$genman" || ! -x "$genbashcomp" ]]; then +if [[ ! -x "$gendocs" || ! -x "$genman" || ! -x "$genbashcomp" || ! -x "$mungedocs" ]]; then { - echo "It looks as if you don't have a compiled gendocs, genman, or genbashcomp binary" + echo "It looks as if you don't have a compiled gendocs, genman, genbashcomp or mungedocs binary" echo echo "If you are running from a clone of the git repo, please run" - echo "'./hack/build-go.sh cmd/gendocs cmd/genman cmd/genbashcomp'." + echo "'./hack/build-go.sh cmd/gendocs cmd/genman cmd/genbashcomp cmd/mungedocs'." } >&2 exit 1 fi @@ -44,6 +46,17 @@ _tmp="${KUBE_ROOT}/_tmp" mkdir -p "${_tmp}" cp -a "${DOCROOT}" "${TMP_DOCROOT}" +"${mungedocs}" "--verify=true" "--root-dir=${TMP_DOCROOT}" +ret=$? +if [[ $ret -eq 1 ]]; then + echo "${DOCROOT} is out of date. Please run hack/run-gendocs.sh" + exit 1 +fi +if [[ $ret -eq 2 ]]; then + echo "Error running mungedocs" + exit 1 +fi + kube::util::gen-doc "${genman}" "${TMP_DOCROOT}/man/man1/" kube::util::gen-doc "${gendocs}" "${TMP_DOCROOT}" echo "diffing ${DOCROOT} against freshly generated docs"