Fix tooling for apis/experimental's new home

* fix package name
* add a script to auto-gofmt everything, useful after grep/sed incantations
* update conversion/deep copy generation
* doc update

cmd/genconversion/conversion.go

@@ -44,6 +44,24 @@ var (
 	groupVersion = flag.StringP("version", "v", "api/v1", "groupPath/version for conversion.")
 )
 
+// We're moving to pkg/apis/group/version. This handles new and legacy packages.
+func pkgPath(group, version string) string {
+	if group == "" {
+		group = "api"
+	}
+	gv := group
+	if version != "" {
+		gv = path.Join(group, version)
+	}
+	switch {
+	case group == "api":
+		// TODO(lavalamp): remove this special case when we move api to apis/api
+		return path.Join(pkgBase, gv)
+	default:
+		return path.Join(pkgBase, "apis", gv)
+	}
+}
+
 func main() {
 	runtime.GOMAXPROCS(runtime.NumCPU())
 	flag.Parse()
@@ -70,14 +88,14 @@ func main() {
 		glog.Fatalf("error writing package line: %v", err)
 	}
 
-	versionPath := path.Join(pkgBase, group, version)
+	versionPath := pkgPath(group, version)
 	generator := pkg_runtime.NewConversionGenerator(api.Scheme.Raw(), versionPath)
 	apiShort := generator.AddImport(path.Join(pkgBase, "api"))
 	generator.AddImport(path.Join(pkgBase, "api/resource"))
 	// TODO(wojtek-t): Change the overwrites to a flag.
 	generator.OverwritePackage(version, "")
 	for _, knownType := range api.Scheme.KnownTypes(version) {
-		if !strings.HasPrefix(knownType.PkgPath(), versionPath) {
+		if knownType.PkgPath() != versionPath {
 			continue
 		}
 		if err := generator.GenerateConversionsForType(version, knownType); err != nil {

cmd/gendeepcopy/deep_copy.go

@@ -45,6 +45,32 @@ var (
 	overwrites   = flag.StringP("overwrites", "o", "", "Comma-separated overwrites for package names")
 )
 
+// types inside the api package don't need to say "api.Scheme"; all others do.
+func destScheme(group, version string) string {
+	if group == "api" && version == "" {
+		return "Scheme"
+	}
+	return "api.Scheme"
+}
+
+// We're moving to pkg/apis/group/version. This handles new and legacy packages.
+func pkgPath(group, version string) string {
+	if group == "" {
+		group = "api"
+	}
+	gv := group
+	if version != "" {
+		gv = path.Join(group, version)
+	}
+	switch {
+	case group == "api":
+		// TODO(lavalamp): remove this special case when we move api to apis/api
+		return path.Join(pkgBase, gv)
+	default:
+		return path.Join(pkgBase, "apis", gv)
+	}
+}
+
 func main() {
 	runtime.GOMAXPROCS(runtime.NumCPU())
 	flag.Parse()
@@ -65,10 +91,7 @@ func main() {
 
 	group, version := path.Split(*groupVersion)
 	group = strings.TrimRight(group, "/")
-	registerTo := "api.Scheme"
+	registerTo := destScheme(group, version)
-	if *groupVersion == "api/" {
-		registerTo = "Scheme"
-	}
 	pkgname := group
 	if len(version) != 0 {
 		pkgname = version
@@ -79,7 +102,7 @@ func main() {
 		glog.Fatalf("error writing package line: %v", err)
 	}
 
-	versionPath := path.Join(pkgBase, group, version)
+	versionPath := pkgPath(group, version)
 	generator := pkg_runtime.NewDeepCopyGenerator(api.Scheme.Raw(), versionPath, sets.NewString("k8s.io/kubernetes"))
 	generator.AddImport(path.Join(pkgBase, "api"))
 
@@ -93,7 +116,7 @@ func main() {
 		}
 	}
 	for _, knownType := range api.Scheme.KnownTypes(version) {
-		if !strings.HasPrefix(knownType.PkgPath(), versionPath) {
+		if knownType.PkgPath() != versionPath {
 			continue
 		}
 		if err := generator.AddType(knownType); err != nil {

docs/devel/api_changes.md

@@ -38,7 +38,7 @@ with a number of existing API types and with the [API
 conventions](api-conventions.md).  If creating a new API
 type/resource, we also recommend that you first send a PR containing
 just a proposal for the new API types, and that you initially target
-the experimental API (pkg/expapi).
+the experimental API (pkg/apis/experimental).
 
 The Kubernetes API has two major components - the internal structures and
 the versioned APIs.  The versioned APIs are intended to be stable, while the
@@ -399,10 +399,10 @@ The conversion code resides with each versioned API. There are two files:
      functions
    - `pkg/api/<version>/conversion_generated.go` containing auto-generated
      conversion functions
-   - `pkg/expapi/<version>/conversion.go` containing manually written conversion
+   - `pkg/apis/experimental/<version>/conversion.go` containing manually written
-     functions
-   - `pkg/expapi/<version>/conversion_generated.go` containing auto-generated
      conversion functions
+   - `pkg/apis/experimental/<version>/conversion_generated.go` containing
+     auto-generated conversion functions
 
 Since auto-generated conversion functions are using manually written ones,
 those manually written should be named with a defined convention, i.e. a function
@@ -437,7 +437,7 @@ of your versioned api objects.
 
 The deep copy code resides with each versioned API:
    - `pkg/api/<version>/deep_copy_generated.go` containing auto-generated copy functions
-   - `pkg/expapi/<version>/deep_copy_generated.go` containing auto-generated copy functions
+   - `pkg/apis/experimental/<version>/deep_copy_generated.go` containing auto-generated copy functions
 
 To regenerate them:
    - run
@@ -446,6 +446,23 @@ To regenerate them:
 hack/update-generated-deep-copies.sh
 ```
 
+## Making a new API Group
+
+This section is under construction, as we make the tooling completely generic.
+
+At the moment, you'll have to make a new directory under pkg/apis/; copy the
+directory structure from pkg/apis/experimental. Add the new group/version to all
+of the hack/{verify,update}-generated-{deep-copy,conversions,swagger}.sh files
+in the appropriate places--it should just require adding your new group/version
+to a bash array. You will also need to make sure your new types are imported by
+the generation commands (cmd/gendeepcopy/ & cmd/genconversion). These
+instructions may not be complete and will be updated as we gain experience.
+
+Adding API groups outside of the pkg/apis/ directory is not currently supported,
+but is clearly desirable. The deep copy & conversion generators need to work by
+parsing go files instead of by reflection; then they will be easy to point at
+arbitrary directories: see issue [#13775](http://issue.k8s.io/13775).
+
 ## Update the fuzzer
 
 Part of our testing regimen for APIs is to "fuzz" (fill with random values) API

hack/after-build/update-generated-conversions.sh

@@ -26,10 +26,10 @@ kube::golang::setup_env
 genconversion=$(kube::util::find-binary "genconversion")
 
 function generate_version() {
-	local version=$1
+  local group_version=$1
   local TMPFILE="/tmp/conversion_generated.$(date +%s).go"
 
-	echo "Generating for ${version}"
+  echo "Generating for ${group_version}"
 
   sed 's/YEAR/2015/' hack/boilerplate/boilerplate.go.txt > "$TMPFILE"
   cat >> "$TMPFILE" <<EOF
@@ -37,13 +37,14 @@ function generate_version() {
 
 EOF
 
-  "${genconversion}" -v "${version}" -f - >>  "$TMPFILE"
+  "${genconversion}" -v "${group_version}" -f - >>  "$TMPFILE"
 
-	mv "$TMPFILE" "pkg/${version}/conversion_generated.go"
+  mv "$TMPFILE" "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/conversion_generated.go"
 }
 
-DEFAULT_VERSIONS="api/v1 expapi/v1"
+# TODO(lavalamp): get this list by listing the pkg/apis/ directory?
-VERSIONS=${VERSIONS:-$DEFAULT_VERSIONS}
+DEFAULT_GROUP_VERSIONS="api/v1 experimental/v1"
+VERSIONS=${VERSIONS:-$DEFAULT_GROUP_VERSIONS}
 for ver in $VERSIONS; do
   # Ensure that the version being processed is registered by setting
   # KUBE_API_VERSIONS.

hack/after-build/update-generated-deep-copies.sh

@@ -25,16 +25,11 @@ kube::golang::setup_env
 
 gendeepcopy=$(kube::util::find-binary "gendeepcopy")
 
-function result_file_name() {
-	local version=$1
-	echo "pkg/${version}/deep_copy_generated.go"
-}
-
 function generate_version() {
-	local version=$1
+  local group_version=$1
   local TMPFILE="/tmp/deep_copy_generated.$(date +%s).go"
 
-	echo "Generating for ${version}"
+  echo "Generating for ${group_version}"
 
   sed 's/YEAR/2015/' hack/boilerplate/boilerplate.go.txt > $TMPFILE
   cat >> $TMPFILE <<EOF
@@ -42,18 +37,16 @@ function generate_version() {
 
 EOF
 
-	"${gendeepcopy}" -v "${version}" -f - -o "${version}=" >>  "$TMPFILE"
+  "${gendeepcopy}" -v "${group_version}" -f - -o "${group_version}=" >>  "$TMPFILE"
 
-	mv "$TMPFILE" `result_file_name ${version}`
+  local dest="pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/deep_copy_generated.go"
+  rm -f "${dest}"
+  mv "${TMPFILE}" "${dest}"
 }
 
 function generate_deep_copies() {
-	local versions="$@"
+  local group_versions="$@"
-	# To avoid compile errors, remove the currently existing files.
+  for ver in ${group_versions}; do
-	for ver in ${versions}; do
-		rm -f `result_file_name ${ver}`
-	done
-	for ver in ${versions}; do
     # Ensure that the version being processed is registered by setting
     # KUBE_API_VERSIONS.
     apiVersions="${ver##*/}"
@@ -61,6 +54,6 @@ function generate_deep_copies() {
   done
 }
 
-DEFAULT_VERSIONS="api/ api/v1 expapi/ expapi/v1"
+DEFAULT_VERSIONS="api/ api/v1 experimental/ experimental/v1"
 VERSIONS=${VERSIONS:-$DEFAULT_VERSIONS}
 generate_deep_copies "$VERSIONS"

hack/after-build/verify-description.sh

@@ -40,7 +40,7 @@ find_files() {
       \) -prune \
     \) \
     \( -wholename '*pkg/api/v*/types.go' \
-       -o -wholename '*pkg/expapi/v*/types.go' \
+       -o -wholename '*pkg/apis/*/v*/types.go' \
     \)
 }
 
@@ -61,7 +61,7 @@ for file in $versioned_api_files; do
   fi
 done
 
-internal_types_files="${KUBE_ROOT}/pkg/api/types.go ${KUBE_ROOT}/pkg/expapi/types.go"
+internal_types_files="${KUBE_ROOT}/pkg/api/types.go ${KUBE_ROOT}/pkg/apis/experimental/types.go"
 for internal_types_file in $internal_types_files; do
   if grep json: "${internal_types_file}" | grep -v // | grep description: ; then
     echo "Internal API types should not contain descriptions"

hack/after-build/verify-generated-conversions.sh

@@ -23,7 +23,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 
 kube::golang::setup_env
 
-APIROOTS=${APIROOTS:-pkg/api pkg/expapi}
+APIROOTS=${APIROOTS:-pkg/api pkg/apis/experimental}
 _tmp="${KUBE_ROOT}/_tmp"
 
 cleanup() {

hack/after-build/verify-generated-deep-copies.sh

@@ -25,7 +25,7 @@ kube::golang::setup_env
 
 gendeepcopy=$(kube::util::find-binary "gendeepcopy")
 
-APIROOTS=${APIROOTS:-pkg/api pkg/expapi}
+APIROOTS=${APIROOTS:-pkg/api pkg/apis/experimental}
 _tmp="${KUBE_ROOT}/_tmp"
 
 cleanup() {

hack/lib/util.sh

@@ -220,4 +220,35 @@ kube::util::analytics-link() {
   echo "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/${path}?pixel)]()"
 }
 
+# Takes a group/version and returns the path to its location on disk, sans
+# "pkg". E.g.:
+# * default behavior: experimental/v1 -> apis/experimental/v1
+# * legacy behavior: api/v1 -> api/v1
+# * Special handling for only a group: experimental -> apis/experimental
+# * Special handling for only "api" group: api -> api
+# * Very special handling for "v1": v1 -> api/v1
+kube::util::group-version-to-pkg-path() {
+  local group_version="$1"
+  # Special cases first.
+  # TODO(lavalamp): Simplify this by moving pkg/api/v1 and splitting pkg/api,
+  # moving the results to pkg/apis/api.
+  case "${group_version}" in
+    v1)
+      echo "api/v1"
+      ;;
+    api)
+      echo "api/v1"
+      ;;
+    api/*)
+      echo "${group_version}"
+      ;;
+    api/*)
+      echo "${group_version}"
+      ;;
+    *)
+      echo "apis/${group_version}"
+      ;;
+  esac
+}
+
 # ex: ts=2 sw=2 et filetype=sh

hack/update-generated-swagger-docs.sh

@@ -24,13 +24,13 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 kube::golang::setup_env
 
 function generate_version() {
-	local groupVersion=$1
+  local group_version=$1
   local TMPFILE="/tmp/types_swagger_doc_generated.$(date +%s).go"
 
-	echo "Generating swagger type docs for ${groupVersion}"
+  echo "Generating swagger type docs for ${group_version}"
 
   sed 's/YEAR/2015/' hack/boilerplate/boilerplate.go.txt > $TMPFILE
-  echo "package ${groupVersion##*/}" >> $TMPFILE
+  echo "package ${group_version##*/}" >> $TMPFILE
   cat >> $TMPFILE <<EOF
 
 // This file contains a collection of methods that can be used from go-resful to
@@ -46,21 +46,23 @@ function generate_version() {
 // AUTO-GENERATED FUNCTIONS START HERE
 EOF
 
-	GOPATH=$(godep path):$GOPATH go run cmd/genswaggertypedocs/swagger_type_docs.go -s "pkg/${groupVersion}/types.go" -f - >>  $TMPFILE
+  GOPATH=$(godep path):$GOPATH go run cmd/genswaggertypedocs/swagger_type_docs.go -s \
+    "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/types.go" -f - \
+    >>  $TMPFILE
 
   echo "// AUTO-GENERATED FUNCTIONS END HERE" >> $TMPFILE
 
   gofmt -w -s $TMPFILE
-	mv $TMPFILE "pkg/${groupVersion}/types_swagger_doc_generated.go"
+  mv $TMPFILE "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/types_swagger_doc_generated.go"
 }
 
-GROUP_VERSIONS="api/v1 expapi/v1"
+GROUP_VERSIONS="api/v1 experimental/v1"
 # To avoid compile errors, remove the currently existing files.
-for groupVersion in $GROUP_VERSIONS; do
+for group_version in $GROUP_VERSIONS; do
-	rm -f "pkg/${groupVersion}/types_swagger_doc_generated.go"
+  rm -f "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/types_swagger_doc_generated.go"
 done
-for groupVersion in $GROUP_VERSIONS; do
+for group_version in $GROUP_VERSIONS; do
-	generate_version "${groupVersion}"
+  generate_version "${group_version}"
 done
 
 "${KUBE_ROOT}/hack/update-swagger-spec.sh"

hack/update-gofmt.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# GoFmt apparently is changing @ head...
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+
+GO_VERSION=($(go version))
+
+if [[ -z $(echo "${GO_VERSION[2]}" | grep -E 'go1.2|go1.3|go1.4|go1.5') ]]; then
+  echo "Unknown go version '${GO_VERSION}', skipping gofmt."
+  exit 0
+fi
+
+cd "${KUBE_ROOT}"
+
+find_files() {
+  find . -not \( \
+      \( \
+        -wholename './output' \
+        -o -wholename './_output' \
+        -o -wholename './release' \
+        -o -wholename './target' \
+        -o -wholename '*/third_party/*' \
+        -o -wholename '*/Godeps/*' \
+      \) -prune \
+    \) -name '*.go'
+}
+
+GOFMT="gofmt -s -w"
+find_files | xargs $GOFMT

pkg/apis/experimental/deep_copy_generated.go

@@ -16,7 +16,7 @@ limitations under the License.
 
 // DO NOT EDIT. THIS FILE IS AUTO-GENERATED BY $KUBEROOT/hack/update-generated-deep-copies.sh.
 
-package expapi
+package experimental
 
 import (
 	time "time"

pkg/apis/experimental/register.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package expapi
+package experimental
 
 import (
 	"k8s.io/kubernetes/pkg/api"