Merge pull request #115245 from thockin/codegen-10-protobuf-from-update-codegen

Call update-generated-protobuf from update-codegen

hack/make-rules/update.sh

@@ -36,7 +36,6 @@ if ! ${ALL} ; then
 fi
 
 BASH_TARGETS=(
-	update-generated-protobuf
 	update-codegen
 	update-generated-api-compatibility-data
 	update-generated-docs

hack/update-codegen.sh

@@ -83,6 +83,41 @@ fi
 # Code generation logic.
 #
 
+# protobuf generation
+#
+# Some of the later codegens depend on the results of this, so it needs to come
+# first in the case of regenerating everything.
+function codegen::protobuf() {
+    # NOTE: All output from this script needs to be copied back to the calling
+    # source tree.  This is managed in kube::build::copy_output in build/common.sh.
+    # If the output set is changed update that function.
+
+    local apis=()
+    kube::util::read-array apis < <(
+        git grep --untracked --null -l \
+            -e '// +k8s:protobuf-gen=package' \
+            -- \
+            cmd pkg staging \
+            | xargs -0 -n1 dirname \
+            | sed 's|^|k8s.io/kubernetes/|;s|k8s.io/kubernetes/staging/src/||' \
+            | LC_ALL=C sort -u)
+
+    kube::log::status "Generating protobufs for ${#apis[@]} targets"
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
+        kube::log::status "DBG: generating protobufs for:"
+        for dir in "${apis[@]}"; do
+            kube::log::status "DBG:     $dir"
+        done
+    fi
+
+    git_find -z \
+        ':(glob)**/generated.proto' \
+        ':(glob)**/generated.pb.go' \
+        | xargs -0 rm -f
+
+    build/run.sh hack/update-generated-protobuf-dockerized.sh "${apis[@]}"
+}
+
 # prerelease-lifecycle generation
 #
 # Any package that wants prerelease-lifecycle functions generated must include a

hack/update-generated-protobuf-dockerized.sh

@@ -16,7 +16,7 @@
 
 # This script genertates `*/api.pb.go` from the protobuf file `*/api.proto`.
 # Usage: 
-#     hack/update-generated-protobuf-dockerized.sh "${APIROOTS}"
+#     hack/update-generated-protobuf-dockerized.sh "${APIROOTS[@]}"
 #     An example APIROOT is: "k8s.io/api/admissionregistration/v1"
 
 set -o errexit
@@ -43,11 +43,6 @@ fi
 
 gotoprotobuf=$(kube::util::find-binary "go-to-protobuf")
 
-while IFS=$'\n' read -r line; do
-  APIROOTS+=( "$line" );
-done <<< "${1}"
-shift
-
 # requires the 'proto' tag to build (will remove when ready)
 # searches for the protoc-gen-gogo extension in the output directory
 # satisfies import of github.com/gogo/protobuf/gogoproto/gogo.proto and the
@@ -56,6 +51,5 @@ PATH="${KUBE_ROOT}/_output/bin:${PATH}" \
   "${gotoprotobuf}" \
   --proto-import="${KUBE_ROOT}/vendor" \
   --proto-import="${KUBE_ROOT}/third_party/protobuf" \
-  --packages="$(IFS=, ; echo "${APIROOTS[*]}")" \
+  --packages="$(IFS=, ; echo "$*")" \
-  --go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \
+  --go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt"
-  "$@"

hack/update-generated-protobuf.sh

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script generates all go files from the corresponding protobuf files.
-# Usage: `hack/update-generated-protobuf.sh`.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-
-# NOTE: All output from this script needs to be copied back to the calling
-# source tree.  This is managed in kube::build::copy_output in build/common.sh.
-# If the output set is changed update that function.
-
-APIROOTS=${APIROOTS:-$( \
-    git grep --untracked --null -l \
-        -e '// +k8s:protobuf-gen=package' \
-        -- \
-        cmd pkg staging \
-        | xargs -0 -n1 dirname \
-        | sed 's,^,k8s.io/kubernetes/,;s,k8s.io/kubernetes/staging/src/,,' \
-        | sort -u
-)}
-
-function git_find() {
-    # Similar to find but faster and easier to understand.  We want to include
-    # modified and untracked files because this might be running against code
-    # which is not tracked by git yet.
-    git ls-files -cmo --exclude-standard ':!:vendor/*' "$@"
-}
-
-git_find -z ':(glob)**/generated.proto' | xargs -0 rm -f
-git_find -z ':(glob)**/generated.pb.go' | xargs -0 rm -f
-
-"${KUBE_ROOT}/build/run.sh" hack/update-generated-protobuf-dockerized.sh "${APIROOTS}" "$@"

hack/verify-generated-protobuf.sh

@@ -1,65 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script checks whether updating of the packages which explicitly
-# requesting generation is needed or not. We should run
-# `hack/update-generated-protobuf.sh` if those packages are out of date.
-# Usage: `hack/verify-generated-protobuf.sh`.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
-
-kube::golang::setup_env
-
-APIROOTS=$({
-  # gather the packages explicitly requesting generation
-  git grep --files-with-matches -e '// +k8s:protobuf-gen=package' cmd pkg staging | xargs -n 1 dirname
-  # add the root apimachinery pkg containing implicitly generated files (--apimachinery-packages)
-  echo staging/src/k8s.io/apimachinery/pkg
-} | sort | uniq)
-
-_tmp="${KUBE_ROOT}/_tmp"
-
-cleanup() {
-  rm -rf "${_tmp}"
-}
-
-trap "cleanup" EXIT SIGINT
-
-cleanup
-for APIROOT in ${APIROOTS}; do
-  mkdir -p "${_tmp}/${APIROOT}"
-  cp -a "${KUBE_ROOT}/${APIROOT}"/* "${_tmp}/${APIROOT}/"
-done
-
-KUBE_VERBOSE=3 "${KUBE_ROOT}/hack/update-generated-protobuf.sh"
-for APIROOT in ${APIROOTS}; do
-  TMP_APIROOT="${_tmp}/${APIROOT}"
-  echo "diffing ${APIROOT} against freshly generated protobuf"
-  ret=0
-  diff -Naupr -I 'Auto generated by' -x 'zz_generated.*' -x '.github' -x '.import-restrictions' "${KUBE_ROOT}/${APIROOT}" "${TMP_APIROOT}" || ret=$?
-  cp -a "${TMP_APIROOT}"/* "${KUBE_ROOT}/${APIROOT}/"
-  if [[ $ret -eq 0 ]]; then
-    echo "${APIROOT} up to date."
-  else
-    echo "${APIROOT} is out of date. Please run hack/update-generated-protobuf.sh"
-    exit 1
-  fi
-done

staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/cmd.go

@@ -273,11 +273,9 @@ func Run(g *Generator) {
 		// generate the gogoprotobuf protoc
 		cmd := exec.Command("protoc", append(args, path)...)
 		out, err := cmd.CombinedOutput()
-		if len(out) > 0 {
-			log.Print(string(out))
-		}
 		if err != nil {
 			log.Println(strings.Join(cmd.Args, " "))
+			log.Println(string(out))
 			log.Fatalf("Unable to generate protoc on %s: %v", p.PackageName, err)
 		}
 
@@ -397,9 +395,9 @@ func importOrder(deps map[string][]string) ([]string, error) {
 	if len(remainingNodes) > 0 {
 		return nil, fmt.Errorf("cycle: remaining nodes: %#v, remaining edges: %#v", remainingNodes, graph)
 	}
-	for _, n := range sorted {
+	//for _, n := range sorted {
-		fmt.Println("topological order", n)
+	//	fmt.Println("topological order", n)
-	}
+	//}
 	return sorted, nil
 }