diff --git a/.gitignore b/.gitignore index 889e9c8481e..de1b35148e5 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,9 @@ network_closure.sh /third_party/etcd* /default.etcd +# Also ignore protoc installed by hack/install-protoc.sh +/third_party/protoc* + # User cluster configs .kubeconfig diff --git a/build/dependencies.yaml b/build/dependencies.yaml index 7edab99463c..2f0a5de394b 100644 --- a/build/dependencies.yaml +++ b/build/dependencies.yaml @@ -53,6 +53,13 @@ dependencies: - path: cluster/gce/gci/configure.sh match: DEFAULT_CRICTL_VERSION= + # protoc + - name: "protoc" + version: 3.19.4 + refPaths: + - path: hack/lib/protoc.sh + match: PROTOC_VERSION= + # etcd - name: "etcd" version: 3.5.6 diff --git a/hack/install-protoc.sh b/hack/install-protoc.sh new file mode 100755 index 00000000000..9361b01c98b --- /dev/null +++ b/hack/install-protoc.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is convenience to download and install protoc in third_party. +# Usage: `hack/install-protoc.sh`. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/protoc.sh" + +kube::protoc::install diff --git a/hack/lib/init.sh b/hack/lib/init.sh index 17f69c91c53..90086ad830a 100755 --- a/hack/lib/init.sh +++ b/hack/lib/init.sh @@ -18,6 +18,9 @@ set -o errexit set -o nounset set -o pipefail +# Short-circuit if init.sh has already been sourced +[[ $(type -t kube::init::loaded) == function ]] && return 0 + # Unset CDPATH so that path interpolation can work correctly # https://github.com/kubernetes/kubernetes/issues/52255 unset CDPATH @@ -211,3 +214,8 @@ kube::realpath() { fi kube::readlinkdashf "${1}" } + +# Marker function to indicate init.sh has been fully sourced +kube::init::loaded() { + return 0 +} diff --git a/hack/lib/protoc.sh b/hack/lib/protoc.sh index 1b02310f544..5c1e9a372a1 100644 --- a/hack/lib/protoc.sh +++ b/hack/lib/protoc.sh @@ -18,10 +18,15 @@ set -o errexit set -o nounset set -o pipefail +# Short-circuit if protoc.sh has already been sourced +[[ $(type -t kube::protoc::loaded) == function ]] && return 0 + # The root of the build/dist directory KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" source "${KUBE_ROOT}/hack/lib/init.sh" +PROTOC_VERSION=3.19.4 + # Generates $1/api.pb.go from the protobuf file $1/api.proto # and formats it correctly # $1: Full path to the directory where the api.proto file is @@ -36,17 +41,15 @@ function kube::protoc::generate_proto() { kube::protoc::format "${package}" } -# Checks that the current protoc version is at least version 3.0.0-beta1 +# Checks that the current protoc version matches the required version and # exit 1 if it's not the case function kube::protoc::check_protoc() { - if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3."* ]]; then - echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and" - echo "install the platform appropriate Protobuf package for your OS: " - echo - echo " https://github.com/protocolbuffers/protobuf/releases" - echo - echo "WARNING: Protobuf changes are not being validated" - exit 1 + if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc ${PROTOC_VERSION}"* ]]; then + echo "Generating protobuf requires protoc ${PROTOC_VERSION}." + echo "Run hack/install-protoc.sh or download and install the" + echo "platform-appropriate Protobuf package for your OS from" + echo "https://github.com/protocolbuffers/protobuf/releases" + return 1 fi } @@ -66,6 +69,7 @@ function kube::protoc::protoc() { PATH="${gogopath}:${PATH}" protoc \ --proto_path="$(pwd -P)" \ --proto_path="${KUBE_ROOT}/vendor" \ + --proto_path="${KUBE_ROOT}/third_party/protobuf" \ --gogo_out=paths=source_relative,plugins=grpc:. \ api.proto ) @@ -94,3 +98,48 @@ function kube::protoc::diff() { exit 1 fi } + +function kube::protoc::install() { + # run in a subshell to isolate caller from directory changes + ( + local os + local arch + local download_folder + local download_file + + os=$(kube::util::host_os) + arch=$(kube::util::host_arch) + download_folder="protoc-v${PROTOC_VERSION}-${os}-${arch}" + download_file="${download_folder}.zip" + + cd "${KUBE_ROOT}/third_party" || return 1 + if [[ $(readlink protoc) != "${download_folder}" ]]; then + local url + if [[ ${os} == "darwin" ]]; then + # TODO: switch to universal binary when updating to 3.20+ + url="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-osx-x86_64.zip" + elif [[ ${os} == "linux" && ${arch} == "amd64" ]]; then + url="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" + elif [[ ${os} == "linux" && ${arch} == "arm64" ]]; then + url="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-aarch_64.zip" + else + kube::log::info "This install script does not support ${os}/${arch}" + return 1 + fi + kube::util::download_file "${url}" "${download_file}" + unzip -o "${download_file}" -d "${download_folder}" + ln -fns "${download_folder}" protoc + mv protoc/bin/protoc protoc/protoc + chmod -R +rX protoc/protoc + rm -fr protoc/include + rm "${download_file}" + fi + kube::log::info "protoc v${PROTOC_VERSION} installed. To use:" + kube::log::info "export PATH=\"$(pwd)/protoc:\${PATH}\"" + ) +} + +# Marker function to indicate protoc.sh has been fully sourced +kube::protoc::loaded() { + return 0 +} diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 18e1b09e481..ae451465b1b 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -24,6 +24,7 @@ KUBE_VERBOSE="${KUBE_VERBOSE:-1}" KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" +source "${KUBE_ROOT}/hack/lib/protoc.sh" cd "${KUBE_ROOT}" kube::golang::setup_env @@ -115,7 +116,12 @@ function codegen::protobuf() { ':(glob)**/generated.pb.go' \ | xargs -0 rm -f - build/run.sh hack/update-generated-protobuf-dockerized.sh "${apis[@]}" + if kube::protoc::check_protoc >/dev/null; then + hack/update-generated-protobuf-dockerized.sh "${apis[@]}" + else + kube::log::status "protoc ${PROTOC_VERSION} not found (can install with hack/install-protoc.sh); generating containerized..." + build/run.sh hack/update-generated-protobuf-dockerized.sh "${apis[@]}" + fi } # Generates types_swagger_doc_generated file for the given group version. @@ -918,10 +924,15 @@ function codegen::protobindings() { | xargs -0 rm -f done - # NOTE: All output from this script needs to be copied back to the calling - # source tree. This is managed in kube::build::copy_output in build/common.sh. - # If the output set is changed update that function. - build/run.sh hack/update-generated-proto-bindings-dockerized.sh "${apis[@]}" + if kube::protoc::check_protoc >/dev/null; then + hack/update-generated-proto-bindings-dockerized.sh "${apis[@]}" + else + kube::log::status "protoc ${PROTOC_VERSION} not found (can install with hack/install-protoc.sh); generating containerized..." + # NOTE: All output from this script needs to be copied back to the calling + # source tree. This is managed in kube::build::copy_output in build/common.sh. + # If the output set is changed update that function. + build/run.sh hack/update-generated-proto-bindings-dockerized.sh "${apis[@]}" + fi } # diff --git a/hack/update-generated-proto-bindings-dockerized.sh b/hack/update-generated-proto-bindings-dockerized.sh index cd5dab862cb..163143b1af4 100755 --- a/hack/update-generated-proto-bindings-dockerized.sh +++ b/hack/update-generated-proto-bindings-dockerized.sh @@ -22,6 +22,7 @@ set -o pipefail KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd -P)" +source "${KUBE_ROOT}/hack/lib/init.sh" source "${KUBE_ROOT}/hack/lib/protoc.sh" source "${KUBE_ROOT}/hack/lib/util.sh" @@ -30,6 +31,8 @@ if [ "$#" == 0 ]; then exit 1 fi +kube::protoc::check_protoc + for api; do # This can't use `git ls-files` because it runs in a container without the # .git dir synced. diff --git a/hack/update-generated-protobuf-dockerized.sh b/hack/update-generated-protobuf-dockerized.sh index d56218165c9..64dda03ec5b 100755 --- a/hack/update-generated-protobuf-dockerized.sh +++ b/hack/update-generated-protobuf-dockerized.sh @@ -25,22 +25,14 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" +source "${KUBE_ROOT}/hack/lib/protoc.sh" +kube::protoc::check_protoc kube::golang::setup_env GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/go-to-protobuf GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo -if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3."* ]]; then - echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and" - echo "install the platform appropriate Protobuf package for your OS: " - echo - echo " https://github.com/protocolbuffers/protobuf/releases" - echo - echo "WARNING: Protobuf changes are not being validated" - exit 1 -fi - gotoprotobuf=$(kube::util::find-binary "go-to-protobuf") # requires the 'proto' tag to build (will remove when ready)