From bc8d572f3a57988529bc9075564422f783248ec4 Mon Sep 17 00:00:00 2001 From: Chao Xu Date: Fri, 5 Aug 2016 22:56:30 -0700 Subject: [PATCH] added copy.sh which cuts the client repo; made necessary changes to scripts in the repo to cope with the client repo staging area; added symlink vendor/k8s.io/client-go which pointed to the staging area. --- hack/godep-save.sh | 8 ++ hack/make-rules/helpers/cache_go_dirs.sh | 1 + hack/make-rules/test.sh | 12 +- hack/make-rules/vet.sh | 2 +- hack/update-codecgen.sh | 1 + staging/src/k8s.io/client-go/README.md | 9 ++ staging/src/k8s.io/client-go/copy.sh | 158 +++++++++++++++++++++++ vendor/k8s.io/client-go | 1 + 8 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 staging/src/k8s.io/client-go/README.md create mode 100755 staging/src/k8s.io/client-go/copy.sh create mode 120000 vendor/k8s.io/client-go diff --git a/hack/godep-save.sh b/hack/godep-save.sh index a23fa7abe6a..1ad351f7f31 100755 --- a/hack/godep-save.sh +++ b/hack/godep-save.sh @@ -19,8 +19,11 @@ set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" +export GOPATH=${GOPATH}:${KUBE_ROOT}/staging GODEP="${GODEP:-godep}" + # Some things we want in godeps aren't code dependencies, so ./... # won't pick them up. REQUIRED_BINS=( @@ -31,4 +34,9 @@ REQUIRED_BINS=( pushd "${KUBE_ROOT}" > /dev/null GO15VENDOREXPERIMENT=1 ${GODEP} save "${REQUIRED_BINS[@]}" + # create a symlink in vendor directory pointing to the staging client. This + # let other packages use the staging client as if it were vendored. + if [ ! -e "vendor/k8s.io/client-go" ]; then + ln -s ../../staging/src/k8s.io/client-go vendor/k8s.io/client-go + fi popd > /dev/null diff --git a/hack/make-rules/helpers/cache_go_dirs.sh b/hack/make-rules/helpers/cache_go_dirs.sh index 133a2d22fa6..693dc37d18b 100755 --- a/hack/make-rules/helpers/cache_go_dirs.sh +++ b/hack/make-rules/helpers/cache_go_dirs.sh @@ -38,6 +38,7 @@ function kfind() { -not \( \ \( \ -path ./vendor -o \ + -path ./staging -o \ -path ./_\* -o \ -path ./.\* -o \ -path ./docs -o \ diff --git a/hack/make-rules/test.sh b/hack/make-rules/test.sh index a2a7b413de0..5a336c0fca7 100755 --- a/hack/make-rules/test.sh +++ b/hack/make-rules/test.sh @@ -26,7 +26,7 @@ kube::golang::setup_env kube::test::find_dirs() { ( cd ${KUBE_ROOT} - find . -not \( \ + find -L . -not \( \ \( \ -path './_artifacts/*' \ -o -path './_output/*' \ @@ -39,10 +39,16 @@ kube::test::find_dirs() { -o -path './test/e2e_node/*' \ -o -path './test/integration/*' \ -o -path './test/component/scheduler/perf/*' \ - -o -path './third_party/*'\ - -o -path './vendor/*'\ + -o -path './third_party/*' \ + -o -path './staging/*' \ + -o -path './vendor/*' \ \) -prune \ \) -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | sort -u + + find -L . \ + -path './_output' -prune \ + -o -path './vendor/k8s.io/client-go/*' \ + -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | sort -u ) } diff --git a/hack/make-rules/vet.sh b/hack/make-rules/vet.sh index fc0e264a590..de468ebc4f4 100755 --- a/hack/make-rules/vet.sh +++ b/hack/make-rules/vet.sh @@ -43,7 +43,7 @@ done if [[ ${#targets[@]} -eq 0 ]]; then # Do not run on third_party directories. - targets=$(go list ./... | egrep -v "/(third_party|vendor)/") + targets=$(go list -e ./... | egrep -v "/(third_party|vendor|staging)/") fi go vet "${goflags[@]:+${goflags[@]}}" ${targets[@]} diff --git a/hack/update-codecgen.sh b/hack/update-codecgen.sh index a09feee16d2..06da0032d8f 100755 --- a/hack/update-codecgen.sh +++ b/hack/update-codecgen.sh @@ -31,6 +31,7 @@ generated_files=($( \( \ -wholename './output' \ -o -wholename './_output' \ + -o -wholename './staging' \ -o -wholename './release' \ -o -wholename './target' \ -o -wholename '*/third_party/*' \ diff --git a/staging/src/k8s.io/client-go/README.md b/staging/src/k8s.io/client-go/README.md new file mode 100644 index 00000000000..1947eafd9c1 --- /dev/null +++ b/staging/src/k8s.io/client-go/README.md @@ -0,0 +1,9 @@ +This _staging/src/k8s.io/client-go directory is the staging area of the client repo. It contains a versioned client, tools built around the client like the reflector, and all the client dependencies. The content will be periodically published to k8s.io/client-go repo. +The staged content is copied from the main repo, i.e., k8s.io/kubernetes, with directory rearrangement and necessary rewritings. To sync the content with the latest code in your local k8s.io/kubernetes, you need to run `godep restore` in k8s root directory, then run _staging/src/k8s.io/client-go/copy.sh. +vendor/k8s.io/client-go is a symlink pointing to this staging area, so to use the packages in the staging area, you can import it as "vendor/client-go/", as if the client were vendored. The client will be vendored from k8s.io/client-go for real after the test matrix is converted to vendor k8s components. + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/_staging/src/k8s.io/client-go/README.md?pixel)]() + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/staging/src/k8s.io/client-go/README.md?pixel)]() diff --git a/staging/src/k8s.io/client-go/copy.sh b/staging/src/k8s.io/client-go/copy.sh new file mode 100755 index 00000000000..2e5027b3717 --- /dev/null +++ b/staging/src/k8s.io/client-go/copy.sh @@ -0,0 +1,158 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# PREREQUISITES: run `godep restore` in the main repo before calling this script. +RELEASE="1.4" +MAIN_REPO_FROM_SRC="${1:-"k8s.io/kubernetes"}" +MAIN_REPO="${GOPATH%:*}/src/${MAIN_REPO_FROM_SRC}" +CLIENT_REPO_FROM_SRC="${2:-"k8s.io/client-go/${RELEASE}"}" +CLIENT_REPO="${MAIN_REPO}/staging/src/${CLIENT_REPO_FROM_SRC}" +CLIENT_REPO_TEMP="${CLIENT_REPO}"/_tmp + +# working in the ${CLIENT_REPO_TEMP} so 'godep save' won't complain about dirty working tree. +echo "creating the _tmp directory" +mkdir -p "${CLIENT_REPO_TEMP}" +cd "${CLIENT_REPO}" + +# mkcp copies file from the main repo to the client repo, it creates the directory if it doesn't exist in the client repo. +function mkcp() { + mkdir -p "${CLIENT_REPO_TEMP}/$2" && cp -r "${MAIN_REPO}/$1" "${CLIENT_REPO_TEMP}/$2" +} + +echo "copying client packages" +mkcp "pkg/client/clientset_generated/release_1_4" "pkg/client/clientset_generated" +mkcp "/pkg/client/record/" "/pkg/client" +mkcp "/pkg/client/cache/" "/pkg/client" +# TODO: make this test file not depending on pkg/client/unversioned +rm "${CLIENT_REPO_TEMP}"/pkg/client/cache/listwatch_test.go +mkcp "/pkg/client/restclient" "/pkg/client" +mkcp "/pkg/client/testing" "/pkg/client" +# remove this test because it imports the internal clientset +rm "${CLIENT_REPO_TEMP}"/pkg/client/testing/core/fake_test.go +mkcp "/pkg/client/transport" "/pkg/client" +mkcp "/pkg/client/typed" "/pkg/client" + +mkcp "/pkg/client/unversioned/auth" "/pkg/client/unversioned" +mkcp "/pkg/client/unversioned/clientcmd" "/pkg/client/unversioned" +mkcp "/pkg/client/unversioned/portforward" "/pkg/client/unversioned" +# remove this test because it imports the internal clientset +rm "${CLIENT_REPO_TEMP}"/pkg/client/unversioned/portforward/portforward_test.go + +pushd "${CLIENT_REPO_TEMP}" +echo "generating vendor/" +GO15VENDOREXPERIMENT=1 godep save ./... +popd + +echo "move to the client repo" +# clean the ${CLIENT_REPO} +ls "${CLIENT_REPO}" | grep -v '_tmp' | xargs rm -r +mv "${CLIENT_REPO_TEMP}"/* "${CLIENT_REPO}" +rm -r "${CLIENT_REPO_TEMP}" +rm -r "${CLIENT_REPO}"/Godeps + +echo "moving vendor/k8s.io/kuberentes" +cp -rn "${CLIENT_REPO}"/vendor/k8s.io/kubernetes/. "${CLIENT_REPO}"/ +# client-go will share the vendor of the main repo for now. When client-go +# becomes a standalone repo, it will have its own vendor +rm -r "${CLIENT_REPO}"/vendor +# remove the pkg/util/net/sets/README.md to silent hack/verify-munge-docs.sh +# TODO: probably we should convert the README.md a doc.go +find ./ -name "README.md" -delete + +echo "rewriting imports" +grep -Rl "\"${MAIN_REPO_FROM_SRC}" ./ | grep ".go" | grep -v "vendor/" | xargs sed -i "s|\"${MAIN_REPO_FROM_SRC}|\"${CLIENT_REPO_FROM_SRC}|g" + +echo "converting pkg/client/record to v1" +# need a v1 version of ref.go +cp "${CLIENT_REPO}"/pkg/api/ref.go "${CLIENT_REPO}"/pkg/api/v1/ref.go +gofmt -w -r 'api.a -> v1.a' "${CLIENT_REPO}"/pkg/api/v1/ref.go +gofmt -w -r 'Scheme -> api.Scheme' "${CLIENT_REPO}"/pkg/api/v1/ref.go +# rewriting package name to v1 +sed -i 's/package api/package v1/g' "${CLIENT_REPO}"/pkg/api/v1/ref.go +# ref.go refers api.Scheme, so manually import /pkg/api +sed -i "s,import (,import (\n\"${CLIENT_REPO_FROM_SRC}/pkg/api\",g" "${CLIENT_REPO}"/pkg/api/v1/ref.go +gofmt -w "${CLIENT_REPO}"/pkg/api/v1/ref.go +# rewrite pkg/client/record to v1 +gofmt -w -r 'api.a -> v1.a' "${CLIENT_REPO}"/pkg/client/record +# need to call sed to rewrite the strings in test cases... +find "${CLIENT_REPO}"/pkg/client/record -type f -name "*.go" -print0 | xargs -0 sed -i "s/api.ObjectReference/v1.ObjectReference/g" +# rewrite the imports +find "${CLIENT_REPO}"/pkg/client/record -type f -name "*.go" -print0 | xargs -0 sed -i 's,pkg/api",pkg/api/v1",g' +# gofmt the changed files + +echo "rewrite conflicting Prometheus registration" +sed -i "s/request_latency_microseconds/request_latency_microseconds_copy/g" "${CLIENT_REPO}"/pkg/client/metrics/metrics.go +sed -i "s/request_status_codes/request_status_codes_copy/g" "${CLIENT_REPO}"/pkg/client/metrics/metrics.go +sed -i "s/kubernetes_build_info/kubernetes_build_info_copy/g" "${CLIENT_REPO}"/pkg/version/version.go + +echo "rewrite proto names in proto.RegisterType" +find "${CLIENT_REPO}" -type f -name "generated.pb.go" -print0 | xargs -0 sed -i "s/k8s\.io\.kubernetes/k8s.io.client-go.1.4/g" + +echo "rearranging directory layout" +# $1 and $2 are relative to ${CLIENT_REPO} +function mvfolder { + local src=${1%/#/} + local dst=${2%/#/} + # create the parent directory of dst + if [ "${dst%/*}" != "${dst}" ]; then + mkdir -p "${CLIENT_REPO}/${dst%/*}" + fi + # move + mv "${CLIENT_REPO}/${src}" "${CLIENT_REPO}/${dst}" + # rewrite package + local src_package="${src##*/}" + local dst_package="${dst##*/}" + find "${CLIENT_REPO}" -type f -name "*.go" -print0 | xargs -0 sed -i "s,package ${src_package},package ${dst_package},g" + # rewrite imports + # the first rule is to convert import lines like `restclient "k8s.io/client-go/pkg/client/restclient"`, + # where a package alias is the same the package name. + find "${CLIENT_REPO}" -type f -name "*.go" -print0 | \ + xargs -0 sed -i "s,${src_package} \"${CLIENT_REPO_FROM_SRC}/${src},${dst_package} \"${CLIENT_REPO_FROM_SRC}/${dst},g" + find "${CLIENT_REPO}" -type f -name "*.go" -print0 | \ + xargs -0 sed -i "s,\"${CLIENT_REPO_FROM_SRC}/${src},\"${CLIENT_REPO_FROM_SRC}/${dst},g" + # rewrite import invocation + if [ "${src_package}" != "${dst_package}" ]; then + find "${CLIENT_REPO}" -type f -name "*.go" -print0 | xargs -0 sed -i "s,\<${src_package}\.\([a-zA-Z]\),${dst_package}\.\1,g" + fi +} + +mvfolder pkg/client/clientset_generated/release_1_4 kubernetes +mvfolder pkg/client/typed/discovery discovery +mvfolder pkg/client/typed/dynamic dynamic +mvfolder pkg/client/transport transport +mvfolder pkg/client/record tools/record +mvfolder pkg/client/restclient rest +mvfolder pkg/client/cache tools/cache +mvfolder pkg/client/unversioned/auth tools/auth +mvfolder pkg/client/unversioned/clientcmd tools/clientcmd +mvfolder pkg/client/unversioned/portforward tools/portforward +mvfolder pkg/client/metrics tools/metrics +mvfolder pkg/client/testing/core testing +if [ "$(find "${CLIENT_REPO}"/pkg/client -type f -name "*.go")" ]; then + echo "${CLIENT_REPO}/pkg/client is expected to be empty" + exit 1 +else + rm -r "${CLIENT_REPO}"/pkg/client +fi +mvfolder third_party pkg/third_party +mvfolder federation pkg/federation + +echo "running gofmt" +find "${CLIENT_REPO}" -type f -name "*.go" -print0 | xargs -0 gofmt -w diff --git a/vendor/k8s.io/client-go b/vendor/k8s.io/client-go new file mode 120000 index 00000000000..6707417f032 --- /dev/null +++ b/vendor/k8s.io/client-go @@ -0,0 +1 @@ +../../staging/src/k8s.io/client-go \ No newline at end of file