From c1c3fdefe3d5624e7ae075630472c9f0a325a226 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Sat, 27 Feb 2021 23:49:21 -0800 Subject: [PATCH 1/3] port conformance yaml generation to hack --- hack/update-conformance-yaml.sh | 27 +++++++++++++++++++ hack/verify-conformance-yaml.sh | 33 ++++++++++++++++++++++++ test/conformance/README.md | 7 +---- test/conformance/gen-conformance-yaml.sh | 28 ++++++++++++++++++++ test/conformance/gen-specsummaries.sh | 29 +++++++++++++++++++++ test/conformance/spec-to-yaml.sh | 26 +++++++++++++++++++ 6 files changed, 144 insertions(+), 6 deletions(-) create mode 100755 hack/update-conformance-yaml.sh create mode 100755 hack/verify-conformance-yaml.sh create mode 100755 test/conformance/gen-conformance-yaml.sh create mode 100755 test/conformance/gen-specsummaries.sh create mode 100755 test/conformance/spec-to-yaml.sh diff --git a/hack/update-conformance-yaml.sh b/hack/update-conformance-yaml.sh new file mode 100755 index 00000000000..fc66e411f24 --- /dev/null +++ b/hack/update-conformance-yaml.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +cd "${KUBE_ROOT}" + +# generate json spec -> yaml +test/conformance/gen-conformance-yaml.sh +# replace checked-in yaml +cp _output/conformance.yaml test/conformance/testdata/conformance.yaml diff --git a/hack/verify-conformance-yaml.sh b/hack/verify-conformance-yaml.sh new file mode 100755 index 00000000000..0795d208e3f --- /dev/null +++ b/hack/verify-conformance-yaml.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +cd "${KUBE_ROOT}" + +# generate json spec -> yaml +test/conformance/gen-conformance-yaml.sh + +# diff generated and checked-in +if diff -u test/conformance/testdata/conformance.yaml _output/conformance.yaml; then + echo PASS + exit 0 +fi +echo 'See instructions in test/conformance/README.md' +exit 1 diff --git a/test/conformance/README.md b/test/conformance/README.md index 7a5a82904f8..221bab4fa44 100644 --- a/test/conformance/README.md +++ b/test/conformance/README.md @@ -5,11 +5,6 @@ If you add or remove a conformance test, this test will fail and you will need to update the golden list of tests stored in `testdata/`. Changes to that file require review by sig-architecture. -To update the list, run - -```console -bazel build //test/conformance:list_conformance_tests -cp bazel-bin/test/conformance/conformance.yaml test/conformance/testdata -``` +To update the list, run `hack/update-conformance-yaml.sh` Add the changed file to your PR, then send for review. diff --git a/test/conformance/gen-conformance-yaml.sh b/test/conformance/gen-conformance-yaml.sh new file mode 100755 index 00000000000..4f25087670e --- /dev/null +++ b/test/conformance/gen-conformance-yaml.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# This will canonicalize the path +KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd -P) +cd "${KUBE_ROOT}" + +# test sources -> json spec _output/specsummaries.json +test/conformance/gen-specsummaries.sh +# specsummaries.json -> conformance.yaml +test/conformance/spec-to-yaml.sh diff --git a/test/conformance/gen-specsummaries.sh b/test/conformance/gen-specsummaries.sh new file mode 100755 index 00000000000..16d9a1280e2 --- /dev/null +++ b/test/conformance/gen-specsummaries.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# This will canonicalize the path +KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd -P) +cd "${KUBE_ROOT}" + +# build ginkgo and e2e.test +make WHAT='vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test' + +# dump spec +./_output/bin/ginkgo --dryRun=true --focus='[Conformance]' ./_output/bin/e2e.test -- --spec-dump "${KUBE_ROOT}/_output/specsummaries.json" > /dev/null diff --git a/test/conformance/spec-to-yaml.sh b/test/conformance/spec-to-yaml.sh new file mode 100755 index 00000000000..14a39640d1c --- /dev/null +++ b/test/conformance/spec-to-yaml.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# This will canonicalize the path +KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd -P) +cd "${KUBE_ROOT}" + +# convert dumped spec (see dump-spec.sh) to conformance.yaml +go run ./test/conformance/walk.go --source="${KUBE_ROOT}" ./_output/specsummaries.json > ./_output/conformance.yaml \ No newline at end of file From ed09168e8a63acd4680f3284905e96da81d1c63e Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Sat, 27 Feb 2021 23:49:53 -0800 Subject: [PATCH 2/3] fix generated conformance filepaths when test binary is not stripped --- test/conformance/walk.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/conformance/walk.go b/test/conformance/walk.go index c44a43f76dc..d31d66883a8 100644 --- a/test/conformance/walk.go +++ b/test/conformance/walk.go @@ -209,11 +209,14 @@ func getConformanceDataFromStackTrace(fullstackstrace string) (*ConformanceData, i += 2 } - // filenames have `/go/src/k8s.io` prefix which dont exist locally - for i, v := range frames { - frames[i].File = strings.Replace(v.File, - "/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes", - *k8sPath, 1) + // filenames are in one of two special GOPATHs depending on if they were + // built dockerized or with the host go + // we want to trim this prefix to produce portable relative paths + k8sSRC := *k8sPath + "/_output/local/go/src/k8s.io/kubernetes/" + for i := range frames { + trimmedFile := strings.TrimPrefix(frames[i].File, k8sSRC) + trimmedFile = strings.TrimPrefix(trimmedFile, "/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/") + frames[i].File = trimmedFile } for _, curFrame := range frames { From 025e6de3ebc4addc719328c0feb5029d202d0346 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Sat, 27 Feb 2021 23:58:21 -0800 Subject: [PATCH 3/3] don't run generated_files when generating specsummaries.json --- test/conformance/gen-specsummaries.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/conformance/gen-specsummaries.sh b/test/conformance/gen-specsummaries.sh index 16d9a1280e2..f1376f20a11 100755 --- a/test/conformance/gen-specsummaries.sh +++ b/test/conformance/gen-specsummaries.sh @@ -23,7 +23,10 @@ KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd -P) cd "${KUBE_ROOT}" # build ginkgo and e2e.test -make WHAT='vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test' +# NOTE: we do *not* use `make WHAT=...` because we do *not* want to be running +# make generated_files when diffing things (see: hack/verify-conformance-yaml.sh) +# other update/verify already handle the generated files +hack/make-rules/build.sh vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test # dump spec ./_output/bin/ginkgo --dryRun=true --focus='[Conformance]' ./_output/bin/e2e.test -- --spec-dump "${KUBE_ROOT}/_output/specsummaries.json" > /dev/null