mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-19 01:40:13 +00:00
Merge pull request #88486 from Jefftree/schnake-pr
Adds custom reporter and logic to generate conformance docs and list of tests
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
9821d0e47e
@ -2,9 +2,16 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["walk.go"],
|
||||
srcs = [
|
||||
"doc.go",
|
||||
"walk.go",
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/test/conformance",
|
||||
visibility = ["//visibility:private"],
|
||||
deps = [
|
||||
"//vendor/github.com/onsi/ginkgo/types:go_default_library",
|
||||
"//vendor/gopkg.in/yaml.v2:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_binary(
|
||||
@ -32,22 +39,35 @@ filegroup(
|
||||
)
|
||||
|
||||
genrule(
|
||||
name = "list_conformance_tests",
|
||||
name = "list_conformance_specs",
|
||||
srcs = [
|
||||
"//test/e2e:all-srcs",
|
||||
"//test/e2e_node:all-srcs",
|
||||
"//test/e2e:e2e.test_binary",
|
||||
"//vendor/github.com/onsi/ginkgo/ginkgo",
|
||||
],
|
||||
outs = ["conformance.txt"],
|
||||
cmd = "./$(location :conformance) $(locations //test/e2e:all-srcs) > $@",
|
||||
outs = ["specsummaries.json"],
|
||||
cmd = "$(location //vendor/github.com/onsi/ginkgo/ginkgo) --dryRun=true --focus=[Conformance] $(location //test/e2e:e2e.test_binary) -- --spec-dump $$(pwd)/$@ > /dev/null",
|
||||
message = "Getting all conformance spec summaries.",
|
||||
)
|
||||
|
||||
genrule(
|
||||
name = "list_conformance_tests",
|
||||
srcs = [
|
||||
":list_conformance_specs",
|
||||
":conformance",
|
||||
"//test/e2e:all-srcs",
|
||||
],
|
||||
outs = ["conformance.yaml"],
|
||||
cmd = "$(location :conformance) $(location :list_conformance_specs) > $@",
|
||||
message = "Listing all conformance tests.",
|
||||
tools = [":conformance"],
|
||||
)
|
||||
|
||||
sh_test(
|
||||
name = "conformance_test",
|
||||
srcs = ["conformance_test.sh"],
|
||||
data = [
|
||||
"testdata/conformance.txt",
|
||||
"testdata/conformance.yaml",
|
||||
":list_conformance_tests",
|
||||
],
|
||||
)
|
||||
@ -58,3 +78,16 @@ go_test(
|
||||
data = glob(["testdata/**"]),
|
||||
embed = [":go_default_library"],
|
||||
)
|
||||
|
||||
genrule(
|
||||
name = "gen_conformance_docs",
|
||||
srcs = [
|
||||
":list_conformance_specs",
|
||||
":conformance",
|
||||
"//test/e2e:all-srcs",
|
||||
":package-srcs",
|
||||
],
|
||||
outs = ["conformance.md"],
|
||||
cmd = "$(location :conformance) --docs $(location :list_conformance_specs) > $@",
|
||||
message = "Listing all conformance tests.",
|
||||
)
|
||||
|
||||
@ -9,7 +9,7 @@ To update the list, run
|
||||
|
||||
```console
|
||||
bazel build //test/conformance:list_conformance_tests
|
||||
cp bazel-genfiles/test/conformance/conformance.txt test/conformance/testdata
|
||||
cp bazel-genfiles/test/conformance/conformance.yaml test/conformance/testdata
|
||||
```
|
||||
|
||||
Add the changed file to your PR, then send for review.
|
||||
|
||||
@ -15,13 +15,11 @@ clean up the tests.
|
||||
Example:
|
||||
```
|
||||
/*
|
||||
Testname: Kubelet-OutputToLogs
|
||||
Release: v1.9
|
||||
Description: By default the stdout and stderr from the process
|
||||
being executed in a pod MUST be sent to the pod's logs.
|
||||
Release : v1.13
|
||||
Testname: Kubelet, log output, default
|
||||
Description: By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.
|
||||
*/
|
||||
// Note this test needs to be fixed to also test for stderr
|
||||
It("it should print the output to logs [Conformance]", func() {
|
||||
framework.ConformanceIt("should print the output to logs [NodeConformance]", func() {
|
||||
```
|
||||
|
||||
would generate the following documentation for the test. Note that the "TestName" from the Documentation above will
|
||||
@ -29,12 +27,12 @@ be used to document the test which make it more human readable. The "Description
|
||||
documentation for that test.
|
||||
|
||||
### **Output:**
|
||||
## [Kubelet-OutputToLogs](https://github.com/kubernetes/kubernetes/blob/release-1.9/test/e2e_node/kubelet_test.go#L42)
|
||||
## [Kubelet, log output, default](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/common/kubelet.go#L48)
|
||||
|
||||
### Release v1.9
|
||||
By default the stdout and stderr from the process
|
||||
being executed in a pod MUST be sent to the pod's logs.
|
||||
Note this test needs to be fixed to also test for stderr
|
||||
- Added to conformance in release v1.13
|
||||
- Defined in code as: [k8s.io] Kubelet when scheduling a busybox command in a pod should print the output to logs [NodeConformance] [Conformance]
|
||||
|
||||
By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.
|
||||
|
||||
Notational Conventions when documenting the tests with the key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119).
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@
|
||||
|
||||
set -o errexit
|
||||
|
||||
if diff -u test/conformance/testdata/conformance.txt test/conformance/conformance.txt; then
|
||||
if diff -u test/conformance/testdata/conformance.yaml test/conformance/conformance.yaml; then
|
||||
echo PASS
|
||||
exit 0
|
||||
fi
|
||||
|
||||
30
test/conformance/doc.go
Normal file
30
test/conformance/doc.go
Normal file
@ -0,0 +1,30 @@
|
||||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
/*
|
||||
This stand-alone package is utilized for dynamically generating/maintaining a list of
|
||||
conformance tests. It utilizes a two step approach:
|
||||
- The test binary is built
|
||||
- The test binary is run in dry mode with a custom ginkgo reporter dumping out
|
||||
types.SpecSummary objects which contain full test names and file/code information.
|
||||
- The SpecSummary information is parsed to get file/line info on Conformance tests and
|
||||
then we use a simplified AST parser to grab the comments above the test.
|
||||
|
||||
Due to the complicated nature of how tests can be declared/wrapped in various contexts,
|
||||
this approach is much simpler to maintain than a pure-AST parser and allows us to easily
|
||||
capture the full test names/locations of the tests using the pre-existing ginkgo logic.
|
||||
*/
|
||||
package main
|
||||
277
test/conformance/testdata/conformance.txt
vendored
277
test/conformance/testdata/conformance.txt
vendored
@ -1,277 +0,0 @@
|
||||
test/e2e/apimachinery/aggregator.go: "Should be able to support the 1.17 Sample API Server using the current Aggregator"
|
||||
test/e2e/apimachinery/crd_conversion_webhook.go: "should be able to convert from CR v1 to CR v2"
|
||||
test/e2e/apimachinery/crd_conversion_webhook.go: "should be able to convert a non homogeneous list of CRs"
|
||||
test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD with validation schema"
|
||||
test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD without validation schema"
|
||||
test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD preserving unknown fields at the schema root"
|
||||
test/e2e/apimachinery/crd_publish_openapi.go: "works for CRD preserving unknown fields in an embedded object"
|
||||
test/e2e/apimachinery/crd_publish_openapi.go: "works for multiple CRDs of different groups"
|
||||
test/e2e/apimachinery/crd_publish_openapi.go: "works for multiple CRDs of same group but different versions"
|
||||
test/e2e/apimachinery/crd_publish_openapi.go: "works for multiple CRDs of same group and version but different kinds"
|
||||
test/e2e/apimachinery/crd_publish_openapi.go: "updates the published spec when one version gets renamed"
|
||||
test/e2e/apimachinery/crd_publish_openapi.go: "removes definition from spec when one version gets changed to not be served"
|
||||
test/e2e/apimachinery/crd_watch.go: "watch on custom resource definition objects"
|
||||
test/e2e/apimachinery/custom_resource_definition.go: "creating/deleting custom resource definition objects works"
|
||||
test/e2e/apimachinery/custom_resource_definition.go: "listing custom resource definition objects works"
|
||||
test/e2e/apimachinery/custom_resource_definition.go: "getting/updating/patching custom resource definition status sub-resource works"
|
||||
test/e2e/apimachinery/custom_resource_definition.go: "should include custom resource definition resources in discovery documents"
|
||||
test/e2e/apimachinery/custom_resource_definition.go: "custom resource defaulting for requests and from storage works"
|
||||
test/e2e/apimachinery/garbage_collector.go: "should delete pods created by rc when not orphaning"
|
||||
test/e2e/apimachinery/garbage_collector.go: "should orphan pods created by rc if delete options say so"
|
||||
test/e2e/apimachinery/garbage_collector.go: "should delete RS created by deployment when not orphaning"
|
||||
test/e2e/apimachinery/garbage_collector.go: "should orphan RS created by deployment when deleteOptions.PropagationPolicy is Orphan"
|
||||
test/e2e/apimachinery/garbage_collector.go: "should keep the rc around until all its pods are deleted if the deleteOptions says so"
|
||||
test/e2e/apimachinery/garbage_collector.go: "should not delete dependents that have both valid owner and owner that's waiting for dependents to be deleted"
|
||||
test/e2e/apimachinery/garbage_collector.go: "should not be blocked by dependency circle"
|
||||
test/e2e/apimachinery/namespace.go: "should ensure that all pods are removed when a namespace is deleted"
|
||||
test/e2e/apimachinery/namespace.go: "should ensure that all services are removed when a namespace is deleted"
|
||||
test/e2e/apimachinery/namespace.go: "should patch a Namespace"
|
||||
test/e2e/apimachinery/resource_quota.go: "should create a ResourceQuota and ensure its status is promptly calculated."
|
||||
test/e2e/apimachinery/resource_quota.go: "should create a ResourceQuota and capture the life of a service."
|
||||
test/e2e/apimachinery/resource_quota.go: "should create a ResourceQuota and capture the life of a secret."
|
||||
test/e2e/apimachinery/resource_quota.go: "should create a ResourceQuota and capture the life of a pod."
|
||||
test/e2e/apimachinery/resource_quota.go: "should create a ResourceQuota and capture the life of a configMap."
|
||||
test/e2e/apimachinery/resource_quota.go: "should create a ResourceQuota and capture the life of a replication controller."
|
||||
test/e2e/apimachinery/resource_quota.go: "should create a ResourceQuota and capture the life of a replica set."
|
||||
test/e2e/apimachinery/resource_quota.go: "should verify ResourceQuota with terminating scopes."
|
||||
test/e2e/apimachinery/resource_quota.go: "should verify ResourceQuota with best effort scope."
|
||||
test/e2e/apimachinery/resource_quota.go: "should be able to update and delete ResourceQuota."
|
||||
test/e2e/apimachinery/table_conversion.go: "should return a 406 for a backend which does not implement metadata"
|
||||
test/e2e/apimachinery/watch.go: "should observe add, update, and delete watch notifications on configmaps"
|
||||
test/e2e/apimachinery/watch.go: "should be able to start watching from a specific resource version"
|
||||
test/e2e/apimachinery/watch.go: "should be able to restart watching from the last resource version observed by the previous watch"
|
||||
test/e2e/apimachinery/watch.go: "should observe an object deletion if it stops meeting the requirements of the selector"
|
||||
test/e2e/apimachinery/watch.go: "should receive events on concurrent watches in same order"
|
||||
test/e2e/apimachinery/webhook.go: "should include webhook resources in discovery documents"
|
||||
test/e2e/apimachinery/webhook.go: "should be able to deny pod and configmap creation"
|
||||
test/e2e/apimachinery/webhook.go: "should be able to deny attaching pod"
|
||||
test/e2e/apimachinery/webhook.go: "should be able to deny custom resource creation, update and deletion"
|
||||
test/e2e/apimachinery/webhook.go: "should unconditionally reject operations on fail closed webhook"
|
||||
test/e2e/apimachinery/webhook.go: "should mutate configmap"
|
||||
test/e2e/apimachinery/webhook.go: "should mutate pod and apply defaults after mutation"
|
||||
test/e2e/apimachinery/webhook.go: "should not be able to mutate or prevent deletion of webhook configuration objects"
|
||||
test/e2e/apimachinery/webhook.go: "should mutate custom resource"
|
||||
test/e2e/apimachinery/webhook.go: "should deny crd creation"
|
||||
test/e2e/apimachinery/webhook.go: "should mutate custom resource with different stored version"
|
||||
test/e2e/apimachinery/webhook.go: "should mutate custom resource with pruning"
|
||||
test/e2e/apimachinery/webhook.go: "should honor timeout"
|
||||
test/e2e/apimachinery/webhook.go: "patching/updating a validating webhook should work"
|
||||
test/e2e/apimachinery/webhook.go: "patching/updating a mutating webhook should work"
|
||||
test/e2e/apimachinery/webhook.go: "listing validating webhooks should work"
|
||||
test/e2e/apimachinery/webhook.go: "listing mutating webhooks should work"
|
||||
test/e2e/apps/daemon_set.go: "should run and stop simple daemon"
|
||||
test/e2e/apps/daemon_set.go: "should run and stop complex daemon"
|
||||
test/e2e/apps/daemon_set.go: "should retry creating failed daemon pods"
|
||||
test/e2e/apps/daemon_set.go: "should update pod when spec was updated and update strategy is RollingUpdate"
|
||||
test/e2e/apps/daemon_set.go: "should rollback without unnecessary restarts"
|
||||
test/e2e/apps/deployment.go: "RollingUpdateDeployment should delete old pods and create new ones"
|
||||
test/e2e/apps/deployment.go: "RecreateDeployment should delete old pods and create new ones"
|
||||
test/e2e/apps/deployment.go: "deployment should delete old replica sets"
|
||||
test/e2e/apps/deployment.go: "deployment should support rollover"
|
||||
test/e2e/apps/deployment.go: "deployment should support proportional scaling"
|
||||
test/e2e/apps/job.go: "should run a job to completion when tasks sometimes fail and are locally restarted"
|
||||
test/e2e/apps/job.go: "should delete a job"
|
||||
test/e2e/apps/job.go: "should adopt matching orphans and release non-matching pods"
|
||||
test/e2e/apps/rc.go: "should serve a basic image on each replica with a public image"
|
||||
test/e2e/apps/rc.go: "should surface a failure condition on a common issue like exceeded quota"
|
||||
test/e2e/apps/rc.go: "should adopt matching pods on creation"
|
||||
test/e2e/apps/rc.go: "should release no longer matching pods"
|
||||
test/e2e/apps/replica_set.go: "should serve a basic image on each replica with a public image"
|
||||
test/e2e/apps/replica_set.go: "should adopt matching pods on creation and release no longer matching pods"
|
||||
test/e2e/apps/statefulset.go: "should perform rolling updates and roll backs of template modifications"
|
||||
test/e2e/apps/statefulset.go: "should perform canary updates and phased rolling updates of template modifications"
|
||||
test/e2e/apps/statefulset.go: "Scaling should happen in predictable order and halt if any stateful pod is unhealthy"
|
||||
test/e2e/apps/statefulset.go: "Burst scaling should run to completion even with unhealthy pods"
|
||||
test/e2e/apps/statefulset.go: "Should recreate evicted statefulset"
|
||||
test/e2e/apps/statefulset.go: "should have a working scale subresource"
|
||||
test/e2e/auth/service_accounts.go: "should mount an API token into pods"
|
||||
test/e2e/auth/service_accounts.go: "should allow opting out of API token automount"
|
||||
test/e2e/common/configmap.go: "should be consumable via environment variable"
|
||||
test/e2e/common/configmap.go: "should be consumable via the environment"
|
||||
test/e2e/common/configmap.go: "should fail to create ConfigMap with empty key"
|
||||
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume"
|
||||
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with defaultMode set"
|
||||
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume as non-root"
|
||||
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with mappings"
|
||||
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with mappings and Item mode set"
|
||||
test/e2e/common/configmap_volume.go: "should be consumable from pods in volume with mappings as non-root"
|
||||
test/e2e/common/configmap_volume.go: "updates should be reflected in volume"
|
||||
test/e2e/common/configmap_volume.go: "binary data should be reflected in volume"
|
||||
test/e2e/common/configmap_volume.go: "optional updates should be reflected in volume"
|
||||
test/e2e/common/configmap_volume.go: "should be consumable in multiple volumes in the same pod"
|
||||
test/e2e/common/container_probe.go: "with readiness probe should not be ready before initial delay and never restart"
|
||||
test/e2e/common/container_probe.go: "with readiness probe that fails should never be ready and never restart"
|
||||
test/e2e/common/container_probe.go: "should be restarted with a exec \\\"cat /tmp/health\\\" liveness probe"
|
||||
test/e2e/common/container_probe.go: "should *not* be restarted with a exec \\\"cat /tmp/health\\\" liveness probe"
|
||||
test/e2e/common/container_probe.go: "should be restarted with a /healthz http liveness probe"
|
||||
test/e2e/common/container_probe.go: "should *not* be restarted with a tcp:8080 liveness probe"
|
||||
test/e2e/common/container_probe.go: "should have monotonically increasing restart count"
|
||||
test/e2e/common/container_probe.go: "should *not* be restarted with a /healthz http liveness probe"
|
||||
test/e2e/common/docker_containers.go: "should use the image defaults if command and args are blank"
|
||||
test/e2e/common/docker_containers.go: "should be able to override the image's default arguments (docker cmd)"
|
||||
test/e2e/common/docker_containers.go: "should be able to override the image's default command (docker entrypoint)"
|
||||
test/e2e/common/docker_containers.go: "should be able to override the image's default command and arguments"
|
||||
test/e2e/common/downward_api.go: "should provide pod name, namespace and IP address as env vars"
|
||||
test/e2e/common/downward_api.go: "should provide host IP as an env var"
|
||||
test/e2e/common/downward_api.go: "should provide container's limits.cpu/memory and requests.cpu/memory as env vars"
|
||||
test/e2e/common/downward_api.go: "should provide default limits.cpu/memory from node allocatable"
|
||||
test/e2e/common/downward_api.go: "should provide pod UID as env vars"
|
||||
test/e2e/common/downwardapi_volume.go: "should provide podname only"
|
||||
test/e2e/common/downwardapi_volume.go: "should set DefaultMode on files"
|
||||
test/e2e/common/downwardapi_volume.go: "should set mode on item file"
|
||||
test/e2e/common/downwardapi_volume.go: "should update labels on modification"
|
||||
test/e2e/common/downwardapi_volume.go: "should update annotations on modification"
|
||||
test/e2e/common/downwardapi_volume.go: "should provide container's cpu limit"
|
||||
test/e2e/common/downwardapi_volume.go: "should provide container's memory limit"
|
||||
test/e2e/common/downwardapi_volume.go: "should provide container's cpu request"
|
||||
test/e2e/common/downwardapi_volume.go: "should provide container's memory request"
|
||||
test/e2e/common/downwardapi_volume.go: "should provide node allocatable (cpu) as default cpu limit if the limit is not set"
|
||||
test/e2e/common/downwardapi_volume.go: "should provide node allocatable (memory) as default memory limit if the limit is not set"
|
||||
test/e2e/common/empty_dir.go: "volume on tmpfs should have the correct mode"
|
||||
test/e2e/common/empty_dir.go: "should support (root,0644,tmpfs)"
|
||||
test/e2e/common/empty_dir.go: "should support (root,0666,tmpfs)"
|
||||
test/e2e/common/empty_dir.go: "should support (root,0777,tmpfs)"
|
||||
test/e2e/common/empty_dir.go: "should support (non-root,0644,tmpfs)"
|
||||
test/e2e/common/empty_dir.go: "should support (non-root,0666,tmpfs)"
|
||||
test/e2e/common/empty_dir.go: "should support (non-root,0777,tmpfs)"
|
||||
test/e2e/common/empty_dir.go: "volume on default medium should have the correct mode"
|
||||
test/e2e/common/empty_dir.go: "should support (root,0644,default)"
|
||||
test/e2e/common/empty_dir.go: "should support (root,0666,default)"
|
||||
test/e2e/common/empty_dir.go: "should support (root,0777,default)"
|
||||
test/e2e/common/empty_dir.go: "should support (non-root,0644,default)"
|
||||
test/e2e/common/empty_dir.go: "should support (non-root,0666,default)"
|
||||
test/e2e/common/empty_dir.go: "should support (non-root,0777,default)"
|
||||
test/e2e/common/empty_dir.go: "pod should support shared volumes between containers"
|
||||
test/e2e/common/expansion.go: "should allow composing env vars into new env vars"
|
||||
test/e2e/common/expansion.go: "should allow substituting values in a container's command"
|
||||
test/e2e/common/expansion.go: "should allow substituting values in a container's args"
|
||||
test/e2e/common/host_path.go: "should give a volume the correct mode"
|
||||
test/e2e/common/init_container.go: "should invoke init containers on a RestartNever pod"
|
||||
test/e2e/common/init_container.go: "should invoke init containers on a RestartAlways pod"
|
||||
test/e2e/common/init_container.go: "should not start app containers if init containers fail on a RestartAlways pod"
|
||||
test/e2e/common/init_container.go: "should not start app containers and fail the pod if init containers fail on a RestartNever pod"
|
||||
test/e2e/common/kubelet.go: "should print the output to logs"
|
||||
test/e2e/common/kubelet.go: "should have an terminated reason"
|
||||
test/e2e/common/kubelet.go: "should be possible to delete"
|
||||
test/e2e/common/kubelet.go: "should write entries to /etc/hosts"
|
||||
test/e2e/common/kubelet.go: "should not write to root filesystem"
|
||||
test/e2e/common/kubelet_etc_hosts.go: "should test kubelet managed /etc/hosts file"
|
||||
test/e2e/common/lease.go: "lease API should be available"
|
||||
test/e2e/common/lifecycle_hook.go: "should execute poststart exec hook properly"
|
||||
test/e2e/common/lifecycle_hook.go: "should execute prestop exec hook properly"
|
||||
test/e2e/common/lifecycle_hook.go: "should execute poststart http hook properly"
|
||||
test/e2e/common/lifecycle_hook.go: "should execute prestop http hook properly"
|
||||
test/e2e/common/networking.go: "should function for intra-pod communication: http"
|
||||
test/e2e/common/networking.go: "should function for intra-pod communication: udp"
|
||||
test/e2e/common/networking.go: "should function for node-pod communication: http"
|
||||
test/e2e/common/networking.go: "should function for node-pod communication: udp"
|
||||
test/e2e/common/pods.go: "should get a host IP"
|
||||
test/e2e/common/pods.go: "should be submitted and removed"
|
||||
test/e2e/common/pods.go: "should be updated"
|
||||
test/e2e/common/pods.go: "should allow activeDeadlineSeconds to be updated"
|
||||
test/e2e/common/pods.go: "should contain environment variables for services"
|
||||
test/e2e/common/pods.go: "should support remote command execution over websockets"
|
||||
test/e2e/common/pods.go: "should support retrieving logs from the container over websockets"
|
||||
test/e2e/common/projected_combined.go: "should project all components that make up the projection API"
|
||||
test/e2e/common/projected_configmap.go: "should be consumable from pods in volume"
|
||||
test/e2e/common/projected_configmap.go: "should be consumable from pods in volume with defaultMode set"
|
||||
test/e2e/common/projected_configmap.go: "should be consumable from pods in volume as non-root"
|
||||
test/e2e/common/projected_configmap.go: "should be consumable from pods in volume with mappings"
|
||||
test/e2e/common/projected_configmap.go: "should be consumable from pods in volume with mappings and Item mode set"
|
||||
test/e2e/common/projected_configmap.go: "should be consumable from pods in volume with mappings as non-root"
|
||||
test/e2e/common/projected_configmap.go: "updates should be reflected in volume"
|
||||
test/e2e/common/projected_configmap.go: "optional updates should be reflected in volume"
|
||||
test/e2e/common/projected_configmap.go: "should be consumable in multiple volumes in the same pod"
|
||||
test/e2e/common/projected_downwardapi.go: "should provide podname only"
|
||||
test/e2e/common/projected_downwardapi.go: "should set DefaultMode on files"
|
||||
test/e2e/common/projected_downwardapi.go: "should set mode on item file"
|
||||
test/e2e/common/projected_downwardapi.go: "should update labels on modification"
|
||||
test/e2e/common/projected_downwardapi.go: "should update annotations on modification"
|
||||
test/e2e/common/projected_downwardapi.go: "should provide container's cpu limit"
|
||||
test/e2e/common/projected_downwardapi.go: "should provide container's memory limit"
|
||||
test/e2e/common/projected_downwardapi.go: "should provide container's cpu request"
|
||||
test/e2e/common/projected_downwardapi.go: "should provide container's memory request"
|
||||
test/e2e/common/projected_downwardapi.go: "should provide node allocatable (cpu) as default cpu limit if the limit is not set"
|
||||
test/e2e/common/projected_downwardapi.go: "should provide node allocatable (memory) as default memory limit if the limit is not set"
|
||||
test/e2e/common/projected_secret.go: "should be consumable from pods in volume"
|
||||
test/e2e/common/projected_secret.go: "should be consumable from pods in volume with defaultMode set"
|
||||
test/e2e/common/projected_secret.go: "should be consumable from pods in volume as non-root with defaultMode and fsGroup set"
|
||||
test/e2e/common/projected_secret.go: "should be consumable from pods in volume with mappings"
|
||||
test/e2e/common/projected_secret.go: "should be consumable from pods in volume with mappings and Item Mode set"
|
||||
test/e2e/common/projected_secret.go: "should be consumable in multiple volumes in a pod"
|
||||
test/e2e/common/projected_secret.go: "optional updates should be reflected in volume"
|
||||
test/e2e/common/runtime.go: "should run with the expected status"
|
||||
test/e2e/common/runtime.go: "should report termination message if TerminationMessagePath is set as non-root user and at a non-default path"
|
||||
test/e2e/common/runtime.go: "should report termination message from log output if TerminationMessagePolicy FallbackToLogsOnError is set"
|
||||
test/e2e/common/runtime.go: "should report termination message as empty when pod succeeds and TerminationMessagePolicy FallbackToLogsOnError is set"
|
||||
test/e2e/common/runtime.go: "should report termination message from file when pod succeeds and TerminationMessagePolicy FallbackToLogsOnError is set"
|
||||
test/e2e/common/secrets.go: "should be consumable from pods in env vars"
|
||||
test/e2e/common/secrets.go: "should be consumable via the environment"
|
||||
test/e2e/common/secrets.go: "should fail to create secret due to empty secret key"
|
||||
test/e2e/common/secrets.go: "should patch a secret"
|
||||
test/e2e/common/secrets_volume.go: "should be consumable from pods in volume"
|
||||
test/e2e/common/secrets_volume.go: "should be consumable from pods in volume with defaultMode set"
|
||||
test/e2e/common/secrets_volume.go: "should be consumable from pods in volume as non-root with defaultMode and fsGroup set"
|
||||
test/e2e/common/secrets_volume.go: "should be consumable from pods in volume with mappings"
|
||||
test/e2e/common/secrets_volume.go: "should be consumable from pods in volume with mappings and Item Mode set"
|
||||
test/e2e/common/secrets_volume.go: "should be able to mount in a volume regardless of a different secret existing with same name in different namespace"
|
||||
test/e2e/common/secrets_volume.go: "should be consumable in multiple volumes in a pod"
|
||||
test/e2e/common/secrets_volume.go: "optional updates should be reflected in volume"
|
||||
test/e2e/common/security_context.go: "should run the container with uid 65534"
|
||||
test/e2e/common/security_context.go: "should run the container with writable rootfs when readOnlyRootFilesystem=false"
|
||||
test/e2e/common/security_context.go: "should run the container as unprivileged when false"
|
||||
test/e2e/common/security_context.go: "should not allow privilege escalation when false"
|
||||
test/e2e/kubectl/kubectl.go: "should create and stop a replication controller"
|
||||
test/e2e/kubectl/kubectl.go: "should scale a replication controller"
|
||||
test/e2e/kubectl/kubectl.go: "should create and stop a working application"
|
||||
test/e2e/kubectl/kubectl.go: "should check if v1 is in available api versions"
|
||||
test/e2e/kubectl/kubectl.go: "should check if Kubernetes master services is included in cluster-info"
|
||||
test/e2e/kubectl/kubectl.go: "should check if kubectl describe prints relevant information for rc and pods"
|
||||
test/e2e/kubectl/kubectl.go: "should create services for rc"
|
||||
test/e2e/kubectl/kubectl.go: "should update the label on a resource"
|
||||
test/e2e/kubectl/kubectl.go: "should be able to retrieve and filter logs"
|
||||
test/e2e/kubectl/kubectl.go: "should add annotations for pods in rc"
|
||||
test/e2e/kubectl/kubectl.go: "should check is all data is printed"
|
||||
test/e2e/kubectl/kubectl.go: "should create a pod from an image when restart is Never"
|
||||
test/e2e/kubectl/kubectl.go: "should update a single-container pod's image"
|
||||
test/e2e/kubectl/kubectl.go: "should support proxy with --port 0"
|
||||
test/e2e/kubectl/kubectl.go: "should support --unix-socket=/path"
|
||||
test/e2e/network/dns.go: "should provide DNS for the cluster"
|
||||
test/e2e/network/dns.go: "should provide /etc/hosts entries for the cluster"
|
||||
test/e2e/network/dns.go: "should provide DNS for services"
|
||||
test/e2e/network/dns.go: "should resolve DNS of partial qualified names for services"
|
||||
test/e2e/network/dns.go: "should provide DNS for pods for Hostname"
|
||||
test/e2e/network/dns.go: "should provide DNS for pods for Subdomain"
|
||||
test/e2e/network/dns.go: "should provide DNS for ExternalName services"
|
||||
test/e2e/network/dns.go: "should support configurable pod DNS nameservers"
|
||||
test/e2e/network/proxy.go: "should proxy logs on node with explicit kubelet port using proxy subresource"
|
||||
test/e2e/network/proxy.go: "should proxy logs on node using proxy subresource"
|
||||
test/e2e/network/proxy.go: "should proxy through a service and a pod"
|
||||
test/e2e/network/service.go: "should provide secure master service"
|
||||
test/e2e/network/service.go: "should serve a basic endpoint from pods"
|
||||
test/e2e/network/service.go: "should serve multiport endpoints from pods"
|
||||
test/e2e/network/service.go: "should be able to create a functioning NodePort service"
|
||||
test/e2e/network/service.go: "should be able to change the type from ExternalName to ClusterIP"
|
||||
test/e2e/network/service.go: "should be able to change the type from ExternalName to NodePort"
|
||||
test/e2e/network/service.go: "should be able to change the type from ClusterIP to ExternalName"
|
||||
test/e2e/network/service.go: "should be able to change the type from NodePort to ExternalName"
|
||||
test/e2e/network/service.go: "should find a service from listing all namespaces"
|
||||
test/e2e/network/service_latency.go: "should not be very high"
|
||||
test/e2e/node/events.go: "should be sent by kubelets and the scheduler about pods scheduling and running"
|
||||
test/e2e/node/pods.go: "should be set on Pods with matching resource requests and limits for memory and cpu"
|
||||
test/e2e/node/pre_stop.go: "should call prestop when killing a pod"
|
||||
test/e2e/scheduling/limit_range.go: "should create a LimitRange with defaults and ensure pod has those defaults applied."
|
||||
test/e2e/scheduling/predicates.go: "validates resource limits of pods that are allowed to run"
|
||||
test/e2e/scheduling/predicates.go: "validates that NodeSelector is respected if not matching"
|
||||
test/e2e/scheduling/predicates.go: "validates that NodeSelector is respected if matching"
|
||||
test/e2e/scheduling/predicates.go: "validates that there is no conflict between pods with same hostPort but different hostIP and protocol"
|
||||
test/e2e/scheduling/predicates.go: "validates that there exists conflict between pods with same hostPort and protocol but one using 0.0.0.0 hostIP"
|
||||
test/e2e/scheduling/taints.go: "removing taint cancels eviction"
|
||||
test/e2e/scheduling/taints.go: "evicts pods with minTolerationSeconds"
|
||||
test/e2e/storage/empty_dir_wrapper.go: "should not conflict"
|
||||
test/e2e/storage/empty_dir_wrapper.go: "should not cause race condition when used for configmaps"
|
||||
test/e2e/storage/subpath.go: "should support subpaths with secret pod"
|
||||
test/e2e/storage/subpath.go: "should support subpaths with configmap pod"
|
||||
test/e2e/storage/subpath.go: "should support subpaths with configmap pod with mountPath of existing file"
|
||||
test/e2e/storage/subpath.go: "should support subpaths with downward pod"
|
||||
test/e2e/storage/subpath.go: "should support subpaths with projected pod"
|
||||
2512
test/conformance/testdata/conformance.yaml
vendored
Executable file
2512
test/conformance/testdata/conformance.yaml
vendored
Executable file
File diff suppressed because it is too large
Load Diff
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
@ -14,84 +14,289 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package main provides a tool that scans kubernetes e2e test source code
|
||||
// looking for conformance test declarations, which it emits on stdout. It
|
||||
// also looks for legacy, manually added "[Conformance]" tags and reports an
|
||||
// error if it finds any.
|
||||
//
|
||||
// This approach is not air tight, but it will serve our purpose as a
|
||||
// pre-submit check.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"gopkg.in/yaml.v2"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"github.com/onsi/ginkgo/types"
|
||||
)
|
||||
|
||||
var (
|
||||
baseURL = flag.String("url", "https://github.com/kubernetes/kubernetes/tree/master/", "location of the current source")
|
||||
confDoc = flag.Bool("conformance", false, "write a conformance document")
|
||||
version = flag.String("version", "v1.9", "version of this conformance document")
|
||||
totalConfTests, totalLegacyTests, missingComments int
|
||||
baseURL = flag.String("url", "https://github.com/kubernetes/kubernetes/tree/master/", "location of the current source")
|
||||
k8sPath = flag.String("source", "", "location of the current source on the current machine")
|
||||
confDoc = flag.Bool("docs", false, "write a conformance document")
|
||||
version = flag.String("version", "v1.9", "version of this conformance document")
|
||||
|
||||
// If a test name contains any of these tags, it is ineligble for promotion to conformance
|
||||
regexIneligibleTags = regexp.MustCompile(`\[(Alpha|Feature:[^\]]+|Flaky)\]`)
|
||||
|
||||
// Conformance comments should be within this number of lines to the call itself.
|
||||
// Allowing for more than one in case a spare comment or two is below it.
|
||||
conformanceCommentsLineWindow = 5
|
||||
|
||||
seenLines map[string]struct{}
|
||||
)
|
||||
|
||||
const regexDescribe = "Describe|KubeDescribe|SIGDescribe"
|
||||
const regexContext = "^Context$"
|
||||
type frame struct {
|
||||
Function string
|
||||
|
||||
type visitor struct {
|
||||
FileSet *token.FileSet
|
||||
describes []describe
|
||||
cMap ast.CommentMap
|
||||
//list of all the conformance tests in the path
|
||||
tests []conformanceData
|
||||
}
|
||||
|
||||
//describe contains text associated with ginkgo describe container
|
||||
type describe struct {
|
||||
rparen token.Pos
|
||||
text string
|
||||
lastContext context
|
||||
}
|
||||
|
||||
//context contain the text associated with the Context clause
|
||||
type context struct {
|
||||
text string
|
||||
// File and Line are the file name and line number of the
|
||||
// location in this frame. For non-leaf frames, this will be
|
||||
// the location of a call. These may be the empty string and
|
||||
// zero, respectively, if not known.
|
||||
File string
|
||||
Line int
|
||||
}
|
||||
|
||||
type conformanceData struct {
|
||||
// A URL to the line of code in the kube src repo for the test
|
||||
URL string
|
||||
// A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number.
|
||||
URL string `yaml:"-"`
|
||||
// Extracted from the "Testname:" comment before the test
|
||||
TestName string
|
||||
// CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]`
|
||||
CodeName string
|
||||
// Extracted from the "Description:" comment before the test
|
||||
Description string
|
||||
// Version when this test is added or modified ex: v1.12, v1.13
|
||||
Release string
|
||||
// File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes.
|
||||
File string
|
||||
}
|
||||
|
||||
func (v *visitor) convertToConformanceData(at *ast.BasicLit) {
|
||||
cd := conformanceData{}
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
comment := v.comment(at)
|
||||
pos := v.FileSet.Position(at.Pos())
|
||||
cd.URL = fmt.Sprintf("%s%s#L%d", *baseURL, pos.Filename, pos.Line)
|
||||
if len(flag.Args()) < 1 {
|
||||
log.Fatalln("Requires the name of the test details file as first and only argument.")
|
||||
}
|
||||
testDetailsFile := flag.Args()[0]
|
||||
f, err := os.Open(testDetailsFile)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to open file %v: %v", testDetailsFile, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
seenLines = map[string]struct{}{}
|
||||
dec := json.NewDecoder(f)
|
||||
testInfos := []*conformanceData{}
|
||||
for {
|
||||
var spec *types.SpecSummary
|
||||
if err := dec.Decode(&spec); err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if isConformance(spec) {
|
||||
testInfo := getTestInfo(spec)
|
||||
if testInfo != nil {
|
||||
testInfos = append(testInfos, testInfo)
|
||||
}
|
||||
|
||||
if err := validateTestName(testInfo.CodeName); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(testInfos, func(i, j int) bool { return testInfos[i].CodeName < testInfos[j].CodeName })
|
||||
saveAllTestInfo(testInfos)
|
||||
}
|
||||
|
||||
func isConformance(spec *types.SpecSummary) bool {
|
||||
return strings.Contains(getTestName(spec), "[Conformance]")
|
||||
}
|
||||
|
||||
func getTestInfo(spec *types.SpecSummary) *conformanceData {
|
||||
var c *conformanceData
|
||||
var err error
|
||||
// The key to this working is that we don't need to parse every file or walk
|
||||
// every componentCodeLocation. The last componentCodeLocation is going to typically start
|
||||
// with the ConformanceIt(...) call and the next call in that callstack will be the
|
||||
// ast.Node which is attached to the comment that we want.
|
||||
for i := len(spec.ComponentCodeLocations) - 1; i > 0; i-- {
|
||||
fullstacktrace := spec.ComponentCodeLocations[i].FullStackTrace
|
||||
c, err = getConformanceDataFromStackTrace(fullstacktrace)
|
||||
if err != nil {
|
||||
log.Printf("Error looking for conformance data: %v", err)
|
||||
}
|
||||
if c != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if c == nil {
|
||||
log.Printf("Did not find test info for spec: %#v\n", getTestName(spec))
|
||||
return nil
|
||||
}
|
||||
|
||||
c.CodeName = getTestName(spec)
|
||||
return c
|
||||
}
|
||||
|
||||
func getTestName(spec *types.SpecSummary) string {
|
||||
return strings.Join(spec.ComponentTexts[1:], " ")
|
||||
}
|
||||
|
||||
func saveAllTestInfo(dataSet []*conformanceData) {
|
||||
if *confDoc {
|
||||
// Note: this assumes that you're running from the root of the kube src repo
|
||||
templ, err := template.ParseFiles("./test/conformance/cf_header.md")
|
||||
if err != nil {
|
||||
fmt.Printf("Error reading the Header file information: %s\n\n", err)
|
||||
}
|
||||
data := struct {
|
||||
Version string
|
||||
}{
|
||||
Version: *version,
|
||||
}
|
||||
templ.Execute(os.Stdout, data)
|
||||
|
||||
for _, data := range dataSet {
|
||||
fmt.Printf("## [%s](%s)\n\n", data.TestName, data.URL)
|
||||
fmt.Printf("- Added to conformance in release %s\n", data.Release)
|
||||
fmt.Printf("- Defined in code as: %s\n\n", data.CodeName)
|
||||
fmt.Printf("%s\n\n", data.Description)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Serialize the list as a whole. Generally meant to end up as conformance.txt which tracks the set of tests.
|
||||
b, err := yaml.Marshal(dataSet)
|
||||
if err != nil {
|
||||
log.Printf("Error marshalling data into YAML: %v", err)
|
||||
}
|
||||
fmt.Println(string(b))
|
||||
}
|
||||
|
||||
func getConformanceDataFromStackTrace(fullstackstrace string) (*conformanceData, error) {
|
||||
// The full stacktrace to parse from ginkgo is of the form:
|
||||
// k8s.io/kubernetes/test/e2e/storage/utils.SIGDescribe(0x51f4c4f, 0xf, 0x53a0dd8, 0xc000ab6e01)\n\ttest/e2e/storage/utils/framework.go:23 +0x75\n ... ...
|
||||
// So we need to split it into lines, remove whitespace, and then grab the files/lines.
|
||||
stack := strings.Replace(fullstackstrace, "\t", "", -1)
|
||||
calls := strings.Split(stack, "\n")
|
||||
frames := []frame{}
|
||||
i := 0
|
||||
for i < len(calls) {
|
||||
fileLine := strings.Split(calls[i+1], " ")
|
||||
lineinfo := strings.Split(fileLine[0], ":")
|
||||
line, err := strconv.Atoi(lineinfo[1])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
frames = append(frames, frame{
|
||||
Function: calls[i],
|
||||
File: lineinfo[0],
|
||||
Line: line,
|
||||
})
|
||||
i += 2
|
||||
}
|
||||
|
||||
// filenames have `/go/src/k8s.io` prefix which dont exist locally
|
||||
for i, v := range frames {
|
||||
frames[i].File = strings.Replace(v.File,
|
||||
"/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes",
|
||||
*k8sPath, 1)
|
||||
}
|
||||
|
||||
for _, curFrame := range frames {
|
||||
if _, seen := seenLines[fmt.Sprintf("%v:%v", curFrame.File, curFrame.Line)]; seen {
|
||||
continue
|
||||
}
|
||||
|
||||
freader, err := os.Open(curFrame.File)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer freader.Close()
|
||||
cd, err := scanFileForFrame(curFrame.File, freader, curFrame)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cd != nil {
|
||||
return cd, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// scanFileForFrame will scan the target and look for a conformance comment attached to the function
|
||||
// described by the target frame. If the comment can't be found then nil, nil is returned.
|
||||
func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*conformanceData, error) {
|
||||
fset := token.NewFileSet() // positions are relative to fset
|
||||
f, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cmap := ast.NewCommentMap(fset, f, f.Comments)
|
||||
for _, cs := range cmap {
|
||||
for _, c := range cs {
|
||||
if cd := tryCommentGroupAndFrame(fset, c, targetFrame); cd != nil {
|
||||
return cd, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func validateTestName(s string) error {
|
||||
matches := regexIneligibleTags.FindAllString(s, -1)
|
||||
if matches != nil {
|
||||
return fmt.Errorf("'%s' cannot have invalid tags %v", s, strings.Join(matches, ","))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *conformanceData {
|
||||
if !shouldProcessCommentGroup(fset, cg, f) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Each file/line will either be some helper function (not a conformance comment) or apply to just a single test. Don't revisit.
|
||||
if seenLines != nil {
|
||||
seenLines[fmt.Sprintf("%v:%v", f.File, f.Line)] = struct{}{}
|
||||
}
|
||||
cd := commentToConformanceData(cg.Text())
|
||||
if cd == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
cd.URL = fmt.Sprintf("%s%s#L%d", *baseURL, f.File, f.Line)
|
||||
cd.File = f.File
|
||||
return cd
|
||||
}
|
||||
|
||||
func shouldProcessCommentGroup(fset *token.FileSet, cg *ast.CommentGroup, f frame) bool {
|
||||
lineDiff := f.Line - fset.Position(cg.End()).Line
|
||||
return lineDiff > 0 && lineDiff <= conformanceCommentsLineWindow
|
||||
}
|
||||
|
||||
func commentToConformanceData(comment string) *conformanceData {
|
||||
lines := strings.Split(comment, "\n")
|
||||
cd.Description = ""
|
||||
descLines := []string{}
|
||||
cd := &conformanceData{}
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
if sline := regexp.MustCompile("^Testname\\s*:\\s*").Split(line, -1); len(sline) == 2 {
|
||||
cd.TestName = sline[1]
|
||||
continue
|
||||
@ -103,326 +308,12 @@ func (v *visitor) convertToConformanceData(at *ast.BasicLit) {
|
||||
if sline := regexp.MustCompile("^Description\\s*:\\s*").Split(line, -1); len(sline) == 2 {
|
||||
line = sline[1]
|
||||
}
|
||||
cd.Description += line + "\n"
|
||||
descLines = append(descLines, line)
|
||||
}
|
||||
if cd.Release == "" && cd.TestName == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if cd.TestName == "" {
|
||||
testName := v.getDescription(at.Value)
|
||||
i := strings.Index(testName, "[Conformance]")
|
||||
if i > 0 {
|
||||
cd.TestName = strings.TrimSpace(testName[:i])
|
||||
} else {
|
||||
cd.TestName = testName
|
||||
}
|
||||
}
|
||||
|
||||
v.tests = append(v.tests, cd)
|
||||
}
|
||||
|
||||
func newVisitor() *visitor {
|
||||
return &visitor{
|
||||
FileSet: token.NewFileSet(),
|
||||
}
|
||||
}
|
||||
|
||||
func (v *visitor) isConformanceCall(call *ast.CallExpr) bool {
|
||||
switch fun := call.Fun.(type) {
|
||||
case *ast.SelectorExpr:
|
||||
if fun.Sel != nil {
|
||||
return fun.Sel.Name == "ConformanceIt"
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (v *visitor) isLegacyItCall(call *ast.CallExpr) bool {
|
||||
switch fun := call.Fun.(type) {
|
||||
case *ast.Ident:
|
||||
if fun.Name != "It" {
|
||||
return false
|
||||
}
|
||||
if len(call.Args) < 1 {
|
||||
v.failf(call, "Not enough arguments to It()")
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
switch arg := call.Args[0].(type) {
|
||||
case *ast.BasicLit:
|
||||
if arg.Kind != token.STRING {
|
||||
v.failf(arg, "Unexpected non-string argument to It()")
|
||||
}
|
||||
if strings.Contains(arg.Value, "[Conformance]") {
|
||||
return true
|
||||
}
|
||||
default:
|
||||
// non-literal argument to It()... we just ignore these even though they could be a way to "sneak in" a conformance test
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (v *visitor) failf(expr ast.Expr, format string, a ...interface{}) {
|
||||
msg := fmt.Sprintf(format, a...)
|
||||
fmt.Fprintf(os.Stderr, "ERROR at %v: %s\n", v.FileSet.Position(expr.Pos()), msg)
|
||||
}
|
||||
|
||||
func (v *visitor) comment(x *ast.BasicLit) string {
|
||||
for _, comm := range v.cMap.Comments() {
|
||||
testOffset := int(x.Pos()-comm.End()) - len("framework.ConformanceIt(\"")
|
||||
//Cannot assume the offset is within three or four tabs from the test block itself.
|
||||
//It is better to trim the newlines, tabs, etc and then we if the comment is followed
|
||||
//by the test block itself so that we can associate the comment with it properly.
|
||||
if 0 <= testOffset && testOffset <= 10 {
|
||||
b1 := make([]byte, x.Pos()-comm.End())
|
||||
//if we fail to open the file to compare the content we just assume the
|
||||
//proximity of the comment and apply it.
|
||||
myf, err := os.Open(v.FileSet.File(x.Pos()).Name())
|
||||
if err == nil {
|
||||
defer myf.Close()
|
||||
if _, err := myf.Seek(int64(comm.End()), 0); err == nil {
|
||||
if _, err := myf.Read(b1); err == nil {
|
||||
if strings.Compare(strings.Trim(string(b1), "\t \r\n"), "framework.ConformanceIt(\"") == 0 {
|
||||
return comm.Text()
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//comment section's end is noticed within 10 characters from framework.ConformanceIt block
|
||||
return comm.Text()
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (v *visitor) emit(arg ast.Expr) {
|
||||
switch at := arg.(type) {
|
||||
case *ast.BasicLit:
|
||||
if at.Kind != token.STRING {
|
||||
v.failf(at, "framework.ConformanceIt() called with non-string argument")
|
||||
return
|
||||
}
|
||||
|
||||
description := v.getDescription(at.Value)
|
||||
err := validateTestName(description)
|
||||
if err != nil {
|
||||
v.failf(at, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
at.Value = normalizeTestName(at.Value)
|
||||
if *confDoc {
|
||||
v.convertToConformanceData(at)
|
||||
} else {
|
||||
fmt.Printf("%s: %q\n", v.FileSet.Position(at.Pos()).Filename, at.Value)
|
||||
}
|
||||
default:
|
||||
v.failf(at, "framework.ConformanceIt() called with non-literal argument")
|
||||
fmt.Fprintf(os.Stderr, "ERROR: non-literal argument %v at %v\n", arg, v.FileSet.Position(arg.Pos()))
|
||||
}
|
||||
}
|
||||
|
||||
func (v *visitor) getDescription(value string) string {
|
||||
tokens := []string{}
|
||||
for _, describe := range v.describes {
|
||||
tokens = append(tokens, describe.text)
|
||||
if len(describe.lastContext.text) > 0 {
|
||||
tokens = append(tokens, describe.lastContext.text)
|
||||
}
|
||||
}
|
||||
tokens = append(tokens, value)
|
||||
|
||||
trimmed := []string{}
|
||||
for _, token := range tokens {
|
||||
trimmed = append(trimmed, strings.Trim(token, "\""))
|
||||
}
|
||||
|
||||
return strings.Join(trimmed, " ")
|
||||
}
|
||||
|
||||
var (
|
||||
regexTag = regexp.MustCompile(`(\[[a-zA-Z0-9:-]+\])`)
|
||||
)
|
||||
|
||||
// normalizeTestName removes tags (e.g., [Feature:Foo]), double quotes and trim
|
||||
// the spaces to normalize the test name.
|
||||
func normalizeTestName(s string) string {
|
||||
r := regexTag.ReplaceAllString(s, "")
|
||||
r = strings.Trim(r, "\"")
|
||||
return strings.TrimSpace(r)
|
||||
}
|
||||
|
||||
func validateTestName(s string) error {
|
||||
matches := regexIneligibleTags.FindAllString(s, -1)
|
||||
if matches != nil {
|
||||
return fmt.Errorf("'%s' cannot have invalid tags %v", s, strings.Join(matches, ","))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// funcName converts a selectorExpr with two idents into a string,
|
||||
// x.y -> "x.y"
|
||||
func funcName(n ast.Expr) string {
|
||||
if sel, ok := n.(*ast.SelectorExpr); ok {
|
||||
if x, ok := sel.X.(*ast.Ident); ok {
|
||||
return x.String() + "." + sel.Sel.String()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// isSprintf returns whether the given node is a call to fmt.Sprintf
|
||||
func isSprintf(n ast.Expr) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
return ok && funcName(call.Fun) == "fmt.Sprintf" && len(call.Args) != 0
|
||||
}
|
||||
|
||||
// firstArg attempts to statically determine the value of the first
|
||||
// argument. It only handles strings, and converts any unknown values
|
||||
// (fmt.Sprintf interpolations) into *.
|
||||
func (v *visitor) firstArg(n *ast.CallExpr) string {
|
||||
if len(n.Args) == 0 {
|
||||
return ""
|
||||
}
|
||||
var lit *ast.BasicLit
|
||||
if isSprintf(n.Args[0]) {
|
||||
return v.firstArg(n.Args[0].(*ast.CallExpr))
|
||||
}
|
||||
lit, ok := n.Args[0].(*ast.BasicLit)
|
||||
if ok && lit.Kind == token.STRING {
|
||||
val, err := strconv.Unquote(lit.Value)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if strings.Contains(val, "%") {
|
||||
val = strings.Replace(val, "%d", "*", -1)
|
||||
val = strings.Replace(val, "%v", "*", -1)
|
||||
val = strings.Replace(val, "%s", "*", -1)
|
||||
}
|
||||
return val
|
||||
}
|
||||
if ident, ok := n.Args[0].(*ast.Ident); ok {
|
||||
return ident.String()
|
||||
}
|
||||
return "*"
|
||||
}
|
||||
|
||||
// matchFuncName returns the first argument of a function if it's
|
||||
// a Ginkgo-relevant function (Describe/KubeDescribe/Context),
|
||||
// and the empty string otherwise.
|
||||
func (v *visitor) matchFuncName(n *ast.CallExpr, pattern string) string {
|
||||
switch x := n.Fun.(type) {
|
||||
case *ast.SelectorExpr:
|
||||
if match, err := regexp.MatchString(pattern, x.Sel.Name); err == nil && match {
|
||||
return v.firstArg(n)
|
||||
}
|
||||
case *ast.Ident:
|
||||
if match, err := regexp.MatchString(pattern, x.Name); err == nil && match {
|
||||
return v.firstArg(n)
|
||||
}
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Visit visits each node looking for either calls to framework.ConformanceIt,
|
||||
// which it will emit in its list of conformance tests, or legacy calls to
|
||||
// It() with a manually embedded [Conformance] tag, which it will complain
|
||||
// about.
|
||||
func (v *visitor) Visit(node ast.Node) (w ast.Visitor) {
|
||||
lastDescribe := len(v.describes) - 1
|
||||
|
||||
switch t := node.(type) {
|
||||
case *ast.CallExpr:
|
||||
if name := v.matchFuncName(t, regexDescribe); name != "" && len(t.Args) >= 2 {
|
||||
v.describes = append(v.describes, describe{text: name, rparen: t.Rparen})
|
||||
} else if name := v.matchFuncName(t, regexContext); name != "" && len(t.Args) >= 2 {
|
||||
if lastDescribe > -1 {
|
||||
v.describes[lastDescribe].lastContext = context{text: name}
|
||||
}
|
||||
} else if v.isConformanceCall(t) {
|
||||
totalConfTests++
|
||||
v.emit(t.Args[0])
|
||||
return nil
|
||||
} else if v.isLegacyItCall(t) {
|
||||
totalLegacyTests++
|
||||
v.failf(t, "Using It() with manual [Conformance] tag is no longer allowed. Use framework.ConformanceIt() instead.")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// If we're past the position of the last describe's rparen, pop the describe off
|
||||
if lastDescribe > -1 && node != nil {
|
||||
if node.Pos() > v.describes[lastDescribe].rparen {
|
||||
v.describes = v.describes[:lastDescribe]
|
||||
}
|
||||
}
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
func scanfile(path string, src interface{}) []conformanceData {
|
||||
v := newVisitor()
|
||||
file, err := parser.ParseFile(v.FileSet, path, src, parser.ParseComments)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
v.cMap = ast.NewCommentMap(v.FileSet, file, file.Comments)
|
||||
|
||||
ast.Walk(v, file)
|
||||
return v.tests
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
if len(flag.Args()) < 1 {
|
||||
fmt.Fprintf(os.Stderr, "USAGE: %s <DIR or FILE> [...]\n", os.Args[0])
|
||||
os.Exit(64)
|
||||
}
|
||||
|
||||
if *confDoc {
|
||||
// Note: this assumes that you're running from the root of the kube src repo
|
||||
templ, err := template.ParseFiles("test/conformance/cf_header.md")
|
||||
if err != nil {
|
||||
fmt.Printf("Error reading the Header file information: %s\n\n", err)
|
||||
}
|
||||
data := struct {
|
||||
Version string
|
||||
}{
|
||||
Version: *version,
|
||||
}
|
||||
templ.Execute(os.Stdout, data)
|
||||
}
|
||||
|
||||
totalConfTests = 0
|
||||
totalLegacyTests = 0
|
||||
missingComments = 0
|
||||
for _, arg := range flag.Args() {
|
||||
filepath.Walk(arg, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.HasSuffix(path, ".go") {
|
||||
tests := scanfile(path, nil)
|
||||
for _, cd := range tests {
|
||||
fmt.Printf("## [%s](%s)\n\n", cd.TestName, cd.URL)
|
||||
fmt.Printf("### Release %s\n", cd.Release)
|
||||
fmt.Printf("%s\n\n", cd.Description)
|
||||
if len(cd.Description) < 10 {
|
||||
missingComments++
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if *confDoc {
|
||||
fmt.Println("\n## **Summary**")
|
||||
fmt.Printf("\nTotal Conformance Tests: %d, total legacy tests that need conversion: %d, while total tests that need comment sections: %d\n\n", totalConfTests, totalLegacyTests, missingComments)
|
||||
}
|
||||
cd.Description = strings.Join(descLines, " ")
|
||||
return cd
|
||||
}
|
||||
|
||||
@ -22,127 +22,152 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
var conformanceCases = []struct {
|
||||
filename string
|
||||
code string
|
||||
output []conformanceData
|
||||
}{
|
||||
// Go unit test
|
||||
{"test/list/main_test.go", `
|
||||
var num = 3
|
||||
func Helper(x int) { return x / 0 }
|
||||
var _ = Describe("Feature", func() {
|
||||
/*
|
||||
Testname: Kubelet-OutputToLogs
|
||||
Description: By default the stdout and stderr from the process
|
||||
being executed in a pod MUST be sent to the pod's logs.
|
||||
*/
|
||||
framework.ConformanceIt("validates describe with ConformanceIt", func() {})
|
||||
})`, []conformanceData{{URL: "https://github.com/kubernetes/kubernetes/tree/master/test/list/main_test.go#L11", TestName: "Kubelet-OutputToLogs",
|
||||
Description: `By default the stdout and stderr from the process
|
||||
being executed in a pod MUST be sent to the pod's logs.` + "\n\n"}},
|
||||
},
|
||||
// Describe + It
|
||||
{"e2e/foo.go", `
|
||||
var _ = Describe("Feature", func() {
|
||||
//It should have comment
|
||||
framework.ConformanceIt("should work properly", func() {})
|
||||
})`, []conformanceData{{URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L5", TestName: "Feature should work properly", Description: "It should have comment\n\n"}},
|
||||
},
|
||||
// KubeDescribe + It
|
||||
{"e2e/foo.go", `
|
||||
var _ = framework.KubeDescribe("Feature", func() {
|
||||
/*It should have comment*/
|
||||
framework.ConformanceIt("should work properly", func() {})
|
||||
})`, []conformanceData{{URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L5", TestName: "Feature should work properly", Description: "It should have comment\n\n"}},
|
||||
},
|
||||
// KubeDescribe + Context + It
|
||||
{"e2e/foo.go", `
|
||||
var _ = framework.KubeDescribe("Feature", func() {
|
||||
Context("when offline", func() {
|
||||
//Testname: Kubelet-OutputToLogs
|
||||
//Description: By default the stdout and stderr from the process
|
||||
//being executed in a pod MUST be sent to the pod's logs.
|
||||
framework.ConformanceIt("should work", func() {})
|
||||
})
|
||||
})`, []conformanceData{{URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L8", TestName: "Kubelet-OutputToLogs",
|
||||
Description: `By default the stdout and stderr from the process
|
||||
being executed in a pod MUST be sent to the pod's logs.` + "\n\n"}},
|
||||
},
|
||||
// SIGDescribe + KubeDescribe + It, Describe + KubeDescribe + It
|
||||
{"e2e/foo.go", `
|
||||
var _ = framework.SIGDescribe("Feature", func() {
|
||||
KubeDescribe("Described by", func() {
|
||||
// Description: description1
|
||||
framework.ConformanceIt("A ConformanceIt", func() {})
|
||||
})
|
||||
Describe("Also described via", func() {
|
||||
KubeDescribe("A nested", func() {
|
||||
// Description: description2
|
||||
framework.ConformanceIt("ConformanceIt", func() {})
|
||||
})
|
||||
})
|
||||
})`, []conformanceData{
|
||||
{URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L6", TestName: "Feature Described by A ConformanceIt", Description: "description1\n\n"},
|
||||
{URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L11", TestName: "Feature Also described via A nested ConformanceIt", Description: "description2\n\n"},
|
||||
}},
|
||||
// KubeDescribe + Context + It
|
||||
{"e2e/foo.go", `
|
||||
var _ = framework.KubeDescribe("Feature", func() {
|
||||
Context("with context", func() {
|
||||
//Description: By default the stdout and stderr from the process
|
||||
//being executed in a pod MUST be sent to the pod's logs.
|
||||
framework.ConformanceIt("should work", func() {})
|
||||
})
|
||||
})`, []conformanceData{{URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L7", TestName: "Feature with context should work",
|
||||
Description: `By default the stdout and stderr from the process
|
||||
being executed in a pod MUST be sent to the pod's logs.` + "\n\n"}},
|
||||
},
|
||||
{"e2e/foo.go", `
|
||||
var _ = framework.KubeDescribe("Feature", func() {
|
||||
Context("with context and extra spaces before It block should still pick up Testname", func() {
|
||||
// Testname: Test with spaces
|
||||
//Description: Should pick up testname even if it is not within 3 spaces
|
||||
//even when executed from memory.
|
||||
framework.ConformanceIt("should work", func() {})
|
||||
})
|
||||
})`, []conformanceData{{URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L8", TestName: "Test with spaces",
|
||||
Description: `Should pick up testname even if it is not within 3 spaces
|
||||
even when executed from memory.` + "\n\n"}},
|
||||
},
|
||||
}
|
||||
|
||||
func TestConformance(t *testing.T) {
|
||||
for _, test := range conformanceCases {
|
||||
code := "package test\n" + test.code
|
||||
*confDoc = true
|
||||
tests := scanfile(test.filename, code)
|
||||
if !reflect.DeepEqual(tests, test.output) {
|
||||
t.Errorf("code:\n%s\ngot %+v\nwant %+v",
|
||||
code, tests, test.output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeTestNames(t *testing.T) {
|
||||
testCases := []struct {
|
||||
rawName string
|
||||
normalizedName string
|
||||
for _, tc := range []struct {
|
||||
desc string
|
||||
filename string
|
||||
code string
|
||||
targetFrame frame
|
||||
output *conformanceData
|
||||
}{
|
||||
{
|
||||
"should have monotonically increasing restart count [Slow]",
|
||||
"should have monotonically increasing restart count",
|
||||
desc: "Grabs comment above test",
|
||||
filename: "test/list/main_test.go",
|
||||
code: `package test
|
||||
|
||||
var num = 3
|
||||
func Helper(x int) { return x / 0 }
|
||||
var _ = Describe("Feature", func() {
|
||||
/*
|
||||
Testname: Kubelet-OutputToLogs
|
||||
Description: By default the stdout and stderr from the process
|
||||
being executed in a pod MUST be sent to the pod's logs.
|
||||
*/
|
||||
framework.ConformanceIt("validates describe with ConformanceIt", func() {})
|
||||
})`,
|
||||
output: &conformanceData{
|
||||
URL: "https://github.com/kubernetes/kubernetes/tree/master/test/list/main_test.go#L11",
|
||||
TestName: "Kubelet-OutputToLogs",
|
||||
Description: `By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.`,
|
||||
File: "test/list/main_test.go",
|
||||
},
|
||||
targetFrame: frame{File: "test/list/main_test.go", Line: 11},
|
||||
}, {
|
||||
desc: "Handles extra spaces",
|
||||
filename: "e2e/foo.go",
|
||||
code: `package test
|
||||
|
||||
var _ = framework.KubeDescribe("Feature", func() {
|
||||
Context("with context and extra spaces before It block should still pick up Testname", func() {
|
||||
// Testname: Test with spaces
|
||||
//Description: Should pick up testname even if it is not within 3 spaces
|
||||
//even when executed from memory.
|
||||
framework.ConformanceIt("should work", func() {})
|
||||
})
|
||||
})`,
|
||||
output: &conformanceData{
|
||||
URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L8",
|
||||
TestName: "Test with spaces",
|
||||
Description: `Should pick up testname even if it is not within 3 spaces even when executed from memory.`,
|
||||
File: "e2e/foo.go",
|
||||
},
|
||||
targetFrame: frame{File: "e2e/foo.go", Line: 8},
|
||||
}, {
|
||||
desc: "Should target the correct comment based on the line numbers (second)",
|
||||
filename: "e2e/foo.go",
|
||||
code: `package test
|
||||
|
||||
var _ = framework.KubeDescribe("Feature", func() {
|
||||
Context("with context and extra spaces before It block should still pick up Testname", func() {
|
||||
// Testname: First test
|
||||
// Description: Should pick up testname even if it is not within 3 spaces
|
||||
// even when executed from memory.
|
||||
framework.ConformanceIt("should work", func() {})
|
||||
|
||||
// Testname: Second test
|
||||
// Description: Should target the correct test/comment based on the line numbers
|
||||
framework.ConformanceIt("should work", func() {})
|
||||
})
|
||||
})`,
|
||||
output: &conformanceData{
|
||||
URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L13",
|
||||
TestName: "Second test",
|
||||
Description: `Should target the correct test/comment based on the line numbers`,
|
||||
File: "e2e/foo.go",
|
||||
},
|
||||
targetFrame: frame{File: "e2e/foo.go", Line: 13},
|
||||
}, {
|
||||
desc: "Should target the correct comment based on the line numbers (first)",
|
||||
filename: "e2e/foo.go",
|
||||
code: `package test
|
||||
|
||||
var _ = framework.KubeDescribe("Feature", func() {
|
||||
Context("with context and extra spaces before It block should still pick up Testname", func() {
|
||||
// Testname: First test
|
||||
// Description: Should target the correct test/comment based on the line numbers
|
||||
framework.ConformanceIt("should work", func() {})
|
||||
|
||||
// Testname: Second test
|
||||
// Description: Should target the correct test/comment based on the line numbers
|
||||
framework.ConformanceIt("should work", func() {})
|
||||
})
|
||||
})`,
|
||||
output: &conformanceData{
|
||||
URL: "https://github.com/kubernetes/kubernetes/tree/master/e2e/foo.go#L8",
|
||||
TestName: "First test",
|
||||
Description: `Should target the correct test/comment based on the line numbers`,
|
||||
File: "e2e/foo.go",
|
||||
},
|
||||
targetFrame: frame{File: "e2e/foo.go", Line: 8},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
*confDoc = true
|
||||
cd, err := scanFileForFrame(tc.filename, tc.code, tc.targetFrame)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if !reflect.DeepEqual(cd, tc.output) {
|
||||
t.Errorf("code:\n%s\ngot %+v\nwant %+v",
|
||||
tc.code, cd, tc.output)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommentToConformanceData(t *testing.T) {
|
||||
tcs := []struct {
|
||||
desc string
|
||||
input string
|
||||
expected *conformanceData
|
||||
}{
|
||||
{
|
||||
" should check is all data is printed ",
|
||||
"should check is all data is printed",
|
||||
desc: "Empty comment leads to nil",
|
||||
}, {
|
||||
desc: "No Release or Testname leads to nil",
|
||||
input: "Description: foo",
|
||||
}, {
|
||||
desc: "Release but no Testname does not result in nil",
|
||||
input: "Release: v1.1\nDescription: foo",
|
||||
expected: &conformanceData{Release: "v1.1", Description: "foo"},
|
||||
}, {
|
||||
desc: "Testname but no Release does not result in nil",
|
||||
input: "Testname: mytest\nDescription: foo",
|
||||
expected: &conformanceData{TestName: "mytest", Description: "foo"},
|
||||
}, {
|
||||
desc: "All fields parsed and newlines and whitespace removed from description",
|
||||
input: "Release: v1.1\n\t\tTestname: mytest\n\t\tDescription: foo\n\t\tbar\ndone",
|
||||
expected: &conformanceData{TestName: "mytest", Release: "v1.1", Description: "foo bar done"},
|
||||
},
|
||||
}
|
||||
for i, tc := range testCases {
|
||||
actualName := normalizeTestName(tc.rawName)
|
||||
if actualName != tc.normalizedName {
|
||||
t.Errorf("test case[%d]: expected normalized name %q, got %q", i, tc.normalizedName, actualName)
|
||||
}
|
||||
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
out := commentToConformanceData(tc.input)
|
||||
if !reflect.DeepEqual(out, tc.expected) {
|
||||
t.Errorf("Expected %#v but got %#v", tc.expected, out)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -46,7 +46,6 @@ var _ = SIGDescribe("ReplicationController", func() {
|
||||
Testname: Replication Controller, run basic image
|
||||
Description: Replication Controller MUST create a Pod with Basic Image and MUST run the service with the provided image. Image MUST be tested by dialing into the service listening through TCP, UDP and HTTP.
|
||||
*/
|
||||
|
||||
framework.ConformanceIt("should serve a basic image on each replica with a public image ", func() {
|
||||
TestReplicationControllerServeImageOrFail(f, "basic", framework.ServeHostnameImage)
|
||||
})
|
||||
|
||||
@ -115,6 +115,12 @@ func RunE2ETests(t *testing.T) {
|
||||
// Stream the progress to stdout and optionally a URL accepting progress updates.
|
||||
r = append(r, e2ereporters.NewProgressReporter(framework.TestContext.ProgressReportURL))
|
||||
|
||||
// The DetailsRepoerter will output details about every test (name, files, lines, etc) which helps
|
||||
// when documenting our tests.
|
||||
if len(framework.TestContext.SpecSummaryOutput) > 0 {
|
||||
r = append(r, e2ereporters.NewDetailsReporterFile(framework.TestContext.SpecSummaryOutput))
|
||||
}
|
||||
|
||||
klog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
|
||||
ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r)
|
||||
}
|
||||
|
||||
@ -170,6 +170,9 @@ type TestContextType struct {
|
||||
|
||||
// SriovdpConfigMapFile is the path to the ConfigMap to configure the SRIOV device plugin on this host.
|
||||
SriovdpConfigMapFile string
|
||||
|
||||
// SpecSummaryOutput is the file to write ginkgo.SpecSummary objects to as tests complete. Useful for debugging and test introspection.
|
||||
SpecSummaryOutput string
|
||||
}
|
||||
|
||||
// NodeKillerConfig describes configuration of NodeKiller -- a utility to
|
||||
@ -298,6 +301,7 @@ func RegisterCommonFlags(flags *flag.FlagSet) {
|
||||
flags.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.")
|
||||
|
||||
flags.StringVar(&TestContext.ProgressReportURL, "progress-report-url", "", "The URL to POST progress updates to as the suite runs to assist in aiding integrations. If empty, no messages sent.")
|
||||
flags.StringVar(&TestContext.SpecSummaryOutput, "spec-dump", "", "The file to dump all ginkgo.SpecSummary to after tests run. If empty, no objects are saved/printed.")
|
||||
}
|
||||
|
||||
// RegisterClusterFlags registers flags specific to the cluster e2e test suite.
|
||||
|
||||
@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["progress.go"],
|
||||
srcs = [
|
||||
"progress.go",
|
||||
"testDetails.go",
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/test/e2e/reporters",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
|
||||
100
test/e2e/reporters/testDetails.go
Normal file
100
test/e2e/reporters/testDetails.go
Normal file
@ -0,0 +1,100 @@
|
||||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package reporters
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/onsi/ginkgo/config"
|
||||
"github.com/onsi/ginkgo/types"
|
||||
"k8s.io/klog"
|
||||
)
|
||||
|
||||
// DetailsReporter is a ginkgo reporter which dumps information regarding the tests which is difficult to get
|
||||
// via an AST app. This allows us to leverage the existing ginkgo logic to walk the tests and such while then following
|
||||
// up with an custom app which leverages AST to generate conformance documentation.
|
||||
type DetailsReporter struct {
|
||||
Writer io.Writer
|
||||
}
|
||||
|
||||
// NewDetailsReporterWithWriter returns a reporter which will write the SpecSummary objects as tests
|
||||
// complete to the given writer.
|
||||
func NewDetailsReporterWithWriter(w io.Writer) *DetailsReporter {
|
||||
return &DetailsReporter{
|
||||
Writer: w,
|
||||
}
|
||||
}
|
||||
|
||||
// NewDetailsReporterFile returns a reporter which will create the file given and dump the specs
|
||||
// to it as they complete.
|
||||
func NewDetailsReporterFile(filename string) *DetailsReporter {
|
||||
absPath, err := filepath.Abs(filename)
|
||||
if err != nil {
|
||||
klog.Errorf("%#v\n", err)
|
||||
panic(err)
|
||||
}
|
||||
f, err := os.Create(absPath)
|
||||
if err != nil {
|
||||
klog.Errorf("%#v\n", err)
|
||||
panic(err)
|
||||
}
|
||||
return NewDetailsReporterWithWriter(f)
|
||||
}
|
||||
|
||||
// SpecSuiteWillBegin is implemented as a noop to satisfy the reporter interface for ginkgo.
|
||||
func (reporter *DetailsReporter) SpecSuiteWillBegin(cfg config.GinkgoConfigType, summary *types.SuiteSummary) {
|
||||
}
|
||||
|
||||
// SpecSuiteDidEnd is implemented as a noop to satisfy the reporter interface for ginkgo.
|
||||
func (reporter *DetailsReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) {}
|
||||
|
||||
// SpecDidComplete is invoked by Ginkgo each time a spec is completed (including skipped specs).
|
||||
func (reporter *DetailsReporter) SpecDidComplete(specSummary *types.SpecSummary) {
|
||||
b, err := json.Marshal(specSummary)
|
||||
if err != nil {
|
||||
klog.Errorf("Error in detail reporter: %v", err)
|
||||
return
|
||||
}
|
||||
_, err = reporter.Writer.Write(b)
|
||||
if err != nil {
|
||||
klog.Errorf("Error saving test details in detail reporter: %v", err)
|
||||
return
|
||||
}
|
||||
// Printing newline between records for easier viewing in various tools.
|
||||
_, err = fmt.Fprintln(reporter.Writer, "")
|
||||
if err != nil {
|
||||
klog.Errorf("Error saving test details in detail reporter: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// SpecWillRun is implemented as a noop to satisfy the reporter interface for ginkgo.
|
||||
func (reporter *DetailsReporter) SpecWillRun(specSummary *types.SpecSummary) {}
|
||||
|
||||
// BeforeSuiteDidRun is implemented as a noop to satisfy the reporter interface for ginkgo.
|
||||
func (reporter *DetailsReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {}
|
||||
|
||||
// AfterSuiteDidRun is implemented as a noop to satisfy the reporter interface for ginkgo.
|
||||
func (reporter *DetailsReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {
|
||||
if c, ok := reporter.Writer.(io.Closer); ok {
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user