diff --git a/docs/devel/e2e-tests.md b/docs/devel/e2e-tests.md index 50163815332..a9c440c73ef 100644 --- a/docs/devel/e2e-tests.md +++ b/docs/devel/e2e-tests.md @@ -185,7 +185,6 @@ We are working on implementing clearer partitioning of our e2e tests to make run - `[Serial]`: If a test cannot be run in parallel with other tests (e.g. it takes too many resources or restarts nodes), it is labeled `[Serial]`, and should be run in serial as part of a separate suite. - `[Disruptive]`: If a test restarts components that might cause other tests to fail or break the cluster completely, it is labeled `[Disruptive]`. Any `[Disruptive]` test is also assumed to qualify for the `[Serial]` label, but need not be labeled as both. These tests are not run against soak clusters to avoid restarting components. - `[Flaky]`: If a test is found to be flaky and we have decided that it's too hard to fix in the short term (e.g. it's going to take a full engineer-week), it receives the `[Flaky]` label until it is fixed. The `[Flaky]` label should be used very sparingly, and should be accompanied with a reference to the issue for de-flaking the test, because while a test remains labeled `[Flaky]`, it is not monitored closely in CI. `[Flaky]` tests are by default not run, unless a `focus` or `skip` argument is explicitly given. -- `[Skipped]`: `[Skipped]` is a legacy label that we're phasing out. If a test is marked `[Skipped]`, there should be an issue open to label it properly. `[Skipped]` tests are by default not run, unless a `focus` or `skip` argument is explicitly given. - `[Feature:.+]`: If a test has non-default requirements to run or targets some non-core functionality, and thus should not be run as part of the standard suite, it receives a `[Feature:.+]` label, e.g. `[Feature:Performance]` or `[Feature:Ingress]`. `[Feature:.+]` tests are not run in our core suites, instead running in custom suites. If a feature is experimental or alpha and is not enabled by default due to being incomplete or potentially subject to breaking changes, it does *not* block the merge-queue, and thus should run in some separate test suites owned by the feature owner(s) (see #continuous_integration below). ### Conformance tests diff --git a/hack/conformance-test.sh b/hack/conformance-test.sh index 641c810c87d..17453200fe5 100755 --- a/hack/conformance-test.sh +++ b/hack/conformance-test.sh @@ -84,4 +84,4 @@ echo "Conformance test checking conformance with Kubernetes version 1.0" declare -x KUBERNETES_CONFORMANCE_TEST="y" declare -x NUM_NODES=4 -exec hack/ginkgo-e2e.sh -ginkgo.focus='\[Conformance\]' -ginkgo.skip='\[Skipped\]' ${TEST_ARGS} +exec hack/ginkgo-e2e.sh -ginkgo.focus='\[Conformance\]' ${TEST_ARGS} diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 729b8ba0553..47fcdd7a3dd 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -225,8 +225,7 @@ case ${JOB_NAME} in : ${E2E_NETWORK:="e2e-gce-${NODE_NAME}-${EXECUTOR_NUMBER}"} : ${GINKGO_PARALLEL:="y"} # This list should match the list in kubernetes-e2e-gce. - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-gce-${NODE_NAME}-${EXECUTOR_NUMBER}"} : ${PROJECT:="kubernetes-jenkins-pull"} # Override GCE defaults @@ -241,8 +240,7 @@ case ${JOB_NAME} in : ${E2E_PUBLISH_GREEN_VERSION:="true"} : ${E2E_NETWORK:="e2e-gce"} # This list should match the list in kubernetes-pull-build-test-e2e-gce. - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${GINKGO_PARALLEL:="y"} : ${KUBE_GCE_INSTANCE_PREFIX="e2e-gce"} : ${PROJECT:="k8s-jkns-e2e-gce"} @@ -253,9 +251,8 @@ case ${JOB_NAME} in kubernetes-e2e-gce-slow) : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-slow"} : ${E2E_NETWORK:="e2e-slow"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Slow\] \ - --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${GINKGO_PARALLEL:="y"} : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-slow"} : ${PROJECT:="k8s-jkns-e2e-gce-slow"} @@ -304,9 +301,8 @@ case ${JOB_NAME} in kubernetes-e2e-gce-flaky) : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-flaky"} : ${E2E_NETWORK:="e2e-flaky"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Flaky\] \ - --ginkgo.skip=\[Feature:.+\]|\[Skipped\]"} + --ginkgo.skip=\[Feature:.+\]"} : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-flaky"} : ${PROJECT:="k8s-jkns-e2e-gce-flaky"} : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} @@ -318,9 +314,8 @@ case ${JOB_NAME} in : ${E2E_CLUSTER_NAME:="parallel-flaky"} : ${E2E_NETWORK:="e2e-parallel-flaky"} : ${GINKGO_PARALLEL:="y"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Flaky\] \ - --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Feature:.+\]|\[Skipped\]"} + --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Feature:.+\]"} : ${KUBE_GCE_INSTANCE_PREFIX:="parallel-flaky"} : ${PROJECT:="k8s-jkns-e2e-gce-prl-flaky"} : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} @@ -337,8 +332,7 @@ case ${JOB_NAME} in : ${E2E_SET_CLUSTER_API_VERSION:=y} : ${PROJECT:="k8s-jkns-e2e-gke-ci"} : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${GINKGO_PARALLEL:="y"} ;; @@ -348,9 +342,8 @@ case ${JOB_NAME} in : ${E2E_SET_CLUSTER_API_VERSION:=y} : ${PROJECT:="k8s-jkns-e2e-gke-slow"} : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Slow\] \ - --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${GINKGO_PARALLEL:="y"} ;; @@ -371,9 +364,8 @@ case ${JOB_NAME} in : ${E2E_SET_CLUSTER_API_VERSION:=y} : ${PROJECT:="k8s-jkns-e2e-gke-ci-flaky"} : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Flaky\] \ - --ginkgo.skip=\[Feature:.+\]|\[Skipped\]"} + --ginkgo.skip=\[Feature:.+\]"} ;; # AWS core jobs @@ -385,8 +377,7 @@ case ${JOB_NAME} in : ${E2E_ZONE:="us-west-2a"} : ${ZONE:="us-west-2a"} : ${E2E_NETWORK:="e2e-aws"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${GINKGO_PARALLEL:="y"} : ${KUBE_GCE_INSTANCE_PREFIX="e2e-aws"} : ${PROJECT:="k8s-jkns-e2e-aws"} @@ -619,9 +610,7 @@ case ${JOB_NAME} in # We should be testing the reliability of a long-running cluster. The # [Disruptive] tests kill/restart components or nodes in the cluster, # defeating the purpose of a soak cluster. (#15722) - # - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${KUBE_GCE_INSTANCE_PREFIX:="gce-soak-weekly"} : ${HAIRPIN_MODE:="false"} : ${PROJECT:="kubernetes-jenkins"} @@ -649,9 +638,7 @@ case ${JOB_NAME} in # We should be testing the reliability of a long-running cluster. The # [Disruptive] tests kill/restart components or nodes in the cluster, # defeating the purpose of a soak cluster. (#15722) - # - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${KUBE_GCE_INSTANCE_PREFIX:="gce-soak-weekly-2"} : ${PROJECT:="kubernetes-jenkins"} ;; @@ -683,9 +670,7 @@ case ${JOB_NAME} in # We should be testing the reliability of a long-running cluster. The # [Disruptive] tests kill/restart components or nodes in the cluster, # defeating the purpose of a soak cluster. (#15722) - # - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} ;; # Upgrade jobs diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 58a010eea52..b76d63d7160 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -221,8 +221,7 @@ func RunE2ETests(t *testing.T) { gomega.RegisterFailHandler(ginkgo.Fail) // Disable skipped tests unless they are explicitly requested. if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" { - // TODO(ihmccreery) Remove [Skipped] once all [Skipped] labels have been reclassified. - config.GinkgoConfig.SkipString = `\[Flaky\]|\[Skipped\]|\[Feature:.+\]` + config.GinkgoConfig.SkipString = `\[Flaky\]|\[Feature:.+\]` } // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins