Merge pull request #78710 from spiffxp/conformance-exclude-tags

Error on tests ineligible for promotion to conformance

test/conformance/walk.go

@@ -41,6 +41,9 @@ var (
 	baseURL                                           = flag.String("url", "https://github.com/kubernetes/kubernetes/tree/master/", "location of the current source")
 	confDoc                                           = flag.Bool("conformance", false, "write a conformance document")
 	totalConfTests, totalLegacyTests, missingComments int
+
+	// If a test name contains any of these tags, it is ineligble for promotion to conformance
+	regexIneligibleTags = regexp.MustCompile(`\[(Alpha|Disruptive|Feature:[^\]]+|Flaky)\]`)
 )
 
 const regexDescribe = "Describe|KubeDescribe|SIGDescribe"
@@ -199,6 +202,12 @@ func (v *visitor) emit(arg ast.Expr) {
 			return
 		}
 
+		err := validateTestName(v.getDescription(at.Value))
+		if err != nil {
+			v.failf(at, err.Error())
+			return
+		}
+
 		at.Value = normalizeTestName(at.Value)
 		if *confDoc {
 			v.convertToConformanceData(at)
@@ -233,6 +242,14 @@ func normalizeTestName(s string) string {
 	return strings.TrimSpace(r)
 }
 
+func validateTestName(s string) error {
+	matches := regexIneligibleTags.FindAllString(s, -1)
+	if matches != nil {
+		return fmt.Errorf("'%s' cannot have invalid tags %v", s, strings.Join(matches, ","))
+	}
+	return nil
+}
+
 // funcName converts a selectorExpr with two idents into a string,
 // x.y -> "x.y"
 func funcName(n ast.Expr) string {

test/conformance/walk_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 )
@@ -127,3 +128,57 @@ func TestNormalizeTestNames(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateTestName(t *testing.T) {
+	testCases := []struct {
+		testName  string
+		tagString string
+	}{
+		{
+			"a test case with no tags",
+			"",
+		},
+		{
+			"a test case with valid tags [LinuxOnly] [NodeConformance] [Serial]",
+			"",
+		},
+		{
+			"a flaky test case that is invalid [Flaky]",
+			"[Flaky]",
+		},
+		{
+			"a disruptive test case that is invalid [Disruptive]",
+			"[Disruptive]",
+		},
+		{
+			"a feature test case that is invalid [Feature:Awesome]",
+			"[Feature:Awesome]",
+		},
+		{
+			"an alpha test case that is invalid [Alpha]",
+			"[Alpha]",
+		},
+		{
+			"a test case with multiple invalid tags [Flaky][Disruptive] [Feature:Awesome] [Alpha]",
+			"[Flaky],[Disruptive],[Feature:Awesome],[Alpha]",
+		},
+		{
+			"[sig-awesome] [Disruptive] a test case with valid and invalid tags [Alpha] [Serial] [Flaky]",
+			"[Disruptive],[Alpha],[Flaky]",
+		},
+	}
+	for i, tc := range testCases {
+		err := validateTestName(tc.testName)
+		if err != nil {
+			if tc.tagString == "" {
+				t.Errorf("test case[%d]: expected no validate error, got %q", i, err.Error())
+			} else {
+				expectedMsg := fmt.Sprintf("'%s' cannot have invalid tags %s", tc.testName, tc.tagString)
+				actualMsg := err.Error()
+				if actualMsg != expectedMsg {
+					t.Errorf("test case[%d]: expected error message %q, got %q", i, expectedMsg, actualMsg)
+				}
+			}
+		}
+	}
+}