From 0341e4c2f3c084ca9489bf2ef5a0ef060a63d24b Mon Sep 17 00:00:00 2001 From: Amim Knabben Date: Mon, 8 Mar 2021 12:19:59 -0500 Subject: [PATCH] Enabling runtime config on E2E node tests --- hack/make-rules/test-e2e-node.sh | 7 +++++-- test/e2e/framework/test_context.go | 2 ++ test/e2e_node/e2e_node_suite_test.go | 1 + test/e2e_node/remote/cadvisor_e2e.go | 2 +- test/e2e_node/remote/node_conformance.go | 2 +- test/e2e_node/remote/node_e2e.go | 6 +++--- test/e2e_node/remote/remote.go | 4 ++-- test/e2e_node/remote/types.go | 3 ++- test/e2e_node/runner/local/run_local.go | 3 ++- test/e2e_node/runner/remote/run_remote.go | 3 ++- test/e2e_node/services/apiserver.go | 3 +++ 11 files changed, 24 insertions(+), 12 deletions(-) diff --git a/hack/make-rules/test-e2e-node.sh b/hack/make-rules/test-e2e-node.sh index ef2a355ec2f..f72ced2a4f0 100755 --- a/hack/make-rules/test-e2e-node.sh +++ b/hack/make-rules/test-e2e-node.sh @@ -46,6 +46,7 @@ test_args=${TEST_ARGS:-""} timeout_arg="" system_spec_name=${SYSTEM_SPEC_NAME:-} extra_envs=${EXTRA_ENVS:-} +runtime_config=${RUNTIME_CONFIG:-} # Parse the flags to pass to ginkgo ginkgoflags="" @@ -168,7 +169,8 @@ if [ "${remote}" = true ] ; then --image-project="${image_project}" --instance-name-prefix="${instance_prefix}" \ --delete-instances="${delete_instances}" --test_args="${test_args}" --instance-metadata="${metadata}" \ --image-config-file="${image_config_file}" --system-spec-name="${system_spec_name}" \ - --preemptible-instances="${preemptible_instances}" --extra-envs="${extra_envs}" --test-suite="${test_suite}" \ + --runtime-config="${runtime_config}" --preemptible-instances="${preemptible_instances}" \ + --extra-envs="${extra_envs}" --test-suite="${test_suite}" \ "${timeout_arg}" \ 2>&1 | tee -i "${artifacts}/build-log.txt" exit $? @@ -203,6 +205,7 @@ else --system-spec-name="${system_spec_name}" --extra-envs="${extra_envs}" \ --ginkgo-flags="${ginkgoflags}" --test-flags="--container-runtime=${runtime} \ --alsologtostderr --v 4 --report-dir=${artifacts} --node-name $(hostname) \ - ${test_args}" --build-dependencies=true 2>&1 | tee -i "${artifacts}/build-log.txt" + ${test_args}" --runtime-config="${runtime_config}" \ + --build-dependencies=true 2>&1 | tee -i "${artifacts}/build-log.txt" exit $? fi diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 474b61cf996..cce5ad97fec 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -217,6 +217,8 @@ type NodeTestContextType struct { KubeletConfig kubeletconfig.KubeletConfiguration // ImageDescription is the description of the image on which the test is running. ImageDescription string + // RuntimeConfig is a map of API server runtime configuration values. + RuntimeConfig map[string]string // SystemSpecName is the name of the system spec (e.g., gke) that's used in // the node e2e test. If empty, the default one (system.DefaultSpec) is // used. The system specs are in test/e2e_node/system/specs/. diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 1d5db9fa93a..900e009442e 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -85,6 +85,7 @@ func registerNodeFlags(flags *flag.FlagSet) { flags.Var(cliflag.NewMapStringString(&framework.TestContext.ExtraEnvs), "extra-envs", "The extra environment variables needed for node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2") flags.StringVar(&framework.TestContext.SriovdpConfigMapFile, "sriovdp-configmap-file", "", "The name of the SRIOV device plugin Config Map to load.") flag.StringVar(&framework.TestContext.ClusterDNSDomain, "dns-domain", "", "The DNS Domain of the cluster.") + flag.Var(cliflag.NewMapStringString(&framework.TestContext.RuntimeConfig), "runtime-config", "The runtime configuration used on node e2e tests.") } func init() { diff --git a/test/e2e_node/remote/cadvisor_e2e.go b/test/e2e_node/remote/cadvisor_e2e.go index 3e475a00390..f1b48bd5aab 100644 --- a/test/e2e_node/remote/cadvisor_e2e.go +++ b/test/e2e_node/remote/cadvisor_e2e.go @@ -63,7 +63,7 @@ func runCommand(command string, args ...string) error { } // RunTest implements TestSuite.RunTest -func (n *CAdvisorE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, extraEnvs string, timeout time.Duration) (string, error) { +func (n *CAdvisorE2ERemote) RunTest(host, workspace, _, _, _, _, _, _, _, _ string, timeout time.Duration) (string, error) { // Kill any running node processes cleanupNodeProcesses(host) diff --git a/test/e2e_node/remote/node_conformance.go b/test/e2e_node/remote/node_conformance.go index ad51d6d41a6..48d99ca9438 100644 --- a/test/e2e_node/remote/node_conformance.go +++ b/test/e2e_node/remote/node_conformance.go @@ -262,7 +262,7 @@ func stopKubelet(host, workspace string) error { } // RunTest runs test on the node. -func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, _, systemSpecName, extraEnvs string, timeout time.Duration) (string, error) { +func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, _, systemSpecName, extraEnvs, _ string, timeout time.Duration) (string, error) { // Install the cni plugins and add a basic CNI configuration. if err := setupCNI(host, workspace); err != nil { return "", err diff --git a/test/e2e_node/remote/node_e2e.go b/test/e2e_node/remote/node_e2e.go index 02e87ed0a92..4ebd1859554 100644 --- a/test/e2e_node/remote/node_e2e.go +++ b/test/e2e_node/remote/node_e2e.go @@ -154,7 +154,7 @@ func getOSDistribution(host string) (string, error) { } // RunTest runs test on the node. -func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, extraEnvs string, timeout time.Duration) (string, error) { +func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, extraEnvs, runtimeConfig string, timeout time.Duration) (string, error) { // Install the cni plugins and add a basic CNI configuration. // TODO(random-liu): Do this in cloud init after we remove containervm test. if err := setupCNI(host, workspace); err != nil { @@ -183,8 +183,8 @@ func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePr klog.V(2).Infof("Starting tests on %q", host) cmd := getSSHCommand(" && ", fmt.Sprintf("cd %s", workspace), - fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-name=%s --system-spec-file=%s --extra-envs=%s --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=\"%s\" %s", - timeout.Seconds(), ginkgoArgs, systemSpecName, systemSpecFile, extraEnvs, host, results, junitFilePrefix, imageDesc, testArgs), + fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-name=%s --system-spec-file=%s --extra-envs=%s --runtime-config=%s --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=\"%s\" %s", + timeout.Seconds(), ginkgoArgs, systemSpecName, systemSpecFile, extraEnvs, runtimeConfig, host, results, junitFilePrefix, imageDesc, testArgs), ) return SSH(host, "sh", "-c", cmd) } diff --git a/test/e2e_node/remote/remote.go b/test/e2e_node/remote/remote.go index 5e3f38bf9ca..6a114ea28da 100644 --- a/test/e2e_node/remote/remote.go +++ b/test/e2e_node/remote/remote.go @@ -66,7 +66,7 @@ func CreateTestArchive(suite TestSuite, systemSpecName string) (string, error) { // RunRemote returns the command output, whether the exit was ok, and any errors // TODO(random-liu): junitFilePrefix is not prefix actually, the file name is junit-junitFilePrefix.xml. Change the variable name. -func RunRemote(suite TestSuite, archive string, host string, cleanup bool, imageDesc, junitFilePrefix string, testArgs string, ginkgoArgs string, systemSpecName string, extraEnvs string) (string, bool, error) { +func RunRemote(suite TestSuite, archive string, host string, cleanup bool, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, extraEnvs, runtimeConfig string) (string, bool, error) { // Create the temp staging directory klog.V(2).Infof("Staging test binaries on %q", host) workspace := newWorkspaceDir() @@ -111,7 +111,7 @@ func RunRemote(suite TestSuite, archive string, host string, cleanup bool, image } klog.V(2).Infof("Running test on %q", host) - output, err := suite.RunTest(host, workspace, resultDir, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, extraEnvs, *testTimeout) + output, err := suite.RunTest(host, workspace, resultDir, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, extraEnvs, runtimeConfig, *testTimeout) aggErrs := []error{} // Do not log the output here, let the caller deal with the test output. diff --git a/test/e2e_node/remote/types.go b/test/e2e_node/remote/types.go index 33d36fca5e9..decbf97fa2d 100644 --- a/test/e2e_node/remote/types.go +++ b/test/e2e_node/remote/types.go @@ -47,6 +47,7 @@ type TestSuite interface { // * systemSpecName is the name of the system spec used for validating the // image on which the test runs. // * extraEnvs is the extra environment variables needed for node e2e tests. + // * runtimeConfig is the API runtime configuration used for node e2e tests. // * timeout is the test timeout. - RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, extraEnvs string, timeout time.Duration) (string, error) + RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, extraEnvs, runtimeConfig string, timeout time.Duration) (string, error) } diff --git a/test/e2e_node/runner/local/run_local.go b/test/e2e_node/runner/local/run_local.go index 6c37d42cba2..e1b8fe3f8d9 100644 --- a/test/e2e_node/runner/local/run_local.go +++ b/test/e2e_node/runner/local/run_local.go @@ -36,6 +36,7 @@ var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of argum var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.") var systemSpecName = flag.String("system-spec-name", "", fmt.Sprintf("The name of the system spec used for validating the image in the node conformance test. The specs are at %s. If unspecified, the default built-in spec (system.DefaultSpec) will be used.", system.SystemSpecPath)) var extraEnvs = flag.String("extra-envs", "", "The extra environment variables needed for node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2") +var runtimeConfig = flag.String("runtime-config", "", "The runtime configuration for the API server on the node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2") func main() { klog.InitFlags(nil) @@ -57,7 +58,7 @@ func main() { ginkgo := filepath.Join(outputDir, "ginkgo") test := filepath.Join(outputDir, "e2e_node.test") - args := []string{*ginkgoFlags, test, "--", *testFlags} + args := []string{*ginkgoFlags, test, "--", *testFlags, fmt.Sprintf("--runtime-config=%s", *runtimeConfig)} if *systemSpecName != "" { rootDir, err := utils.GetK8sRootDir() if err != nil { diff --git a/test/e2e_node/runner/remote/run_remote.go b/test/e2e_node/runner/remote/run_remote.go index b1fdc5f9b00..0b25b2649b0 100644 --- a/test/e2e_node/runner/remote/run_remote.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -66,6 +66,7 @@ var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.") var systemSpecName = flag.String("system-spec-name", "", fmt.Sprintf("The name of the system spec used for validating the image in the node conformance test. The specs are at %s. If unspecified, the default built-in spec (system.DefaultSpec) will be used.", system.SystemSpecPath)) var extraEnvs = flag.String("extra-envs", "", "The extra environment variables needed for node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2") +var runtimeConfig = flag.String("runtime-config", "", "The runtime configuration for the API server on the node e2e tests.. Format: a list of key=value pairs, e.g., env1=val1,env2=val2") // envs is the type used to collect all node envs. The key is the env name, // and the value is the env value @@ -446,7 +447,7 @@ func testHost(host string, deleteFiles bool, imageDesc, junitFilePrefix, ginkgoF } } - output, exitOk, err := remote.RunRemote(suite, path, host, deleteFiles, imageDesc, junitFilePrefix, *testArgs, ginkgoFlagsStr, *systemSpecName, *extraEnvs) + output, exitOk, err := remote.RunRemote(suite, path, host, deleteFiles, imageDesc, junitFilePrefix, *testArgs, ginkgoFlagsStr, *systemSpecName, *extraEnvs, *runtimeConfig) return &TestResult{ output: output, err: err, diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index 18879437869..fe2ff8c69db 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -54,6 +54,9 @@ func (a *APIServer) Start() error { if err != nil { return err } + if len(framework.TestContext.RuntimeConfig) > 0 { + o.APIEnablement.RuntimeConfig = framework.TestContext.RuntimeConfig + } o.SecureServing.BindAddress = net.ParseIP("127.0.0.1") o.ServiceClusterIPRanges = ipnet.String() o.AllowPrivileged = true