diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 51ac128a5ad..e6f034252ac 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -407,7 +407,10 @@ func createKubeConfig(clientCfg *restclient.Config) *clientcmdapi.Config { return configCmd } -func generateSecureToken(tokenLen int) (string, error) { +// GenerateSecureToken returns a string of length tokenLen, consisting +// of random bytes encoded as base64 for use as a Bearer Token during +// communication with an APIServer +func GenerateSecureToken(tokenLen int) (string, error) { // Number of bytes to be tokenLen when base64 encoded. tokenSize := math.Ceil(float64(tokenLen) * 6 / 8) rawToken := make([]byte, int(tokenSize)) @@ -440,11 +443,12 @@ func AfterReadingAllFlags(t *TestContextType) { } if len(t.BearerToken) == 0 { var err error - t.BearerToken, err = generateSecureToken(16) + t.BearerToken, err = GenerateSecureToken(16) if err != nil { klog.Fatalf("Failed to generate bearer token: %v", err) } } + // Allow 1% of nodes to be unready (statistically) - relevant for large clusters. if t.AllowedNotReadyNodes == 0 { t.AllowedNotReadyNodes = t.CloudConfig.NumNodes / 100 diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index bc36f8f2702..c9eb85332fb 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -197,7 +197,6 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { // If the services are expected to keep running after test, they should not monitor the test process. e2es = services.NewE2EServices(*stopServices) gomega.Expect(e2es.Start()).To(gomega.Succeed(), "should be able to start node services.") - klog.Infof("Node services started. Running tests...") } else { klog.Infof("Running tests without starting services.") } diff --git a/test/e2e_node/remote/BUILD b/test/e2e_node/remote/BUILD index 66c81dcb6fe..fcfb11c0497 100644 --- a/test/e2e_node/remote/BUILD +++ b/test/e2e_node/remote/BUILD @@ -19,6 +19,7 @@ go_library( importpath = "k8s.io/kubernetes/test/e2e_node/remote", deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", + "//test/e2e/framework:go_default_library", "//test/e2e_node/builder:go_default_library", "//test/e2e_node/system:go_default_library", "//test/utils:go_default_library", diff --git a/test/e2e_node/remote/node_conformance.go b/test/e2e_node/remote/node_conformance.go index d7d7df80ed2..ad51d6d41a6 100644 --- a/test/e2e_node/remote/node_conformance.go +++ b/test/e2e_node/remote/node_conformance.go @@ -27,6 +27,7 @@ import ( "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e_node/builder" "k8s.io/kubernetes/test/utils" ) @@ -135,6 +136,7 @@ func (c *ConformanceRemote) SetupTestPackage(tardir, systemSpecName string) erro // loadConformanceImage loads node conformance image from tar file. func loadConformanceImage(host, workspace string) error { + klog.Info("Loading conformance image from tarfile") tarfile := filepath.Join(workspace, conformanceTarfile) if output, err := SSH(host, "timeout", conformanceImageLoadTimeout.String(), "docker", "load", "-i", tarfile); err != nil { @@ -173,15 +175,16 @@ func isSystemd(host string) (bool, error) { // with cluster e2e and launch kubelet outside of the test for both regular node e2e and // node conformance test. // TODO(random-liu): Switch to use standard node bootstrap script. -func launchKubelet(host, workspace, results, testArgs string) error { +func launchKubelet(host, workspace, results, testArgs, bearerToken string) error { podManifestPath := getPodPath(workspace) if output, err := SSH(host, "mkdir", podManifestPath); err != nil { return fmt.Errorf("failed to create kubelet pod manifest path %q: error - %v output - %q", podManifestPath, err, output) } startKubeletCmd := fmt.Sprintf("./%s --run-kubelet-mode --logtostderr --node-name=%s"+ + " --bearer-token=%s"+ " --report-dir=%s %s --kubelet-flags=--pod-manifest-path=%s > %s 2>&1", - conformanceTestBinary, host, results, testArgs, podManifestPath, filepath.Join(results, kubeletLauncherLog)) + conformanceTestBinary, host, bearerToken, results, testArgs, podManifestPath, filepath.Join(results, kubeletLauncherLog)) var cmd []string systemd, err := isSystemd(host) if err != nil { @@ -278,8 +281,13 @@ func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFi return "", err } + bearerToken, err := framework.GenerateSecureToken(16) + if err != nil { + return "", err + } + // Launch kubelet. - if err := launchKubelet(host, workspace, results, testArgs); err != nil { + if err := launchKubelet(host, workspace, results, testArgs, bearerToken); err != nil { return "", err } // Stop kubelet. @@ -293,12 +301,7 @@ func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFi // Run the tests klog.V(2).Infof("Starting tests on %q", host) podManifestPath := getPodPath(workspace) - cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result -e TEST_ARGS=--report-prefix=%s -e EXTRA_ENVS=%s %s'", - timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, extraEnvs, getConformanceTestImageName(systemSpecName)) - testOutput, err := SSH(host, "sh", "-c", cmd) - if err != nil { - return testOutput, err - } - - return testOutput, nil + cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result -e TEST_ARGS=--report-prefix=%s -e EXTRA_ENVS=%s -e TEST_ARGS=--bearer-token=%s %s'", + timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, extraEnvs, bearerToken, getConformanceTestImageName(systemSpecName)) + return SSH(host, "sh", "-c", cmd) } diff --git a/test/e2e_node/services/services.go b/test/e2e_node/services/services.go index 4fb73163840..ff9059240c4 100644 --- a/test/e2e_node/services/services.go +++ b/test/e2e_node/services/services.go @@ -64,15 +64,20 @@ func NewE2EServices(monitorParent bool) *E2EServices { // standard kubelet launcher) func (e *E2EServices) Start() error { var err error - if !framework.TestContext.NodeConformance { - if e.services, err = e.startInternalServices(); err != nil { - return fmt.Errorf("failed to start internal services: %v", err) - } + if e.services, err = e.startInternalServices(); err != nil { + return fmt.Errorf("failed to start internal services: %v", err) + } + klog.Infof("Node services started.") + // running the kubelet depends on whether we are running conformance test-suite + if framework.TestContext.NodeConformance { + klog.Info("nothing to do in node-e2e-services, running conformance suite") + } else { // Start kubelet e.kubelet, err = e.startKubelet() if err != nil { return fmt.Errorf("failed to start kubelet: %v", err) } + klog.Infof("Kubelet started.") } return nil }