reenable e2e_node services & debugging improvements

- re-enable e2e_node services
 - call GenerateSecureToken for e2e_node Conformance test-suite
 - add log messages indicating location in process
 - move log messages to some more accurate locations

test/e2e/framework/test_context.go

@@ -407,7 +407,10 @@ func createKubeConfig(clientCfg *restclient.Config) *clientcmdapi.Config {
 	return configCmd
 }
 
-func generateSecureToken(tokenLen int) (string, error) {
+// GenerateSecureToken returns a string of length tokenLen, consisting
+// of random bytes encoded as base64 for use as a Bearer Token during
+// communication with an APIServer
+func GenerateSecureToken(tokenLen int) (string, error) {
 	// Number of bytes to be tokenLen when base64 encoded.
 	tokenSize := math.Ceil(float64(tokenLen) * 6 / 8)
 	rawToken := make([]byte, int(tokenSize))
@@ -440,11 +443,12 @@ func AfterReadingAllFlags(t *TestContextType) {
 	}
 	if len(t.BearerToken) == 0 {
 		var err error
-		t.BearerToken, err = generateSecureToken(16)
+		t.BearerToken, err = GenerateSecureToken(16)
 		if err != nil {
 			klog.Fatalf("Failed to generate bearer token: %v", err)
 		}
 	}
+
 	// Allow 1% of nodes to be unready (statistically) - relevant for large clusters.
 	if t.AllowedNotReadyNodes == 0 {
 		t.AllowedNotReadyNodes = t.CloudConfig.NumNodes / 100

test/e2e_node/e2e_node_suite_test.go

@@ -197,7 +197,6 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 		// If the services are expected to keep running after test, they should not monitor the test process.
 		e2es = services.NewE2EServices(*stopServices)
 		gomega.Expect(e2es.Start()).To(gomega.Succeed(), "should be able to start node services.")
-		klog.Infof("Node services started.  Running tests...")
 	} else {
 		klog.Infof("Running tests without starting services.")
 	}

test/e2e_node/remote/BUILD

@@ -19,6 +19,7 @@ go_library(
     importpath = "k8s.io/kubernetes/test/e2e_node/remote",
     deps = [
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//test/e2e/framework:go_default_library",
         "//test/e2e_node/builder:go_default_library",
         "//test/e2e_node/system:go_default_library",
         "//test/utils:go_default_library",

test/e2e_node/remote/node_conformance.go

@@ -27,6 +27,7 @@ import (
 
 	"k8s.io/klog/v2"
 
+	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e_node/builder"
 	"k8s.io/kubernetes/test/utils"
 )
@@ -135,6 +136,7 @@ func (c *ConformanceRemote) SetupTestPackage(tardir, systemSpecName string) erro
 
 // loadConformanceImage loads node conformance image from tar file.
 func loadConformanceImage(host, workspace string) error {
+	klog.Info("Loading conformance image from tarfile")
 	tarfile := filepath.Join(workspace, conformanceTarfile)
 	if output, err := SSH(host, "timeout", conformanceImageLoadTimeout.String(),
 		"docker", "load", "-i", tarfile); err != nil {
@@ -173,15 +175,16 @@ func isSystemd(host string) (bool, error) {
 // with cluster e2e and launch kubelet outside of the test for both regular node e2e and
 // node conformance test.
 // TODO(random-liu): Switch to use standard node bootstrap script.
-func launchKubelet(host, workspace, results, testArgs string) error {
+func launchKubelet(host, workspace, results, testArgs, bearerToken string) error {
 	podManifestPath := getPodPath(workspace)
 	if output, err := SSH(host, "mkdir", podManifestPath); err != nil {
 		return fmt.Errorf("failed to create kubelet pod manifest path %q: error - %v output - %q",
 			podManifestPath, err, output)
 	}
 	startKubeletCmd := fmt.Sprintf("./%s --run-kubelet-mode --logtostderr --node-name=%s"+
+		" --bearer-token=%s"+
 		" --report-dir=%s %s --kubelet-flags=--pod-manifest-path=%s > %s 2>&1",
-		conformanceTestBinary, host, results, testArgs, podManifestPath, filepath.Join(results, kubeletLauncherLog))
+		conformanceTestBinary, host, bearerToken, results, testArgs, podManifestPath, filepath.Join(results, kubeletLauncherLog))
 	var cmd []string
 	systemd, err := isSystemd(host)
 	if err != nil {
@@ -278,8 +281,13 @@ func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFi
 		return "", err
 	}
 
+	bearerToken, err := framework.GenerateSecureToken(16)
+	if err != nil {
+		return "", err
+	}
+
 	// Launch kubelet.
-	if err := launchKubelet(host, workspace, results, testArgs); err != nil {
+	if err := launchKubelet(host, workspace, results, testArgs, bearerToken); err != nil {
 		return "", err
 	}
 	// Stop kubelet.
@@ -293,12 +301,7 @@ func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFi
 	// Run the tests
 	klog.V(2).Infof("Starting tests on %q", host)
 	podManifestPath := getPodPath(workspace)
-	cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result -e TEST_ARGS=--report-prefix=%s -e EXTRA_ENVS=%s %s'",
+	cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result -e TEST_ARGS=--report-prefix=%s -e EXTRA_ENVS=%s -e TEST_ARGS=--bearer-token=%s %s'",
-		timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, extraEnvs, getConformanceTestImageName(systemSpecName))
+		timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, extraEnvs, bearerToken, getConformanceTestImageName(systemSpecName))
-	testOutput, err := SSH(host, "sh", "-c", cmd)
+	return SSH(host, "sh", "-c", cmd)
-	if err != nil {
-		return testOutput, err
-	}
-
-	return testOutput, nil
 }

test/e2e_node/services/services.go

@@ -64,15 +64,20 @@ func NewE2EServices(monitorParent bool) *E2EServices {
 // standard kubelet launcher)
 func (e *E2EServices) Start() error {
 	var err error
-	if !framework.TestContext.NodeConformance {
+	if e.services, err = e.startInternalServices(); err != nil {
-		if e.services, err = e.startInternalServices(); err != nil {
+		return fmt.Errorf("failed to start internal services: %v", err)
-			return fmt.Errorf("failed to start internal services: %v", err)
+	}
-		}
+	klog.Infof("Node services started.")
+	// running the kubelet depends on whether we are running conformance test-suite
+	if framework.TestContext.NodeConformance {
+		klog.Info("nothing to do in node-e2e-services, running conformance suite")
+	} else {
 		// Start kubelet
 		e.kubelet, err = e.startKubelet()
 		if err != nil {
 			return fmt.Errorf("failed to start kubelet: %v", err)
 		}
+		klog.Infof("Kubelet started.")
 	}
 	return nil
 }