Link kernel log files to have them be copied over as part of node e2e.

Signed-off-by: Vishnu kannan <vishnuk@google.com>

test/e2e_node/e2e_node_suite_test.go

@@ -101,10 +101,13 @@ var _ = BeforeSuite(func() {
 
 // Tear down the kubelet on the node
 var _ = AfterSuite(func() {
-	if e2es != nil && *startServices && *stopServices {
+	if e2es != nil {
+		e2es.getLogFiles()
+		if *startServices && *stopServices {
 			glog.Infof("Stopping node services...")
 			e2es.stop()
 		}
+	}
 	glog.Infof("Tests Finished")
 })
 
@@ -117,6 +120,6 @@ func maskLocksmithdOnCoreos() {
 		if output, err := exec.Command("sudo", "systemctl", "mask", "--now", "locksmithd").CombinedOutput(); err != nil {
 			glog.Fatalf("Could not mask locksmithd: %v, output: %q", err, string(output))
 		}
-	}
 		glog.Infof("Locksmithd is masked successfully")
+	}
 }

test/e2e_node/e2e_remote.go

@@ -188,6 +188,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFileNumber int) (
 	glog.Infof("Copying test artifacts from %s", host)
 	scpErr := getTestArtifacts(host, tmp)
 	exitOk := true
+	glog.Errorf("vishh: %v", scpErr)
 	if scpErr != nil {
 		// Only exit non-0 if the scp failed
 		exitOk = false

test/e2e_node/e2e_service.go

@@ -75,6 +75,71 @@ func (es *e2eService) start() error {
 	return nil
 }
 
+// Get logs of interest either via journalctl or by creating sym links.
+// Since we scp files from the remote directory, symlinks will be treated as normal files and file contents will be copied over.
+func (es *e2eService) getLogFiles() {
+	// Special log files that need to be collected for additional debugging.
+	type logFileData struct {
+		files             []string
+		journalctlCommand []string
+	}
+	var logFiles = map[string]logFileData{
+		"kern.log":   {[]string{"/var/log/kern.log"}, []string{"-k"}},
+		"docker.log": {[]string{"/var/log/docker.log", "/var/log/upstart/docker.log"}, []string{"-u", "docker"}},
+	}
+
+	// Nothing to do if report dir is not specified.
+	if *reportDir == "" {
+		return
+	}
+	journaldFound := isJournaldAvailable()
+	for targetFileName, logFileData := range logFiles {
+		targetLink := path.Join(*reportDir, targetFileName)
+		if journaldFound {
+			// Skip log files that do not have an equivalent in journald based machines.
+			if len(logFileData.journalctlCommand) == 0 {
+				continue
+			}
+			out, err := exec.Command("sudo", append([]string{"journalctl"}, logFileData.journalctlCommand...)...).CombinedOutput()
+			if err != nil {
+				glog.Errorf("failed to get %q from journald: %v, %v", targetFileName, string(out), err)
+			} else {
+				if err = ioutil.WriteFile(targetLink, out, 0755); err != nil {
+					glog.Errorf("failed to write logs to %q: %v", targetLink, err)
+				}
+			}
+			continue
+		}
+		for _, file := range logFileData.files {
+			if _, err := os.Stat(file); err != nil {
+				// Expected file not found on this distro.
+				continue
+			}
+			if err := copyLogFile(file, targetLink); err != nil {
+				glog.Error(err)
+			} else {
+				break
+			}
+		}
+	}
+}
+
+func copyLogFile(src, target string) error {
+	// If not a journald based distro, then just symlink files.
+	if out, err := exec.Command("sudo", "cp", src, target).CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to copy %q to %q: %v, %v", src, target, out, err)
+	}
+	if out, err := exec.Command("sudo", "chmod", "a+r", target).CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to make log file %q world readable: %v, %v", target, out, err)
+	}
+	return nil
+}
+
+func isJournaldAvailable() bool {
+	_, err := exec.LookPath("journalctl")
+	return err == nil
+}
+
 func (es *e2eService) stop() {
 	if es.kubeletCmd != nil {
 		err := es.kubeletCmd.Process.Kill()