Add etcd snapshot fix and more log messages to certificate bundle

Add function to collect stdout and stderr logs from containers

cluster/network.go

@@ -1,14 +1,12 @@
 package cluster
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net"
 	"strings"
 
 	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/go-connections/nat"
 	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
@@ -404,8 +402,6 @@ func (c *Cluster) runServicePortChecks(ctx context.Context) error {
 
 func checkPlaneTCPPortsFromHost(ctx context.Context, host *hosts.Host, portList []string, planeHosts []*hosts.Host, image string, prsMap map[string]v3.PrivateRegistry) error {
 	var hosts []string
-	var containerStdout bytes.Buffer
-	var containerStderr bytes.Buffer
 
 	for _, host := range planeHosts {
 		hosts = append(hosts, host.InternalAddress)
@@ -435,14 +431,10 @@ func checkPlaneTCPPortsFromHost(ctx context.Context, host *hosts.Host, portList
 		return err
 	}
 
-	clogs, err := docker.ReadContainerLogs(ctx, host.DClient, PortCheckContainer, true, "all")
-	if err != nil {
-		return err
+	containerLog, logsErr := docker.GetContainerLogsStdoutStderr(ctx, host.DClient, PortCheckContainer, "1")
+	if logsErr != nil {
+		log.Warnf(ctx, "[network] Failed to get network port check logs: %v", logsErr)
 	}
-	defer clogs.Close()
-
-	stdcopy.StdCopy(&containerStdout, &containerStderr, clogs)
-	containerLog := containerStderr.String()
 	logrus.Debugf("[network] containerLog [%s] on host: %s", containerLog, host.Address)
 
 	if err := docker.RemoveContainer(ctx, host.DClient, host.Address, PortCheckContainer); err != nil {

docker/docker.go

@@ -2,6 +2,7 @@ package docker
 
 import (
 	"archive/tar"
+	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
@@ -17,6 +18,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/rancher/rke/log"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
 	"github.com/sirupsen/logrus"
@@ -362,6 +364,22 @@ func ReadContainerLogs(ctx context.Context, dClient *client.Client, containerNam
 	return dClient.ContainerLogs(ctx, containerName, types.ContainerLogsOptions{Follow: follow, ShowStdout: true, ShowStderr: true, Timestamps: false, Tail: tail})
 }
 
+func GetContainerLogsStdoutStderr(ctx context.Context, dClient *client.Client, containerName, tail string) (string, error) {
+	var containerStderr bytes.Buffer
+	var containerStdout bytes.Buffer
+	var containerLog string
+	clogs, logserr := ReadContainerLogs(ctx, dClient, containerName, false, tail)
+	if logserr != nil {
+		logrus.Debug("logserr: %v", logserr)
+		return containerLog, fmt.Errorf("Failed to get gather logs from container [%s]: %v", containerName, logserr)
+	}
+	defer clogs.Close()
+	stdcopy.StdCopy(&containerStdout, &containerStderr, clogs)
+	containerLog = containerStderr.String()
+	containerLog = strings.TrimSuffix(containerLog, "\n")
+	return containerLog, nil
+}
+
 func tryRegistryAuth(pr v3.PrivateRegistry) types.RequestPrivilegeFunc {
 	return func() (string, error) {
 		return getRegistryAuth(pr)

pki/pki.go

@@ -255,6 +255,7 @@ func SaveBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSystemI
 	if status != 0 {
 		return fmt.Errorf("Failed to run certificate bundle compress, exit status is: %d", status)
 	}
+	log.Infof(ctx, "[certificates] successfully saved certificate bundle [%s/pki.bundle.tar.gz] on host [%s]", etcdSnapshotPath, host.Address)
 	return docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer)
 }
 
@@ -264,7 +265,12 @@ func ExtractBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSyst
 		Cmd: []string{
 			"sh",
 			"-c",
-			fmt.Sprintf("mkdir -p %s; tar xzvf %s -C %s --strip-components %d", fullTempCertPath, BundleCertPath, fullTempCertPath, len(strings.Split(fullTempCertPath, "/"))-1),
+			fmt.Sprintf(
+				"mkdir -p %s; tar xzvf %s -C %s --strip-components %d",
+				fullTempCertPath,
+				BundleCertPath,
+				fullTempCertPath,
+				len(strings.Split(fullTempCertPath, "/"))-1),
 		},
 		Image: alpineSystemImage,
 	}
@@ -284,7 +290,16 @@ func ExtractBackupBundleOnHost(ctx context.Context, host *hosts.Host, alpineSyst
 		return err
 	}
 	if status != 0 {
-		return fmt.Errorf("Failed to run certificate bundle extract, exit status is: %d", status)
+		containerLog, err := docker.GetContainerLogsStdoutStderr(ctx, host.DClient, BundleCertContainer, "5")
+		if err != nil {
+			return err
 		}
+		// removing the container in case of an error too
+		if err := docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer); err != nil {
+			return err
+		}
+		return fmt.Errorf("Failed to run certificate bundle extract, exit status is: %d, container logs: %s", status, containerLog)
+	}
+	log.Infof(ctx, "[certificates] successfully extracted certificate bundle on host [%s] to backup path [%s]", host.Address, fullTempCertPath)
 	return docker.RemoveContainer(ctx, host.DClient, host.Address, BundleCertContainer)
 }

services/etcd.go

@@ -268,10 +268,12 @@ func RestoreEtcdSnapshot(ctx context.Context, etcdHost *hosts.Host, prsMap map[s
 	nodeName := pki.GetEtcdCrtName(etcdHost.InternalAddress)
 	snapshotPath := filepath.Join(EtcdSnapshotPath, snapshotName)
 
+	// make sure that restore path is empty otherwise etcd restore will fail
 	imageCfg := &container.Config{
 		Cmd: []string{
 			"sh", "-c", strings.Join([]string{
-				"/usr/local/bin/etcdctl",
+				"rm -rf", EtcdRestorePath,
+				"&& /usr/local/bin/etcdctl",
 				fmt.Sprintf("--endpoints=[%s:2379]", etcdHost.InternalAddress),
 				"--cacert", pki.GetCertPath(pki.CACertName),
 				"--cert", pki.GetCertPath(nodeName),
@@ -304,7 +306,15 @@ func RestoreEtcdSnapshot(ctx context.Context, etcdHost *hosts.Host, prsMap map[s
 		return err
 	}
 	if status != 0 {
-		return fmt.Errorf("Failed to run etcd restore container, exit status is: %d", status)
+		containerLog, err := docker.GetContainerLogsStdoutStderr(ctx, etcdHost.DClient, EtcdRestoreContainerName, "5")
+		if err != nil {
+			return err
+		}
+		if err := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdRestoreContainerName); err != nil {
+			return err
+		}
+		// printing the restore container's logs
+		return fmt.Errorf("Failed to run etcd restore container, exit status is: %d, container logs: %s", status, containerLog)
 	}
 	return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdRestoreContainerName)
 }

services/healthcheck.go

@@ -1,7 +1,6 @@
 package services
 
 import (
-	"bytes"
 	"context"
 	"crypto/tls"
 	"fmt"
@@ -11,7 +10,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/log"
@@ -28,9 +26,6 @@ const (
 )
 
 func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, localConnDialerFactory hosts.DialerFactory, url string, certMap map[string]pki.CertificatePKI) error {
-	var containerStderr bytes.Buffer
-	var containerStdout bytes.Buffer
-
 	log.Infof(ctx, "[healthcheck] Start Healthcheck on service [%s] on host [%s]", serviceName, host.Address)
 	var x509Pair tls.Certificate
 
@@ -68,16 +63,10 @@ func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, l
 		return nil
 	}
 	logrus.Debug("Checking container logs")
-	clogs, logserr := docker.ReadContainerLogs(ctx, host.DClient, serviceName, false, "1")
-	defer clogs.Close()
+	containerLog, logserr := docker.GetContainerLogsStdoutStderr(ctx, host.DClient, serviceName, "1")
 	if logserr != nil {
-		logrus.Debug("logserr: %v", logserr)
-		return fmt.Errorf("Failed to verify healthcheck: %v", err)
+		return fmt.Errorf("Failed to verify healthcheck for service [%s]: %v", serviceName, logserr)
 	}
-	stdcopy.StdCopy(&containerStdout, &containerStderr, clogs)
-	containerLog := containerStderr.String()
-	containerLog = strings.TrimSuffix(containerLog, "\n")
-
 	return fmt.Errorf("Failed to verify healthcheck: %v, log: %v", err, containerLog)
 }