Remove container dead log links on cluster remove

cluster/logs.go

@@ -2,60 +2,24 @@ package cluster
 
 import (
 	"context"
-	"fmt"
 
-	"github.com/docker/docker/api/types/container"
-	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
-	"github.com/rancher/rke/services"
-	"github.com/rancher/types/apis/management.cattle.io/v3"
-	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 )
 
 func (c *Cluster) CleanDeadLogs(ctx context.Context) error {
-	hosts := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts)
+	hostList := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts)
 
 	var errgrp errgroup.Group
 
-	for _, host := range hosts {
+	for _, host := range hostList {
 		if !host.UpdateWorker {
 			continue
 		}
 		runHost := host
 		errgrp.Go(func() error {
-			return doRunLogCleaner(ctx, runHost, c.SystemImages.Alpine, c.PrivateRegistriesMap)
+			return hosts.DoRunLogCleaner(ctx, runHost, c.SystemImages.Alpine, c.PrivateRegistriesMap)
 		})
 	}
 	return errgrp.Wait()
 }
-
-func doRunLogCleaner(ctx context.Context, host *hosts.Host, alpineImage string, prsMap map[string]v3.PrivateRegistry) error {
-	logrus.Debugf("[cleanup] Starting log link cleanup on host [%s]", host.Address)
-	imageCfg := &container.Config{
-		Image: alpineImage,
-		Tty:   true,
-		Cmd: []string{
-			"sh",
-			"-c",
-			fmt.Sprintf("find %s -type l ! -exec test -e {} \\; -print -delete", services.RKELogsPath),
-		},
-	}
-	hostCfg := &container.HostConfig{
-		Binds: []string{
-			"/var/lib:/var/lib",
-		},
-		Privileged: true,
-	}
-	if err := docker.DoRemoveContainer(ctx, host.DClient, services.LogCleanerContainerName, host.Address); err != nil {
-		return err
-	}
-	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, services.LogCleanerContainerName, host.Address, "cleanup", prsMap); err != nil {
-		return err
-	}
-	if err := docker.DoRemoveContainer(ctx, host.DClient, services.LogCleanerContainerName, host.Address); err != nil {
-		return err
-	}
-	logrus.Debugf("[cleanup] Successfully cleaned up log links on host [%s]", host.Address)
-	return nil
-}

hosts/hosts.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	"github.com/sirupsen/logrus"
 
 	"github.com/docker/docker/client"
 	"github.com/rancher/rke/docker"
@@ -40,14 +41,16 @@ type Host struct {
 }
 
 const (
-	ToCleanEtcdDir       = "/var/lib/etcd/"
+	ToCleanEtcdDir          = "/var/lib/etcd/"
-	ToCleanSSLDir        = "/etc/kubernetes/"
+	ToCleanSSLDir           = "/etc/kubernetes/"
-	ToCleanCNIConf       = "/etc/cni/"
+	ToCleanCNIConf          = "/etc/cni/"
-	ToCleanCNIBin        = "/opt/cni/"
+	ToCleanCNIBin           = "/opt/cni/"
-	ToCleanCNILib        = "/var/lib/cni/"
+	ToCleanCNILib           = "/var/lib/cni/"
-	ToCleanCalicoRun     = "/var/run/calico/"
+	ToCleanCalicoRun        = "/var/run/calico/"
-	ToCleanTempCertPath  = "/etc/kubernetes/.tmp/"
+	ToCleanTempCertPath     = "/etc/kubernetes/.tmp/"
-	CleanerContainerName = "kube-cleaner"
+	CleanerContainerName    = "kube-cleaner"
+	LogCleanerContainerName = "rke-log-cleaner"
+	RKELogsPath             = "/var/lib/rancher/rke/log"
 
 	B2DOS            = "Boot2Docker"
 	B2DPrefixPath    = "/mnt/sda1/rke"
@@ -134,6 +137,10 @@ func (h *Host) CleanUp(ctx context.Context, toCleanPaths []string, cleanerImage
 	if err := docker.RemoveContainer(ctx, h.DClient, h.Address, CleanerContainerName); err != nil {
 		return err
 	}
+	log.Infof(ctx, "[hosts] Removing dead container logs on host [%s]", h.Address)
+	if err := DoRunLogCleaner(ctx, h, cleanerImage, prsMap); err != nil {
+		return err
+	}
 	log.Infof(ctx, "[hosts] Successfully cleaned up host [%s]", h.Address)
 	return nil
 }
@@ -308,3 +315,33 @@ func GetPrefixPath(osType, ClusterPrefixPath string) string {
 	}
 	return prefixPath
 }
+
+func DoRunLogCleaner(ctx context.Context, host *Host, alpineImage string, prsMap map[string]v3.PrivateRegistry) error {
+	logrus.Debugf("[cleanup] Starting log link cleanup on host [%s]", host.Address)
+	imageCfg := &container.Config{
+		Image: alpineImage,
+		Tty:   true,
+		Cmd: []string{
+			"sh",
+			"-c",
+			fmt.Sprintf("find %s -type l ! -exec test -e {} \\; -print -delete", RKELogsPath),
+		},
+	}
+	hostCfg := &container.HostConfig{
+		Binds: []string{
+			"/var/lib:/var/lib",
+		},
+		Privileged: true,
+	}
+	if err := docker.DoRemoveContainer(ctx, host.DClient, LogCleanerContainerName, host.Address); err != nil {
+		return err
+	}
+	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, LogCleanerContainerName, host.Address, "cleanup", prsMap); err != nil {
+		return err
+	}
+	if err := docker.DoRemoveContainer(ctx, host.DClient, LogCleanerContainerName, host.Address); err != nil {
+		return err
+	}
+	logrus.Debugf("[cleanup] Successfully cleaned up log links on host [%s]", host.Address)
+	return nil
+}

services/services.go

@@ -39,8 +39,6 @@ const (
 	KubeControllerPort = 10252
 	KubeletPort        = 10250
 	KubeproxyPort      = 10256
-
-	RKELogsPath = "/var/lib/rancher/rke/log"
 )
 
 func runSidekick(ctx context.Context, host *hosts.Host, prsMap map[string]v3.PrivateRegistry, sidecarProcess v3.Process) error {
@@ -106,14 +104,14 @@ func createLogLink(ctx context.Context, host *hosts.Host, containerName, plane,
 	}
 	containerID := containerInspect.ID
 	containerLogPath := containerInspect.LogPath
-	containerLogLink := fmt.Sprintf("%s/%s_%s.log", RKELogsPath, containerName, containerID)
+	containerLogLink := fmt.Sprintf("%s/%s_%s.log", hosts.RKELogsPath, containerName, containerID)
 	imageCfg := &container.Config{
 		Image: image,
 		Tty:   true,
 		Cmd: []string{
 			"sh",
 			"-c",
-			fmt.Sprintf("mkdir -p %s ; ln -s %s %s", RKELogsPath, containerLogPath, containerLogLink),
+			fmt.Sprintf("mkdir -p %s ; ln -s %s %s", hosts.RKELogsPath, containerLogPath, containerLogLink),
 		},
 	}
 	hostCfg := &container.HostConfig{