diff --git a/cluster/logs.go b/cluster/logs.go index b77b532a..829a1ec1 100644 --- a/cluster/logs.go +++ b/cluster/logs.go @@ -2,60 +2,24 @@ package cluster import ( "context" - "fmt" - "github.com/docker/docker/api/types/container" - "github.com/rancher/rke/docker" "github.com/rancher/rke/hosts" - "github.com/rancher/rke/services" - "github.com/rancher/types/apis/management.cattle.io/v3" - "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) func (c *Cluster) CleanDeadLogs(ctx context.Context) error { - hosts := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) + hostList := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) var errgrp errgroup.Group - for _, host := range hosts { + for _, host := range hostList { if !host.UpdateWorker { continue } runHost := host errgrp.Go(func() error { - return doRunLogCleaner(ctx, runHost, c.SystemImages.Alpine, c.PrivateRegistriesMap) + return hosts.DoRunLogCleaner(ctx, runHost, c.SystemImages.Alpine, c.PrivateRegistriesMap) }) } return errgrp.Wait() } - -func doRunLogCleaner(ctx context.Context, host *hosts.Host, alpineImage string, prsMap map[string]v3.PrivateRegistry) error { - logrus.Debugf("[cleanup] Starting log link cleanup on host [%s]", host.Address) - imageCfg := &container.Config{ - Image: alpineImage, - Tty: true, - Cmd: []string{ - "sh", - "-c", - fmt.Sprintf("find %s -type l ! -exec test -e {} \\; -print -delete", services.RKELogsPath), - }, - } - hostCfg := &container.HostConfig{ - Binds: []string{ - "/var/lib:/var/lib", - }, - Privileged: true, - } - if err := docker.DoRemoveContainer(ctx, host.DClient, services.LogCleanerContainerName, host.Address); err != nil { - return err - } - if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, services.LogCleanerContainerName, host.Address, "cleanup", prsMap); err != nil { - return err - } - if err := docker.DoRemoveContainer(ctx, host.DClient, services.LogCleanerContainerName, host.Address); err != nil { - return err - } - logrus.Debugf("[cleanup] Successfully cleaned up log links on host [%s]", host.Address) - return nil -} diff --git a/hosts/hosts.go b/hosts/hosts.go index e04f8b77..5e5a0055 100644 --- a/hosts/hosts.go +++ b/hosts/hosts.go @@ -8,6 +8,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + "github.com/sirupsen/logrus" "github.com/docker/docker/client" "github.com/rancher/rke/docker" @@ -40,14 +41,16 @@ type Host struct { } const ( - ToCleanEtcdDir = "/var/lib/etcd/" - ToCleanSSLDir = "/etc/kubernetes/" - ToCleanCNIConf = "/etc/cni/" - ToCleanCNIBin = "/opt/cni/" - ToCleanCNILib = "/var/lib/cni/" - ToCleanCalicoRun = "/var/run/calico/" - ToCleanTempCertPath = "/etc/kubernetes/.tmp/" - CleanerContainerName = "kube-cleaner" + ToCleanEtcdDir = "/var/lib/etcd/" + ToCleanSSLDir = "/etc/kubernetes/" + ToCleanCNIConf = "/etc/cni/" + ToCleanCNIBin = "/opt/cni/" + ToCleanCNILib = "/var/lib/cni/" + ToCleanCalicoRun = "/var/run/calico/" + ToCleanTempCertPath = "/etc/kubernetes/.tmp/" + CleanerContainerName = "kube-cleaner" + LogCleanerContainerName = "rke-log-cleaner" + RKELogsPath = "/var/lib/rancher/rke/log" B2DOS = "Boot2Docker" B2DPrefixPath = "/mnt/sda1/rke" @@ -134,6 +137,10 @@ func (h *Host) CleanUp(ctx context.Context, toCleanPaths []string, cleanerImage if err := docker.RemoveContainer(ctx, h.DClient, h.Address, CleanerContainerName); err != nil { return err } + log.Infof(ctx, "[hosts] Removing dead container logs on host [%s]", h.Address) + if err := DoRunLogCleaner(ctx, h, cleanerImage, prsMap); err != nil { + return err + } log.Infof(ctx, "[hosts] Successfully cleaned up host [%s]", h.Address) return nil } @@ -308,3 +315,33 @@ func GetPrefixPath(osType, ClusterPrefixPath string) string { } return prefixPath } + +func DoRunLogCleaner(ctx context.Context, host *Host, alpineImage string, prsMap map[string]v3.PrivateRegistry) error { + logrus.Debugf("[cleanup] Starting log link cleanup on host [%s]", host.Address) + imageCfg := &container.Config{ + Image: alpineImage, + Tty: true, + Cmd: []string{ + "sh", + "-c", + fmt.Sprintf("find %s -type l ! -exec test -e {} \\; -print -delete", RKELogsPath), + }, + } + hostCfg := &container.HostConfig{ + Binds: []string{ + "/var/lib:/var/lib", + }, + Privileged: true, + } + if err := docker.DoRemoveContainer(ctx, host.DClient, LogCleanerContainerName, host.Address); err != nil { + return err + } + if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, LogCleanerContainerName, host.Address, "cleanup", prsMap); err != nil { + return err + } + if err := docker.DoRemoveContainer(ctx, host.DClient, LogCleanerContainerName, host.Address); err != nil { + return err + } + logrus.Debugf("[cleanup] Successfully cleaned up log links on host [%s]", host.Address) + return nil +} diff --git a/services/services.go b/services/services.go index e079ea76..ae22c5dd 100644 --- a/services/services.go +++ b/services/services.go @@ -39,8 +39,6 @@ const ( KubeControllerPort = 10252 KubeletPort = 10250 KubeproxyPort = 10256 - - RKELogsPath = "/var/lib/rancher/rke/log" ) func runSidekick(ctx context.Context, host *hosts.Host, prsMap map[string]v3.PrivateRegistry, sidecarProcess v3.Process) error { @@ -106,14 +104,14 @@ func createLogLink(ctx context.Context, host *hosts.Host, containerName, plane, } containerID := containerInspect.ID containerLogPath := containerInspect.LogPath - containerLogLink := fmt.Sprintf("%s/%s_%s.log", RKELogsPath, containerName, containerID) + containerLogLink := fmt.Sprintf("%s/%s_%s.log", hosts.RKELogsPath, containerName, containerID) imageCfg := &container.Config{ Image: image, Tty: true, Cmd: []string{ "sh", "-c", - fmt.Sprintf("mkdir -p %s ; ln -s %s %s", RKELogsPath, containerLogPath, containerLogLink), + fmt.Sprintf("mkdir -p %s ; ln -s %s %s", hosts.RKELogsPath, containerLogPath, containerLogLink), }, } hostCfg := &container.HostConfig{