mirror of
https://github.com/rancher/rke.git
synced 2025-04-27 19:25:44 +00:00
500 lines
14 KiB
Go
500 lines
14 KiB
Go
package hosts
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"path"
|
|
"strings"
|
|
|
|
"github.com/docker/docker/api/types"
|
|
"github.com/docker/docker/api/types/container"
|
|
"github.com/sirupsen/logrus"
|
|
|
|
"github.com/docker/docker/client"
|
|
"github.com/rancher/rke/docker"
|
|
"github.com/rancher/rke/k8s"
|
|
"github.com/rancher/rke/log"
|
|
v3 "github.com/rancher/rke/types"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/client-go/kubernetes"
|
|
)
|
|
|
|
type Host struct {
|
|
v3.RKEConfigNode
|
|
DClient *client.Client
|
|
LocalConnPort int
|
|
IsControl bool
|
|
IsWorker bool
|
|
IsEtcd bool
|
|
IgnoreDockerVersion bool
|
|
ToAddEtcdMember bool
|
|
ExistingEtcdCluster bool
|
|
SavedKeyPhrase string
|
|
ToAddLabels map[string]string
|
|
ToDelLabels map[string]string
|
|
ToAddTaints []string
|
|
ToDelTaints []string
|
|
DockerInfo types.Info
|
|
UpdateWorker bool
|
|
PrefixPath string
|
|
BastionHost v3.BastionHost
|
|
}
|
|
|
|
const (
|
|
ToCleanEtcdDir = "/var/lib/etcd/"
|
|
ToCleanSSLDir = "/etc/kubernetes/"
|
|
ToCleanCNIConf = "/etc/cni/"
|
|
ToCleanCNIBin = "/opt/cni/"
|
|
ToCleanCNILib = "/var/lib/cni/"
|
|
ToCleanCalicoRun = "/var/run/calico/"
|
|
ToCleanTempCertPath = "/etc/kubernetes/.tmp/"
|
|
CleanerContainerName = "kube-cleaner"
|
|
LogCleanerContainerName = "rke-log-cleaner"
|
|
RKELogsPath = "/var/lib/rancher/rke/log"
|
|
|
|
B2DOS = "Boot2Docker"
|
|
B2DPrefixPath = "/mnt/sda1/rke"
|
|
ROS = "RancherOS"
|
|
ROSPrefixPath = "/opt/rke"
|
|
CoreOS = "CoreOS"
|
|
CoreOSPrefixPath = "/opt/rke"
|
|
FlatcarOS = "Flatcar"
|
|
FlatcarOSPrefixPath = "/opt/rke"
|
|
WindowsOS = "Windows"
|
|
WindowsPrefixPath = "c:/"
|
|
)
|
|
|
|
func (h *Host) CleanUpAll(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry, externalEtcd bool) error {
|
|
log.Infof(ctx, "[hosts] Cleaning up host [%s]", h.Address)
|
|
toCleanPaths := []string{
|
|
path.Join(h.PrefixPath, ToCleanSSLDir),
|
|
ToCleanCNIConf,
|
|
ToCleanCNIBin,
|
|
ToCleanCalicoRun,
|
|
path.Join(h.PrefixPath, ToCleanTempCertPath),
|
|
path.Join(h.PrefixPath, ToCleanCNILib),
|
|
}
|
|
|
|
if !externalEtcd {
|
|
toCleanPaths = append(toCleanPaths, path.Join(h.PrefixPath, ToCleanEtcdDir))
|
|
}
|
|
return h.CleanUp(ctx, toCleanPaths, cleanerImage, prsMap)
|
|
}
|
|
|
|
func (h *Host) CleanUpWorkerHost(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error {
|
|
if h.IsControl || h.IsEtcd {
|
|
log.Infof(ctx, "[hosts] Host [%s] is already a controlplane or etcd host, skipping cleanup.", h.Address)
|
|
return nil
|
|
}
|
|
toCleanPaths := []string{
|
|
path.Join(h.PrefixPath, ToCleanSSLDir),
|
|
ToCleanCNIConf,
|
|
ToCleanCNIBin,
|
|
ToCleanCalicoRun,
|
|
path.Join(h.PrefixPath, ToCleanCNILib),
|
|
}
|
|
return h.CleanUp(ctx, toCleanPaths, cleanerImage, prsMap)
|
|
}
|
|
|
|
func (h *Host) CleanUpControlHost(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error {
|
|
if h.IsWorker || h.IsEtcd {
|
|
log.Infof(ctx, "[hosts] Host [%s] is already a worker or etcd host, skipping cleanup.", h.Address)
|
|
return nil
|
|
}
|
|
toCleanPaths := []string{
|
|
path.Join(h.PrefixPath, ToCleanSSLDir),
|
|
ToCleanCNIConf,
|
|
ToCleanCNIBin,
|
|
ToCleanCalicoRun,
|
|
path.Join(h.PrefixPath, ToCleanCNILib),
|
|
}
|
|
return h.CleanUp(ctx, toCleanPaths, cleanerImage, prsMap)
|
|
}
|
|
|
|
func (h *Host) CleanUpEtcdHost(ctx context.Context, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error {
|
|
toCleanPaths := []string{
|
|
path.Join(h.PrefixPath, ToCleanEtcdDir),
|
|
path.Join(h.PrefixPath, ToCleanSSLDir),
|
|
}
|
|
if h.IsWorker || h.IsControl {
|
|
log.Infof(ctx, "[hosts] Host [%s] is already a worker or control host, skipping cleanup certs.", h.Address)
|
|
toCleanPaths = []string{
|
|
path.Join(h.PrefixPath, ToCleanEtcdDir),
|
|
}
|
|
}
|
|
return h.CleanUp(ctx, toCleanPaths, cleanerImage, prsMap)
|
|
}
|
|
|
|
func (h *Host) CleanUp(ctx context.Context, toCleanPaths []string, cleanerImage string, prsMap map[string]v3.PrivateRegistry) error {
|
|
log.Infof(ctx, "[hosts] Cleaning up host [%s]", h.Address)
|
|
imageCfg, hostCfg := buildCleanerConfig(h, toCleanPaths, cleanerImage)
|
|
log.Infof(ctx, "[hosts] Running cleaner container on host [%s]", h.Address)
|
|
if err := docker.DoRunContainer(ctx, h.DClient, imageCfg, hostCfg, CleanerContainerName, h.Address, CleanerContainerName, prsMap); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := docker.WaitForContainer(ctx, h.DClient, h.Address, CleanerContainerName); err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Infof(ctx, "[hosts] Removing cleaner container on host [%s]", h.Address)
|
|
if err := docker.RemoveContainer(ctx, h.DClient, h.Address, CleanerContainerName); err != nil {
|
|
return err
|
|
}
|
|
log.Infof(ctx, "[hosts] Removing dead container logs on host [%s]", h.Address)
|
|
if err := DoRunLogCleaner(ctx, h, cleanerImage, prsMap); err != nil {
|
|
return err
|
|
}
|
|
log.Infof(ctx, "[hosts] Successfully cleaned up host [%s]", h.Address)
|
|
return nil
|
|
}
|
|
|
|
func (h *Host) OS() string {
|
|
return h.DockerInfo.OSType
|
|
}
|
|
|
|
func (h *Host) IsWindows() bool {
|
|
return h.DockerInfo.OSType == "windows"
|
|
}
|
|
|
|
func (h *Host) IsLinux() bool {
|
|
return h.DockerInfo.OSType == "linux"
|
|
}
|
|
|
|
func (h *Host) ProcessFilter(processes map[string]v3.Process) map[string]v3.Process {
|
|
if h.IsWindows() {
|
|
for name, process := range processes {
|
|
// doesn't support host network on windows
|
|
if process.NetworkMode == "host" {
|
|
process.NetworkMode = ""
|
|
}
|
|
|
|
// doesn't support PID on windows
|
|
if process.PidMode != "" {
|
|
process.PidMode = ""
|
|
}
|
|
|
|
// doesn't support privileged mode on windows
|
|
if process.Privileged {
|
|
process.Privileged = false
|
|
}
|
|
|
|
// doesn't execute health check
|
|
process.HealthCheck = v3.HealthCheck{}
|
|
|
|
processes[name] = process
|
|
}
|
|
}
|
|
|
|
return processes
|
|
}
|
|
|
|
func DeleteNode(ctx context.Context, toDeleteHost *Host, kubeClient *kubernetes.Clientset, hasAnotherRole bool, cloudProvider string) error {
|
|
if hasAnotherRole {
|
|
log.Infof(ctx, "[hosts] host [%s] has another role, skipping delete from kubernetes cluster", toDeleteHost.Address)
|
|
return nil
|
|
}
|
|
log.Infof(ctx, "[hosts] Cordoning host [%s]", toDeleteHost.Address)
|
|
if _, err := k8s.GetNode(kubeClient, toDeleteHost.HostnameOverride); err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
log.Warnf(ctx, "[hosts] Can't find node by name [%s]", toDeleteHost.Address)
|
|
return nil
|
|
}
|
|
return err
|
|
|
|
}
|
|
if err := k8s.CordonUncordon(kubeClient, toDeleteHost.HostnameOverride, true); err != nil {
|
|
return err
|
|
}
|
|
log.Infof(ctx, "[hosts] Deleting host [%s] from the cluster", toDeleteHost.Address)
|
|
if err := k8s.DeleteNode(kubeClient, toDeleteHost.HostnameOverride, cloudProvider); err != nil {
|
|
return err
|
|
}
|
|
log.Infof(ctx, "[hosts] Successfully deleted host [%s] from the cluster", toDeleteHost.Address)
|
|
return nil
|
|
}
|
|
|
|
func RemoveTaintFromHost(ctx context.Context, host *Host, taintKey string, kubeClient *kubernetes.Clientset) error {
|
|
log.Infof(ctx, "[hosts] removing taint [%s] from host [%s]", taintKey, host.Address)
|
|
if err := k8s.RemoveTaintFromNodeByKey(kubeClient, host.HostnameOverride, taintKey); err != nil {
|
|
return err
|
|
}
|
|
log.Infof(ctx, "[hosts] Successfully deleted taint [%s] from host [%s]", taintKey, host.Address)
|
|
return nil
|
|
}
|
|
|
|
func GetToDeleteHosts(currentHosts, configHosts, inactiveHosts []*Host, includeInactive bool) []*Host {
|
|
toDeleteHosts := []*Host{}
|
|
for _, currentHost := range currentHosts {
|
|
found := false
|
|
for _, newHost := range configHosts {
|
|
if currentHost.Address == newHost.Address {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
inactive := false
|
|
for _, inactiveHost := range inactiveHosts {
|
|
if inactiveHost.Address == currentHost.Address {
|
|
inactive = true
|
|
break
|
|
}
|
|
}
|
|
if (inactive && includeInactive) || !inactive {
|
|
toDeleteHosts = append(toDeleteHosts, currentHost)
|
|
}
|
|
}
|
|
}
|
|
return toDeleteHosts
|
|
}
|
|
|
|
func GetToAddHosts(currentHosts, configHosts []*Host) []*Host {
|
|
toAddHosts := []*Host{}
|
|
for _, configHost := range configHosts {
|
|
found := false
|
|
for _, currentHost := range currentHosts {
|
|
if currentHost.Address == configHost.Address {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
toAddHosts = append(toAddHosts, configHost)
|
|
}
|
|
}
|
|
return toAddHosts
|
|
}
|
|
|
|
func IsHostListChanged(currentHosts, configHosts []*Host) bool {
|
|
changed := false
|
|
for _, host := range currentHosts {
|
|
found := false
|
|
for _, configHost := range configHosts {
|
|
if host.Address == configHost.Address {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return true
|
|
}
|
|
}
|
|
for _, host := range configHosts {
|
|
found := false
|
|
for _, currentHost := range currentHosts {
|
|
if host.Address == currentHost.Address {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return true
|
|
}
|
|
}
|
|
return changed
|
|
}
|
|
|
|
func buildCleanerConfig(host *Host, toCleanDirs []string, cleanerImage string) (*container.Config, *container.HostConfig) {
|
|
cmd := []string{
|
|
"sh",
|
|
"-c",
|
|
fmt.Sprintf("find %s -mindepth 1 -delete", strings.Join(toCleanDirs, " ")),
|
|
}
|
|
imageCfg := &container.Config{
|
|
Image: cleanerImage,
|
|
Cmd: cmd,
|
|
}
|
|
bindMounts := []string{}
|
|
for _, vol := range toCleanDirs {
|
|
bindMounts = append(bindMounts, fmt.Sprintf("%s:%s:z", vol, vol))
|
|
}
|
|
hostCfg := &container.HostConfig{
|
|
Binds: bindMounts,
|
|
}
|
|
return imageCfg, hostCfg
|
|
}
|
|
|
|
func NodesToHosts(rkeNodes []v3.RKEConfigNode, nodeRole string) []*Host {
|
|
hostList := make([]*Host, 0)
|
|
// Return all nodes if there is no noderole passed to the function
|
|
if nodeRole == "" {
|
|
for _, node := range rkeNodes {
|
|
newHost := Host{
|
|
RKEConfigNode: node,
|
|
}
|
|
hostList = append(hostList, &newHost)
|
|
}
|
|
return hostList
|
|
}
|
|
for _, node := range rkeNodes {
|
|
for _, role := range node.Role {
|
|
if role == nodeRole {
|
|
newHost := Host{
|
|
RKEConfigNode: node,
|
|
}
|
|
hostList = append(hostList, &newHost)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return hostList
|
|
}
|
|
|
|
func GetUniqueHostList(etcdHosts, cpHosts, workerHosts []*Host) []*Host {
|
|
hostList := []*Host{}
|
|
hostList = append(hostList, etcdHosts...)
|
|
hostList = append(hostList, cpHosts...)
|
|
hostList = append(hostList, workerHosts...)
|
|
// little trick to get a unique host list
|
|
uniqHostMap := make(map[*Host]bool)
|
|
for _, host := range hostList {
|
|
uniqHostMap[host] = true
|
|
}
|
|
uniqHostList := []*Host{}
|
|
for host := range uniqHostMap {
|
|
uniqHostList = append(uniqHostList, host)
|
|
}
|
|
return uniqHostList
|
|
}
|
|
|
|
func (h *Host) SetPrefixPath(clusterPrefixPath string) {
|
|
var prefixPath string
|
|
switch {
|
|
case clusterPrefixPath != "/":
|
|
prefixPath = clusterPrefixPath
|
|
case strings.Contains(h.DockerInfo.OperatingSystem, B2DOS):
|
|
prefixPath = B2DPrefixPath
|
|
case strings.Contains(h.DockerInfo.OperatingSystem, ROS):
|
|
prefixPath = ROSPrefixPath
|
|
case strings.Contains(h.DockerInfo.OperatingSystem, CoreOS):
|
|
prefixPath = CoreOSPrefixPath
|
|
case strings.Contains(h.DockerInfo.OperatingSystem, FlatcarOS):
|
|
prefixPath = FlatcarOSPrefixPath
|
|
case strings.Contains(h.DockerInfo.OperatingSystem, WindowsOS):
|
|
prefixPath = WindowsPrefixPath
|
|
default:
|
|
prefixPath = clusterPrefixPath
|
|
}
|
|
|
|
h.PrefixPath = prefixPath
|
|
}
|
|
|
|
func (h *Host) GetExtraBinds(service v3.BaseService) []string {
|
|
switch {
|
|
case h.OS() == "windows" && len(service.WindowsExtraBinds) > 0:
|
|
return service.WindowsExtraBinds
|
|
default:
|
|
return service.ExtraBinds
|
|
}
|
|
}
|
|
|
|
func (h *Host) GetExtraEnv(service v3.BaseService) []string {
|
|
switch {
|
|
case h.OS() == "windows" && len(service.WindowsExtraEnv) > 0:
|
|
return service.WindowsExtraEnv
|
|
default:
|
|
return service.ExtraEnv
|
|
}
|
|
}
|
|
func (h *Host) GetExtraArgs(service v3.BaseService) map[string]string {
|
|
switch {
|
|
case h.OS() == "windows" && len(service.WindowsExtraArgs) > 0:
|
|
return service.WindowsExtraArgs
|
|
default:
|
|
return service.ExtraArgs
|
|
}
|
|
}
|
|
|
|
func DoRunLogCleaner(ctx context.Context, host *Host, alpineImage string, prsMap map[string]v3.PrivateRegistry) error {
|
|
logrus.Debugf("[cleanup] Starting log link cleanup on host [%s]", host.Address)
|
|
imageCfg := &container.Config{
|
|
Image: alpineImage,
|
|
Tty: true,
|
|
Cmd: []string{
|
|
"sh",
|
|
"-c",
|
|
fmt.Sprintf("find %s -type l ! -exec test -e {} \\; -print -delete", RKELogsPath),
|
|
},
|
|
}
|
|
hostCfg := &container.HostConfig{
|
|
Binds: []string{
|
|
host.DockerInfo.DockerRootDir + ":" + host.DockerInfo.DockerRootDir,
|
|
"/var/lib:/var/lib",
|
|
},
|
|
Privileged: true,
|
|
}
|
|
if err := docker.DoRemoveContainer(ctx, host.DClient, LogCleanerContainerName, host.Address); err != nil {
|
|
return err
|
|
}
|
|
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, LogCleanerContainerName, host.Address, "cleanup", prsMap); err != nil {
|
|
return err
|
|
}
|
|
if err := docker.DoRemoveContainer(ctx, host.DClient, LogCleanerContainerName, host.Address); err != nil {
|
|
return err
|
|
}
|
|
logrus.Debugf("[cleanup] Successfully cleaned up log links on host [%s]", host.Address)
|
|
return nil
|
|
}
|
|
|
|
func IsNodeInList(host *Host, hostList []*Host) bool {
|
|
for _, h := range hostList {
|
|
if h.HostnameOverride == host.HostnameOverride {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func GetHostListIntersect(a []*Host, b []*Host) []*Host {
|
|
s := []*Host{}
|
|
hash := map[string]*Host{}
|
|
for _, h := range a {
|
|
hash[h.Address] = h
|
|
}
|
|
for _, h := range b {
|
|
if _, ok := hash[h.Address]; ok {
|
|
s = append(s, h)
|
|
}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func GetInternalAddressForHosts(hostList []*Host) []string {
|
|
hostAddresses := []string{}
|
|
for _, host := range hostList {
|
|
hostAddresses = append(hostAddresses, host.InternalAddress)
|
|
}
|
|
return hostAddresses
|
|
}
|
|
|
|
func IsDockerSELinuxEnabled(host *Host) bool {
|
|
for _, securityOpt := range host.DockerInfo.SecurityOptions {
|
|
logrus.Tracef("IsDockerSELinuxEnabled: securityOpt found: [%s]", securityOpt)
|
|
// name=selinux was the value returned after removing statically set Docker API version 1.24
|
|
if securityOpt == "selinux" || securityOpt == "name=selinux" {
|
|
logrus.Debugf("Host [%s] has SELinux enabled in Docker", host.Address)
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func IsEnterpriseLinuxHost(host *Host) bool {
|
|
operatingSystem := strings.ToLower(host.DockerInfo.OperatingSystem)
|
|
if strings.Contains(operatingSystem, "centos") || strings.Contains(operatingSystem, "enterprise linux") || strings.Contains(operatingSystem, "oracle linux") {
|
|
logrus.Debugf("Host [%s] with OperatingSystem [%s] is Enterprise Linux", host.Address, operatingSystem)
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func IsEnterpriseLinuxDocker(host *Host) bool {
|
|
dockerInitBinary := host.DockerInfo.InitBinary
|
|
// Init binary for Enterprise Linux Docker (not upstream) is /usr/libexec/docker/docker-init-current
|
|
// Init binary for upstream Docker is docker-init
|
|
if strings.EqualFold(dockerInitBinary, "/usr/libexec/docker/docker-init-current") {
|
|
return true
|
|
}
|
|
return false
|
|
}
|