rancher/rke
1
0
Fork 0
mirror of https://github.com/rancher/rke.git synced 2025-06-28 08:18:58 +00:00
Code Issues Releases Wiki Activity

Merge pull request #1936 from mrajashree/bugfixes

Browse Source 
Check role before including host in inactive host list
This commit is contained in:
Rajashree Mandaogane 2020-03-02 02:38:22 -08:00 committed by GitHub
commit 265553beb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 58 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
cluster/cluster.go
View File

@ -138,16 +138,12 @@ func (c *Cluster) DeployControlPlane(ctx context.Context, svcOptionData map[stri
// Deploy Control plane // Deploy Control plane
cpNodePlanMap := make(map[string]v3.RKEConfigNodePlan) cpNodePlanMap := make(map[string]v3.RKEConfigNodePlan)
// Build cp node plan map // Build cp node plan map
var notReadyHosts []*hosts.Host
for _, cpHost := range c.ControlPlaneHosts { for _, cpHost := range c.ControlPlaneHosts {
svcOptions, err := c.GetKubernetesServicesOptions(cpHost.DockerInfo.OSType, svcOptionData) svcOptions, err := c.GetKubernetesServicesOptions(cpHost.DockerInfo.OSType, svcOptionData)
if err != nil { if err != nil {
return "", err return "", err
} }
cpNodePlanMap[cpHost.Address] = BuildRKEConfigNodePlan(ctx, c, cpHost, cpHost.DockerInfo, svcOptions) cpNodePlanMap[cpHost.Address] = BuildRKEConfigNodePlan(ctx, c, cpHost, cpHost.DockerInfo, svcOptions)
if err := services.CheckNodeReady(kubeClient, cpHost, services.ControlRole); err != nil {
notReadyHosts = append(notReadyHosts, cpHost)
}
} }
if !reconcileCluster { if !reconcileCluster {
@ -162,16 +158,17 @@ func (c *Cluster) DeployControlPlane(ctx context.Context, svcOptionData map[stri
} }
return "", nil return "", nil
} }
return c.UpgradeControlPlane(ctx, kubeClient, cpNodePlanMap, notReadyHosts) return c.UpgradeControlPlane(ctx, kubeClient, cpNodePlanMap)
} }
func (c *Cluster) UpgradeControlPlane(ctx context.Context, kubeClient *kubernetes.Clientset, cpNodePlanMap map[string]v3.RKEConfigNodePlan, notReadyHosts []*hosts.Host) (string, error) { func (c *Cluster) UpgradeControlPlane(ctx context.Context, kubeClient *kubernetes.Clientset, cpNodePlanMap map[string]v3.RKEConfigNodePlan) (string, error) {
inactiveHosts := make(map[string]bool) inactiveHosts := make(map[string]bool)
var controlPlaneHosts []*hosts.Host var controlPlaneHosts, notReadyHosts []*hosts.Host
var notReadyHostNames []string var notReadyHostNames []string
for _, host := range c.InactiveHosts { for _, host := range c.InactiveHosts {
if !c.HostsLabeledToIgnoreUpgrade[host.Address] { // include only hosts with controlplane role
if host.IsControl && !c.HostsLabeledToIgnoreUpgrade[host.Address] {
inactiveHosts[host.HostnameOverride] = true inactiveHosts[host.HostnameOverride] = true
} }
} }
@ -179,23 +176,30 @@ func (c *Cluster) UpgradeControlPlane(ctx context.Context, kubeClient *kubernete
if !c.HostsLabeledToIgnoreUpgrade[host.Address] { if !c.HostsLabeledToIgnoreUpgrade[host.Address] {
controlPlaneHosts = append(controlPlaneHosts, host) controlPlaneHosts = append(controlPlaneHosts, host)
} }
if c.NewHosts[host.HostnameOverride] {
continue
}
// find existing nodes that are in NotReady state
if err := services.CheckNodeReady(kubeClient, host, services.ControlRole); err != nil {
notReadyHosts = append(notReadyHosts, host)
notReadyHostNames = append(notReadyHostNames, host.HostnameOverride)
}
} }
for _, host := range notReadyHosts { if len(notReadyHostNames) > 0 {
notReadyHostNames = append(notReadyHostNames, host.HostnameOverride) // attempt upgrade on NotReady hosts without respecting max_unavailable_controlplane
} logrus.Infof("Attempting upgrade of controlplane components on following hosts in NotReady status: %v", strings.Join(notReadyHostNames, ","))
// attempt upgrade on NotReady hosts without respecting maxUnavailable services.RunControlPlane(ctx, notReadyHosts,
logrus.Infof("Attempting upgrade of controlplane components on following hosts in NotReady status: %v", strings.Join(notReadyHostNames, ",")) c.LocalConnDialerFactory,
c.PrivateRegistriesMap,
services.RunControlPlane(ctx, notReadyHosts, cpNodePlanMap,
c.LocalConnDialerFactory, c.UpdateWorkersOnly,
c.PrivateRegistriesMap, c.SystemImages.Alpine,
cpNodePlanMap, c.Certificates)
c.UpdateWorkersOnly, // Calling CheckNodeReady wil give some time for nodes to get in Ready state
c.SystemImages.Alpine, for _, host := range notReadyHosts {
c.Certificates) services.CheckNodeReady(kubeClient, host, services.ControlRole)
for _, host := range notReadyHosts { }
services.CheckNodeReady(kubeClient, host, services.ControlRole)
} }
// rolling upgrade respecting maxUnavailable // rolling upgrade respecting maxUnavailable
errMsgMaxUnavailableNotFailed, err := services.UpgradeControlPlaneNodes(ctx, kubeClient, controlPlaneHosts, errMsgMaxUnavailableNotFailed, err := services.UpgradeControlPlaneNodes(ctx, kubeClient, controlPlaneHosts,
@ -220,7 +224,6 @@ func (c *Cluster) DeployWorkerPlane(ctx context.Context, svcOptionData map[strin
// Deploy Worker plane // Deploy Worker plane
workerNodePlanMap := make(map[string]v3.RKEConfigNodePlan) workerNodePlanMap := make(map[string]v3.RKEConfigNodePlan)
// Build cp node plan map // Build cp node plan map
var notReadyHosts []*hosts.Host
allHosts := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) allHosts := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts)
for _, host := range allHosts { for _, host := range allHosts {
svcOptions, err := c.GetKubernetesServicesOptions(host.DockerInfo.OSType, svcOptionData) svcOptions, err := c.GetKubernetesServicesOptions(host.DockerInfo.OSType, svcOptionData)
@ -231,9 +234,6 @@ func (c *Cluster) DeployWorkerPlane(ctx context.Context, svcOptionData map[strin
if host.IsControl || c.HostsLabeledToIgnoreUpgrade[host.Address] { if host.IsControl || c.HostsLabeledToIgnoreUpgrade[host.Address] {
continue continue
} }
if err := services.CheckNodeReady(kubeClient, host, services.WorkerRole); err != nil {
notReadyHosts = append(notReadyHosts, host)
}
if !host.IsEtcd { if !host.IsEtcd {
// separating hosts with only worker role so they undergo upgrade in maxUnavailable batches // separating hosts with only worker role so they undergo upgrade in maxUnavailable batches
workerOnlyHosts = append(workerOnlyHosts, host) workerOnlyHosts = append(workerOnlyHosts, host)
@ -257,32 +257,46 @@ func (c *Cluster) DeployWorkerPlane(ctx context.Context, svcOptionData map[strin
return "", nil return "", nil
} }
return c.UpgradeWorkerPlane(ctx, kubeClient, workerNodePlanMap, notReadyHosts, etcdAndWorkerHosts, workerOnlyHosts) return c.UpgradeWorkerPlane(ctx, kubeClient, workerNodePlanMap, etcdAndWorkerHosts, workerOnlyHosts)
} }
func (c *Cluster) UpgradeWorkerPlane(ctx context.Context, kubeClient *kubernetes.Clientset, workerNodePlanMap map[string]v3.RKEConfigNodePlan, notReadyHosts, etcdAndWorkerHosts, workerOnlyHosts []*hosts.Host) (string, error) { func (c *Cluster) UpgradeWorkerPlane(ctx context.Context, kubeClient *kubernetes.Clientset, workerNodePlanMap map[string]v3.RKEConfigNodePlan, etcdAndWorkerHosts, workerOnlyHosts []*hosts.Host) (string, error) {
inactiveHosts := make(map[string]bool) inactiveHosts := make(map[string]bool)
var notReadyHosts []*hosts.Host
var notReadyHostNames []string var notReadyHostNames []string
for _, host := range c.InactiveHosts { for _, host := range c.InactiveHosts {
if !c.HostsLabeledToIgnoreUpgrade[host.Address] { // if host has controlplane role, it already has worker components upgraded
if !host.IsControl && !c.HostsLabeledToIgnoreUpgrade[host.Address] {
inactiveHosts[host.HostnameOverride] = true inactiveHosts[host.HostnameOverride] = true
} }
} }
for _, host := range notReadyHosts { for _, host := range append(etcdAndWorkerHosts, workerOnlyHosts...) {
notReadyHostNames = append(notReadyHostNames, host.HostnameOverride) if c.NewHosts[host.HostnameOverride] {
continue
}
// find existing nodes that are in NotReady state
if err := services.CheckNodeReady(kubeClient, host, services.WorkerRole); err != nil {
notReadyHosts = append(notReadyHosts, host)
notReadyHostNames = append(notReadyHostNames, host.HostnameOverride)
}
} }
// attempt upgrade on NotReady hosts without respecting maxUnavailable if len(notReadyHostNames) > 0 {
logrus.Infof("Attempting upgrade of worker components on following hosts in NotReady status: %v", strings.Join(notReadyHostNames, ",")) // attempt upgrade on NotReady hosts without respecting max_unavailable_worker
services.RunWorkerPlane(ctx, notReadyHosts, logrus.Infof("Attempting upgrade of worker components on following hosts in NotReady status: %v", strings.Join(notReadyHostNames, ","))
c.LocalConnDialerFactory, services.RunWorkerPlane(ctx, notReadyHosts,
c.PrivateRegistriesMap, c.LocalConnDialerFactory,
workerNodePlanMap, c.PrivateRegistriesMap,
c.Certificates, workerNodePlanMap,
c.UpdateWorkersOnly, c.Certificates,
c.SystemImages.Alpine) c.UpdateWorkersOnly,
for _, host := range notReadyHosts { c.SystemImages.Alpine)
services.CheckNodeReady(kubeClient, host, services.WorkerRole) // Calling CheckNodeReady wil give some time for nodes to get in Ready state
for _, host := range notReadyHosts {
services.CheckNodeReady(kubeClient, host, services.WorkerRole)
}
} }
errMsgMaxUnavailableNotFailed, err := services.UpgradeWorkerPlaneForWorkerAndEtcdNodes(ctx, kubeClient, etcdAndWorkerHosts, workerOnlyHosts, inactiveHosts, errMsgMaxUnavailableNotFailed, err := services.UpgradeWorkerPlaneForWorkerAndEtcdNodes(ctx, kubeClient, etcdAndWorkerHosts, workerOnlyHosts, inactiveHosts,
c.LocalConnDialerFactory, c.LocalConnDialerFactory,
c.PrivateRegistriesMap, c.PrivateRegistriesMap,

2
cmd/up.go
View File

@ -169,7 +169,7 @@ func ClusterUp(ctx context.Context, dialersOptions hosts.DialersOptions, flags c
return APIURL, caCrt, clientCert, clientKey, nil, err return APIURL, caCrt, clientCert, clientKey, nil, err
} }
logrus.Infof("Setting maxUnavailable for worker nodes to: %v", maxUnavailableWorker) logrus.Infof("Setting maxUnavailable for worker nodes to: %v", maxUnavailableWorker)
logrus.Infof("Setting maxUnavailable for control nodes to: %v", maxUnavailableControl) logrus.Infof("Setting maxUnavailable for controlplane nodes to: %v", maxUnavailableControl)
kubeCluster.MaxUnavailableForWorkerNodes, kubeCluster.MaxUnavailableForControlNodes = maxUnavailableWorker, maxUnavailableControl kubeCluster.MaxUnavailableForWorkerNodes, kubeCluster.MaxUnavailableForControlNodes = maxUnavailableWorker, maxUnavailableControl
} }