diff --git a/cluster/cluster.go b/cluster/cluster.go index cab32aea..3baec462 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -41,6 +41,7 @@ type Cluster struct { PrivateRegistriesMap map[string]v3.PrivateRegistry K8sWrapTransport k8s.WrapTransport UseKubectlDeploy bool + UpdateWorkersOnly bool } const ( @@ -61,7 +62,7 @@ func (c *Cluster) DeployControlPlane(ctx context.Context) error { if len(c.Services.Etcd.ExternalURLs) > 0 { log.Infof(ctx, "[etcd] External etcd connection string has been specified, skipping etcd plane") } else { - if err := services.RunEtcdPlane(ctx, c.EtcdHosts, etcdProcessHostMap, c.LocalConnDialerFactory, c.PrivateRegistriesMap); err != nil { + if err := services.RunEtcdPlane(ctx, c.EtcdHosts, etcdProcessHostMap, c.LocalConnDialerFactory, c.PrivateRegistriesMap, c.UpdateWorkersOnly); err != nil { return fmt.Errorf("[etcd] Failed to bring up Etcd Plane: %v", err) } } @@ -76,7 +77,7 @@ func (c *Cluster) DeployControlPlane(ctx context.Context) error { if err := services.RunControlPlane(ctx, c.ControlPlaneHosts, c.LocalConnDialerFactory, c.PrivateRegistriesMap, - processMap); err != nil { + processMap, c.UpdateWorkersOnly); err != nil { return fmt.Errorf("[controlPlane] Failed to bring up Control Plane: %v", err) } @@ -101,6 +102,7 @@ func (c *Cluster) DeployWorkerPlane(ctx context.Context) error { processMap, kubeletProcessHostMap, c.Certificates, + c.UpdateWorkersOnly, ); err != nil { return fmt.Errorf("[workerPlane] Failed to bring up Worker Plane: %v", err) } diff --git a/cluster/network.go b/cluster/network.go index 19ae0a9c..969b728d 100644 --- a/cluster/network.go +++ b/cluster/network.go @@ -371,7 +371,7 @@ func (c *Cluster) runServicePortChecks(ctx context.Context) error { } } // check all -> etcd connectivity - log.Infof(ctx, "[network] Running all -> etcd port checks") + log.Infof(ctx, "[network] Running control plane -> etcd port checks") for _, host := range c.ControlPlaneHosts { runHost := host errgrp.Go(func() error { @@ -381,18 +381,8 @@ func (c *Cluster) runServicePortChecks(ctx context.Context) error { if err := errgrp.Wait(); err != nil { return err } - // Workers need to talk to etcd for calico - for _, host := range c.WorkerHosts { - runHost := host - errgrp.Go(func() error { - return checkPlaneTCPPortsFromHost(ctx, runHost, EtcdPortList, c.EtcdHosts, c.SystemImages.Alpine, c.PrivateRegistriesMap) - }) - } - if err := errgrp.Wait(); err != nil { - return err - } // check controle plane -> Workers - log.Infof(ctx, "[network] Running control plane -> etcd port checks") + log.Infof(ctx, "[network] Running control plane -> worker port checks") for _, host := range c.ControlPlaneHosts { runHost := host errgrp.Go(func() error { diff --git a/cluster/reconcile.go b/cluster/reconcile.go index 10ac6d62..eead3ddd 100644 --- a/cluster/reconcile.go +++ b/cluster/reconcile.go @@ -19,11 +19,12 @@ const ( unschedulableEtcdTaint = "node-role.kubernetes.io/etcd=true:NoExecute" ) -func ReconcileCluster(ctx context.Context, kubeCluster, currentCluster *Cluster) error { +func ReconcileCluster(ctx context.Context, kubeCluster, currentCluster *Cluster, updateOnly bool) error { log.Infof(ctx, "[reconcile] Reconciling cluster state") + kubeCluster.UpdateWorkersOnly = updateOnly if currentCluster == nil { log.Infof(ctx, "[reconcile] This is newly generated cluster") - + kubeCluster.UpdateWorkersOnly = false return nil } @@ -67,6 +68,7 @@ func reconcileWorker(ctx context.Context, currentCluster, kubeCluster *Cluster, // attempt to remove unschedulable taint toAddHosts := hosts.GetToAddHosts(currentCluster.WorkerHosts, kubeCluster.WorkerHosts) for _, host := range toAddHosts { + host.UpdateWorker = true if host.IsEtcd { host.ToDelTaints = append(host.ToDelTaints, unschedulableEtcdTaint) } @@ -110,6 +112,7 @@ func reconcileControl(ctx context.Context, currentCluster, kubeCluster *Cluster, // attempt to remove unschedulable taint toAddHosts := hosts.GetToAddHosts(currentCluster.ControlPlaneHosts, kubeCluster.ControlPlaneHosts) for _, host := range toAddHosts { + kubeCluster.UpdateWorkersOnly = false if host.IsEtcd { host.ToDelTaints = append(host.ToDelTaints, unschedulableEtcdTaint) } @@ -173,6 +176,7 @@ func reconcileEtcd(ctx context.Context, currentCluster, kubeCluster *Cluster, ku crtMap := currentCluster.Certificates var err error for _, etcdHost := range etcdToAdd { + kubeCluster.UpdateWorkersOnly = false etcdHost.ToAddEtcdMember = true // Generate new certificate for the new etcd member crtMap, err = pki.RegenerateEtcdCertificate( diff --git a/cmd/up.go b/cmd/up.go index e2dcec5b..a0736479 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -28,6 +28,14 @@ func UpCommand() cli.Command { Name: "local", Usage: "Deploy Kubernetes cluster locally", }, + cli.BoolFlag{ + Name: "update-only", + Usage: "Skip idempotent deployment of control and etcd plane", + }, + cli.BoolFlag{ + Name: "disable-port-check", + Usage: "Disable port check validation between nodes", + }, } upFlags = append(upFlags, sshCliOptions...) @@ -45,7 +53,7 @@ func ClusterUp( rkeConfig *v3.RancherKubernetesEngineConfig, dockerDialerFactory, localConnDialerFactory hosts.DialerFactory, k8sWrapTransport k8s.WrapTransport, - local bool, configDir string) (string, string, string, string, error) { + local bool, configDir string, updateOnly, disablePortCheck bool) (string, string, string, string, error) { log.Infof(ctx, "Building Kubernetes cluster") var APIURL, caCrt, clientCert, clientKey string @@ -63,9 +71,10 @@ func ClusterUp( if err != nil { return APIURL, caCrt, clientCert, clientKey, err } - - if err = kubeCluster.CheckClusterPorts(ctx, currentCluster); err != nil { - return APIURL, caCrt, clientCert, clientKey, err + if !disablePortCheck { + if err = kubeCluster.CheckClusterPorts(ctx, currentCluster); err != nil { + return APIURL, caCrt, clientCert, clientKey, err + } } err = cluster.SetUpAuthentication(ctx, kubeCluster, currentCluster) @@ -73,7 +82,7 @@ func ClusterUp( return APIURL, caCrt, clientCert, clientKey, err } - err = cluster.ReconcileCluster(ctx, kubeCluster, currentCluster) + err = cluster.ReconcileCluster(ctx, kubeCluster, currentCluster, updateOnly) if err != nil { return APIURL, caCrt, clientCert, clientKey, err } @@ -147,8 +156,10 @@ func clusterUpFromCli(ctx *cli.Context) error { if err != nil { return err } + updateOnly := ctx.Bool("update-only") + disablePortCheck := ctx.Bool("disable-port-check") - _, _, _, _, err = ClusterUp(context.Background(), rkeConfig, nil, nil, nil, false, "") + _, _, _, _, err = ClusterUp(context.Background(), rkeConfig, nil, nil, nil, false, "", updateOnly, disablePortCheck) return err } @@ -166,6 +177,6 @@ func clusterUpLocal(ctx *cli.Context) error { } rkeConfig.Nodes = []v3.RKEConfigNode{*cluster.GetLocalRKENodeConfig()} } - _, _, _, _, err = ClusterUp(context.Background(), rkeConfig, nil, hosts.LocalHealthcheckFactory, nil, true, "") + _, _, _, _, err = ClusterUp(context.Background(), rkeConfig, nil, hosts.LocalHealthcheckFactory, nil, true, "", false, false) return err } diff --git a/hosts/hosts.go b/hosts/hosts.go index 76c23a41..fcb9da5e 100644 --- a/hosts/hosts.go +++ b/hosts/hosts.go @@ -32,6 +32,7 @@ type Host struct { ToAddTaints []string ToDelTaints []string DockerInfo types.Info + UpdateWorker bool } const ( diff --git a/services/controlplane.go b/services/controlplane.go index b2fb36ff..7abe80ca 100644 --- a/services/controlplane.go +++ b/services/controlplane.go @@ -9,11 +9,14 @@ import ( "golang.org/x/sync/errgroup" ) -func RunControlPlane(ctx context.Context, controlHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process) error { +func RunControlPlane(ctx context.Context, controlHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, updateWorkersOnly bool) error { log.Infof(ctx, "[%s] Building up Controller Plane..", ControlRole) var errgrp errgroup.Group for _, host := range controlHosts { runHost := host + if updateWorkersOnly { + continue + } errgrp.Go(func() error { return doDeployControlHost(ctx, runHost, localConnDialerFactory, prsMap, processMap) }) diff --git a/services/etcd.go b/services/etcd.go index 57bc0520..f45170f7 100644 --- a/services/etcd.go +++ b/services/etcd.go @@ -20,9 +20,12 @@ const ( EtcdHealthCheckURL = "https://127.0.0.1:2379/health" ) -func RunEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, etcdProcessHostMap map[*hosts.Host]v3.Process, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry) error { +func RunEtcdPlane(ctx context.Context, etcdHosts []*hosts.Host, etcdProcessHostMap map[*hosts.Host]v3.Process, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, updateWorkersOnly bool) error { log.Infof(ctx, "[%s] Building up Etcd Plane..", ETCDRole) for _, host := range etcdHosts { + if updateWorkersOnly { + continue + } imageCfg, hostCfg, _ := GetProcessConfig(etcdProcessHostMap[host]) err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, EtcdContainerName, host.Address, ETCDRole, prsMap) if err != nil { diff --git a/services/workerplane.go b/services/workerplane.go index 4c354e69..b85ab11b 100644 --- a/services/workerplane.go +++ b/services/workerplane.go @@ -14,10 +14,15 @@ const ( unschedulableEtcdTaint = "node-role.kubernetes.io/etcd=true:NoExecute" ) -func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, kubeletProcessHostMap map[*hosts.Host]v3.Process, certMap map[string]pki.CertificatePKI) error { +func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, kubeletProcessHostMap map[*hosts.Host]v3.Process, certMap map[string]pki.CertificatePKI, updateWorkersOnly bool) error { log.Infof(ctx, "[%s] Building up Worker Plane..", WorkerRole) var errgrp errgroup.Group for _, host := range allHosts { + if updateWorkersOnly { + if !host.UpdateWorker { + continue + } + } if !host.IsControl && !host.IsWorker { // Add unschedulable taint host.ToAddTaints = append(host.ToAddTaints, unschedulableEtcdTaint)