Add util command

cluster/cluster.go

@@ -790,15 +790,18 @@ func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error {
 	if len(kubeCluster.ControlPlaneHosts) == 0 {
 		return nil
 	}
+	var activeControlPlaneHostFound bool
 	log.Infof(ctx, "[reconcile] Rebuilding and updating local kube config")
 	var workingConfig, newConfig string
 	currentKubeConfig := kubeCluster.Certificates[pki.KubeAdminCertName]
 	caCrt := kubeCluster.Certificates[pki.CACertName].Certificate
 	for _, cpHost := range kubeCluster.ControlPlaneHosts {
 		if (currentKubeConfig == pki.CertificatePKI{}) {
+			log.Debugf(ctx, "[reconcile] Rebuilding and updating local kube config, creating new address")
 			kubeCluster.Certificates = make(map[string]pki.CertificatePKI)
 			newConfig = getLocalAdminConfigWithNewAddress(kubeCluster.LocalKubeConfigPath, cpHost.Address, kubeCluster.ClusterName)
 		} else {
+			log.Debugf(ctx, "[reconcile] Rebuilding and updating local kube config, creating new kubeconfig")
 			kubeURL := fmt.Sprintf("https://%s:6443", cpHost.Address)
 			caData := string(cert.EncodeCertPEM(caCrt))
 			crtData := string(cert.EncodeCertPEM(currentKubeConfig.Certificate))
@@ -810,9 +813,14 @@ func rebuildLocalAdminConfig(ctx context.Context, kubeCluster *Cluster) error {
 		}
 		workingConfig = newConfig
 		if _, err := GetK8sVersion(kubeCluster.LocalKubeConfigPath, kubeCluster.K8sWrapTransport); err == nil {
-			log.Infof(ctx, "[reconcile] host [%s] is active master on the cluster", cpHost.Address)
+			log.Infof(ctx, "[reconcile] host [%s] is a control plane node with reachable Kubernetes API endpoint in the cluster", cpHost.Address)
+			activeControlPlaneHostFound = true
 			break
 		}
+		log.Warnf(ctx, "[reconcile] host [%s] is a control plane node without reachable Kubernetes API endpoint in the cluster", cpHost.Address)
+	}
+	if !activeControlPlaneHostFound {
+		log.Warnf(ctx, "[reconcile] no control plane node with reachable Kubernetes API endpoint in the cluster found")
 	}
 	currentKubeConfig.Config = workingConfig
 	kubeCluster.Certificates[pki.KubeAdminCertName] = currentKubeConfig

cluster/state.go

@@ -17,6 +17,7 @@ import (
 	"github.com/rancher/rke/k8s"
 	"github.com/rancher/rke/log"
 	"github.com/rancher/rke/pki"
+	"github.com/rancher/rke/services"
 	v3 "github.com/rancher/rke/types"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
@@ -68,6 +69,19 @@ func (c *Cluster) GetClusterState(ctx context.Context, fullState *FullState) (*C
 	return currentCluster, nil
 }
 
+func (c *Cluster) GetStateFileFromConfigMap(ctx context.Context) (string, error) {
+	kubeletImage := c.Services.Kubelet.Image
+	for _, host := range c.ControlPlaneHosts {
+		stateFile, err := services.RunGetStateFileFromConfigMap(ctx, host, c.PrivateRegistriesMap, kubeletImage)
+		if err != nil || stateFile == "" {
+			logrus.Infof("Could not get ConfigMap with cluster state from host [%s]", host.Address)
+			continue
+		}
+		return stateFile, nil
+	}
+	return "", fmt.Errorf("Unable to get ConfigMap with cluster state from any Control Plane host")
+}
+
 func SaveFullStateToKubernetes(ctx context.Context, kubeCluster *Cluster, fullState *FullState) error {
 	k8sClient, err := k8s.NewClient(kubeCluster.LocalKubeConfigPath, kubeCluster.K8sWrapTransport)
 	if err != nil {

cmd/util.go

@@ -0,0 +1,207 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/rancher/rke/cluster"
+	"github.com/rancher/rke/hosts"
+	"github.com/rancher/rke/k8s"
+	"github.com/rancher/rke/pki"
+	v3 "github.com/rancher/rke/types"
+	"github.com/rancher/rke/util"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func UtilCommand() cli.Command {
+	utilCfgFlags := []cli.Flag{
+		cli.StringFlag{
+			Name:   "config",
+			Usage:  "Specify an alternate cluster YAML file",
+			Value:  pki.ClusterConfig,
+			EnvVar: "RKE_CONFIG",
+		},
+	}
+	utilFlags := append(utilCfgFlags, commonFlags...)
+
+	return cli.Command{
+		Name:  "util",
+		Usage: "Various utilities to retrieve cluster related files and troubleshoot",
+		Subcommands: cli.Commands{
+			cli.Command{
+				Name:   "get-state-file",
+				Usage:  "Retrieve state file from cluster",
+				Action: getStateFile,
+				Flags:  utilFlags,
+			},
+			cli.Command{
+				Name:   "get-kubeconfig",
+				Usage:  "Retrieve kubeconfig file from cluster state",
+				Action: getKubeconfigFile,
+				Flags:  utilFlags,
+			},
+		},
+	}
+}
+
+func getKubeconfigFile(ctx *cli.Context) error {
+	logrus.Infof("Creating new kubeconfig file")
+	// Check if we can successfully connect to the cluster using the existing kubeconfig file
+	clusterFile, clusterFilePath, err := resolveClusterFile(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to resolve cluster file: %v", err)
+	}
+
+	// setting up the flags
+	flags := cluster.GetExternalFlags(false, false, false, false, "", clusterFilePath)
+	rkeConfig, err := cluster.ParseConfig(clusterFile)
+	if err != nil {
+		return fmt.Errorf("failed to parse cluster file: %v", err)
+	}
+
+	rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig)
+	if err != nil {
+		return err
+	}
+
+	clusterState, err := cluster.ReadStateFile(context.Background(), cluster.GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir))
+	if err != nil {
+		return err
+	}
+
+	// Creating temp cluster to check if snapshot archive contains state file and retrieve it
+	tempCluster, err := cluster.InitClusterObject(context.Background(), rkeConfig, flags, "")
+	if err != nil {
+		return err
+	}
+
+	// Move current kubeconfig file
+	err = util.CopyFileWithPrefix(tempCluster.LocalKubeConfigPath, "kube_config")
+	if err != nil {
+		return err
+	}
+	kubeCluster, _ := tempCluster.GetClusterState(context.Background(), clusterState)
+
+	if err := cluster.RebuildKubeconfig(context.Background(), kubeCluster); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getStateFile(ctx *cli.Context) error {
+	logrus.Infof("Retrieving state file from cluster")
+	// Check if we can successfully connect to the cluster using the existing kubeconfig file
+	localKubeConfig := pki.GetLocalKubeConfig(ctx.String("config"), "")
+	clusterFile, clusterFilePath, err := resolveClusterFile(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to resolve cluster file: %v", err)
+	}
+	// setting up the flags
+	flags := cluster.GetExternalFlags(false, false, false, false, "", clusterFilePath)
+
+	// not going to use a k8s dialer here.. this is a CLI command
+	serverVersion, err := cluster.GetK8sVersion(localKubeConfig, nil)
+	if err != nil {
+		logrus.Infof("Unable to connect to server using kubeconfig, trying to get state from Control Plane node(s), error: %v", err)
+		// We need to retrieve the state file using Docker on the node(s)
+
+		rkeConfig, err := cluster.ParseConfig(clusterFile)
+		if err != nil {
+			return fmt.Errorf("failed to parse cluster file: %v", err)
+		}
+
+		rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig)
+		if err != nil {
+			return err
+		}
+
+		_, _, _, _, _, err = RetrieveClusterStateConfigMap(context.Background(), rkeConfig, hosts.DialersOptions{}, flags, map[string]interface{}{})
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+	logrus.Infof("Successfully connected to server using kubeconfig, retrieved server version [%s]", serverVersion)
+	// Retrieve full-cluster-state configmap
+	k8sClient, err := k8s.NewClient(localKubeConfig, nil)
+	cfgMap, err := k8s.GetConfigMap(k8sClient, cluster.FullStateConfigMapName)
+	clusterData := cfgMap.Data[cluster.FullStateConfigMapName]
+	rkeFullState := &cluster.FullState{}
+	if err = json.Unmarshal([]byte(clusterData), rkeFullState); err != nil {
+		return err
+	}
+
+	// Move current state file
+	stateFilePath := cluster.GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir)
+	err = util.ReplaceFileWithBackup(stateFilePath, "rkestate")
+	if err != nil {
+		return err
+	}
+
+	// Write new state file
+	err = rkeFullState.WriteStateFile(context.Background(), stateFilePath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func RetrieveClusterStateConfigMap(
+	ctx context.Context,
+	rkeConfig *v3.RancherKubernetesEngineConfig,
+	dialersOptions hosts.DialersOptions,
+	flags cluster.ExternalFlags,
+	data map[string]interface{}) (string, string, string, string, map[string]pki.CertificatePKI, error) {
+	var APIURL, caCrt, clientCert, clientKey string
+
+	rkeFullState := &cluster.FullState{}
+
+	// Creating temp cluster to check if snapshot archive contains state file and retrieve it
+	tempCluster, err := cluster.InitClusterObject(ctx, rkeConfig, flags, "")
+	if err != nil {
+		return APIURL, caCrt, clientCert, clientKey, nil, err
+	}
+	if err := tempCluster.SetupDialers(ctx, dialersOptions); err != nil {
+		return APIURL, caCrt, clientCert, clientKey, nil, err
+	}
+	if err := tempCluster.TunnelHosts(ctx, flags); err != nil {
+		return APIURL, caCrt, clientCert, clientKey, nil, err
+	}
+	// Get ConfigMap containing cluster state from Control Plane Hosts
+	stateFile, err := tempCluster.GetStateFileFromConfigMap(ctx)
+
+	if err != nil {
+		return APIURL, caCrt, clientCert, clientKey, nil, err
+	}
+	rkeFullState, err = cluster.StringToFullState(ctx, stateFile)
+
+	// Move current state file
+	stateFilePath := cluster.GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir)
+	err = util.ReplaceFileWithBackup(stateFilePath, "rkestate")
+	if err != nil {
+		return APIURL, caCrt, clientCert, clientKey, nil, err
+	}
+
+	err = rkeFullState.WriteStateFile(context.Background(), stateFilePath)
+	if err != nil {
+		return APIURL, caCrt, clientCert, clientKey, nil, err
+	}
+
+	// Move current kubeconfig file
+	err = util.CopyFileWithPrefix(tempCluster.LocalKubeConfigPath, "kube_config")
+	if err != nil {
+		return APIURL, caCrt, clientCert, clientKey, nil, err
+	}
+	kubeCluster, _ := tempCluster.GetClusterState(ctx, rkeFullState)
+
+	if err := cluster.RebuildKubeconfig(ctx, kubeCluster); err != nil {
+		return APIURL, caCrt, clientCert, clientKey, nil, nil
+	}
+
+	return APIURL, caCrt, clientCert, clientKey, nil, nil
+}

main.go

@@ -62,6 +62,7 @@ func mainErr() error {
 		cmd.EtcdCommand(),
 		cmd.CertificateCommand(),
 		cmd.EncryptionCommand(),
+		cmd.UtilCommand(),
 	}
 	app.Flags = []cli.Flag{
 		cli.BoolFlag{

pki/deploy.go

@@ -183,6 +183,7 @@ func doRunDeployer(ctx context.Context, host *hosts.Host, containerEnv []string,
 
 func DeployAdminConfig(ctx context.Context, kubeConfig, localConfigPath string) error {
 	if len(kubeConfig) == 0 {
+		logrus.Infof("kubeConfig is empty")
 		return nil
 	}
 	logrus.Debugf("Deploying admin Kubeconfig locally at [%s]", localConfigPath)

services/controlplane.go

@@ -3,9 +3,11 @@ package services
 import (
 	"context"
 	"fmt"
+	"path"
 	"strings"
 	"sync"
 
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/client"
 	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
@@ -366,3 +368,37 @@ func isControlPlaneHostUpgradable(ctx context.Context, host *hosts.Host, process
 	logrus.Debugf("[%s] Host %v is not upgradable", ControlRole, host.HostnameOverride)
 	return false, nil
 }
+
+func RunGetStateFileFromConfigMap(ctx context.Context, controlPlaneHost *hosts.Host, prsMap map[string]v3.PrivateRegistry, dockerImage string) (string, error) {
+	imageCfg := &container.Config{
+		Entrypoint: []string{"bash"},
+		Cmd: []string{
+			"-c",
+			"kubectl --kubeconfig /etc/kubernetes/ssl/kubecfg-kube-node.yaml -n kube-system get configmap full-cluster-state -o json | jq -r .data.\\\"full-cluster-state\\\" | jq -r . > /tmp/configmap.cluster.rkestate",
+		},
+		Image: dockerImage,
+	}
+	hostCfg := &container.HostConfig{
+		Binds: []string{
+			fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(controlPlaneHost.PrefixPath, "/etc/kubernetes")),
+		},
+		NetworkMode:   container.NetworkMode("host"),
+		RestartPolicy: container.RestartPolicy{Name: "no"},
+	}
+
+	if err := docker.DoRemoveContainer(ctx, controlPlaneHost.DClient, ControlPlaneConfigMapStateFileContainerName, controlPlaneHost.Address); err != nil {
+		return "", err
+	}
+	if err := docker.DoRunOnetimeContainer(ctx, controlPlaneHost.DClient, imageCfg, hostCfg, ControlPlaneConfigMapStateFileContainerName, controlPlaneHost.Address, ControlRole, prsMap); err != nil {
+		return "", err
+	}
+	statefile, err := docker.ReadFileFromContainer(ctx, controlPlaneHost.DClient, controlPlaneHost.Address, ControlPlaneConfigMapStateFileContainerName, "/tmp/configmap.cluster.rkestate")
+	if err != nil {
+		return "", err
+	}
+	if err := docker.DoRemoveContainer(ctx, controlPlaneHost.DClient, ControlPlaneConfigMapStateFileContainerName, controlPlaneHost.Address); err != nil {
+		return "", err
+	}
+
+	return statefile, nil
+}

services/services.go

@@ -22,24 +22,25 @@ const (
 	SidekickServiceName   = "sidekick"
 	RBACAuthorizationMode = "rbac"
 
-	KubeAPIContainerName            = "kube-apiserver"
-	KubeletContainerName            = "kubelet"
-	KubeproxyContainerName          = "kube-proxy"
-	KubeControllerContainerName     = "kube-controller-manager"
-	SchedulerContainerName          = "kube-scheduler"
-	EtcdContainerName               = "etcd"
-	EtcdSnapshotContainerName       = "etcd-rolling-snapshots"
-	EtcdSnapshotOnceContainerName   = "etcd-snapshot-once"
-	EtcdSnapshotRemoveContainerName = "etcd-remove-snapshot"
-	EtcdRestoreContainerName        = "etcd-restore"
-	EtcdDownloadBackupContainerName = "etcd-download-backup"
-	EtcdServeBackupContainerName    = "etcd-Serve-backup"
-	EtcdChecksumContainerName       = "etcd-checksum-checker"
-	EtcdStateFileContainerName      = "etcd-extract-statefile"
-	NginxProxyContainerName         = "nginx-proxy"
-	SidekickContainerName           = "service-sidekick"
-	LogLinkContainerName            = "rke-log-linker"
-	LogCleanerContainerName         = "rke-log-cleaner"
+	KubeAPIContainerName                        = "kube-apiserver"
+	KubeletContainerName                        = "kubelet"
+	KubeproxyContainerName                      = "kube-proxy"
+	KubeControllerContainerName                 = "kube-controller-manager"
+	SchedulerContainerName                      = "kube-scheduler"
+	EtcdContainerName                           = "etcd"
+	EtcdSnapshotContainerName                   = "etcd-rolling-snapshots"
+	EtcdSnapshotOnceContainerName               = "etcd-snapshot-once"
+	EtcdSnapshotRemoveContainerName             = "etcd-remove-snapshot"
+	EtcdRestoreContainerName                    = "etcd-restore"
+	EtcdDownloadBackupContainerName             = "etcd-download-backup"
+	EtcdServeBackupContainerName                = "etcd-Serve-backup"
+	EtcdChecksumContainerName                   = "etcd-checksum-checker"
+	EtcdStateFileContainerName                  = "etcd-extract-statefile"
+	ControlPlaneConfigMapStateFileContainerName = "extract-statefile-configmap"
+	NginxProxyContainerName                     = "nginx-proxy"
+	SidekickContainerName                       = "service-sidekick"
+	LogLinkContainerName                        = "rke-log-linker"
+	LogCleanerContainerName                     = "rke-log-cleaner"
 
 	KubeAPIPort        = 6443
 	SchedulerPort      = 10251

util/util.go

@@ -2,6 +2,8 @@ package util
 
 import (
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"reflect"
@@ -180,3 +182,65 @@ func CleanWindowsPath(s string) string {
 	}
 	return strings.Join(new, "\\")
 }
+
+func ReplaceFileWithBackup(originalFile, prefixBackupFile string) error {
+	fileExists, err := IsFileExists(originalFile)
+	if err != nil {
+		return err
+	}
+	if !fileExists {
+		return nil
+	}
+	tmpfile, err := ioutil.TempFile(".", prefixBackupFile)
+	if err != nil {
+		return err
+	}
+	err = os.Rename(originalFile, tmpfile.Name())
+	if err != nil {
+		return err
+	}
+	logrus.Infof("Moved file [%s] to new location [%s] as back-up", originalFile, tmpfile.Name())
+	return nil
+}
+
+func CopyFileWithPrefix(originalFile, prefixDestFile string) error {
+	fileExists, err := IsFileExists(originalFile)
+	if err != nil {
+		return err
+	}
+	if !fileExists {
+		return nil
+	}
+
+	sourceFileStat, err := os.Stat(originalFile)
+	if err != nil {
+		return err
+	}
+
+	if !sourceFileStat.Mode().IsRegular() {
+		return fmt.Errorf("%s is not a regular file", originalFile)
+	}
+
+	source, err := os.Open(originalFile)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	destFile, err := ioutil.TempFile(".", prefixDestFile)
+	if err != nil {
+		return err
+	}
+
+	destination, err := os.Create(destFile.Name())
+	if err != nil {
+		return err
+	}
+	defer destination.Close()
+	_, err = io.Copy(destination, source)
+	if err != nil {
+		return err
+	}
+	logrus.Infof("Copied file [%s] to new location [%s] as back-up", originalFile, destFile.Name())
+	return nil
+}