diff --git a/cluster/state.go b/cluster/state.go
index d3bebb8f..861754e4 100644
--- a/cluster/state.go
+++ b/cluster/state.go
@@ -15,10 +15,10 @@ import (
 	"github.com/rancher/rke/k8s"
 	"github.com/rancher/rke/log"
 	"github.com/rancher/rke/pki"
-	"github.com/rancher/types/apis/management.cattle.io/v3"
+	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 )
 
 const (
@@ -267,7 +267,6 @@ func removeStateFile(ctx context.Context, statePath string) {
 }
 
 func GetStateFromNodes(ctx context.Context, kubeCluster *Cluster) *Cluster {
-	log.Infof(ctx, "[state] Fetching cluster state from Nodes")
 	var currentCluster Cluster
 	var clusterFile string
 	var err error
diff --git a/cmd/common.go b/cmd/common.go
index 6929a965..f49b8d99 100644
--- a/cmd/common.go
+++ b/cmd/common.go
@@ -6,10 +6,12 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/rancher/rke/cluster"
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/log"
+	"github.com/rancher/rke/util"
 	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -82,8 +84,11 @@ func ClusterInit(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfi
 		return err
 	}
 
-	err = doUpgradeLegacyCluster(ctx, kubeCluster, rkeFullState, flags)
+	err = checkLegacyCluster(ctx, kubeCluster, rkeFullState, flags)
 	if err != nil {
+		if strings.Contains(err.Error(), "aborting upgrade") {
+			return err
+		}
 		log.Warnf(ctx, "[state] can't fetch legacy cluster state from Kubernetes")
 	}
 	// check if certificate rotate or normal init
@@ -128,15 +133,32 @@ func setS3OptionsFromCLI(c *cli.Context) *v3.S3BackupConfig {
 	return s3BackupBackend
 }
 
-func doUpgradeLegacyCluster(ctx context.Context, kubeCluster *cluster.Cluster, fullState *cluster.FullState, flags cluster.ExternalFlags) error {
-	if _, err := os.Stat(kubeCluster.LocalKubeConfigPath); os.IsNotExist(err) {
-		// there is no kubeconfig. This is a new cluster
-		logrus.Debug("[state] local kubeconfig not found, this is a new cluster")
+func checkLegacyCluster(ctx context.Context, kubeCluster *cluster.Cluster, fullState *cluster.FullState, flags cluster.ExternalFlags) error {
+	stateFileExists, err := util.IsFileExists(kubeCluster.StateFilePath)
+	if err != nil {
+		return err
+	}
+	if stateFileExists {
+		logrus.Debug("[state] previous state found, this is not a legacy cluster")
 		return nil
 	}
-	if _, err := os.Stat(kubeCluster.StateFilePath); err == nil {
-		// this cluster has a previous state, I don't need to upgrade!
-		logrus.Debug("[state] previous state found, this is not a legacy cluster")
+	logrus.Debug("[state] previous state not found, possible legacy cluster")
+	return fetchAndUpdateStateFromLegacyCluster(ctx, kubeCluster, fullState, flags)
+}
+
+func fetchAndUpdateStateFromLegacyCluster(ctx context.Context, kubeCluster *cluster.Cluster, fullState *cluster.FullState, flags cluster.ExternalFlags) error {
+	kubeConfigExists, err := util.IsFileExists(kubeCluster.LocalKubeConfigPath)
+	if err != nil {
+		return err
+	}
+	if !kubeConfigExists {
+		// if kubeconfig doesn't exist and its a legacy cluster then error out
+		if err := kubeCluster.TunnelHosts(ctx, flags); err != nil {
+			return err
+		}
+		if recoveredCluster := cluster.GetStateFromNodes(ctx, kubeCluster); recoveredCluster != nil {
+			return fmt.Errorf("This is a legacy cluster with no kube config, aborting upgrade. Please re-run rke up with rke 0.1.x to retrieve correct state")
+		}
 		return nil
 	}
 	// We have a kubeconfig and no current state. This is a legacy cluster or a new cluster with old kubeconfig
@@ -153,6 +175,8 @@ func doUpgradeLegacyCluster(ctx context.Context, kubeCluster *cluster.Cluster, f
 		if err != nil {
 			return err
 		}
+		// fetching state/certs from nodes should be removed in rke 0.3.0
+		log.Infof(ctx, "[state] Fetching cluster state from Nodes")
 		recoveredCluster = cluster.GetStateFromNodes(ctx, kubeCluster)
 	}
 	// if we found a recovered cluster, we will need override the current state
diff --git a/cmd/etcd.go b/cmd/etcd.go
index 9fba28e2..7f99b2d6 100644
--- a/cmd/etcd.go
+++ b/cmd/etcd.go
@@ -117,7 +117,7 @@ func RestoreEtcdSnapshot(
 	}
 	stateFilePath := cluster.GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir)
 	rkeFullState, _ := cluster.ReadStateFile(ctx, stateFilePath)
-	if err := doUpgradeLegacyCluster(ctx, kubeCluster, rkeFullState, flags); err != nil {
+	if err := checkLegacyCluster(ctx, kubeCluster, rkeFullState, flags); err != nil {
 		return APIURL, caCrt, clientCert, clientKey, nil, err
 	}
 
diff --git a/util/util.go b/util/util.go
index fcd21e7e..c71a08e8 100644
--- a/util/util.go
+++ b/util/util.go
@@ -103,3 +103,13 @@ func GetTagMajorVersion(tag string) string {
 	}
 	return strings.Join(splitTag[:2], ".")
 }
+
+func IsFileExists(filePath string) (bool, error) {
+	if _, err := os.Stat(filePath); err == nil {
+		return true, nil
+	} else if os.IsNotExist(err) {
+		return false, nil
+	} else {
+		return false, err
+	}
+}