diff --git a/cluster/remove.go b/cluster/remove.go
index ef2e777c..f7a6522c 100644
--- a/cluster/remove.go
+++ b/cluster/remove.go
@@ -70,7 +70,7 @@ func (c *Cluster) CleanupNodes(ctx context.Context) error {
 
 func (c *Cluster) CleanupFiles(ctx context.Context) error {
 	pki.RemoveAdminConfig(ctx, c.LocalKubeConfigPath)
-	removeStateFile(ctx, c.StateFilePath)
+	RemoveStateFile(ctx, c.StateFilePath)
 	return nil
 }
 
diff --git a/cluster/state.go b/cluster/state.go
index 6ff0682a..a6ecc86e 100644
--- a/cluster/state.go
+++ b/cluster/state.go
@@ -264,7 +264,7 @@ func ReadStateFile(ctx context.Context, statePath string) (*FullState, error) {
 	return rkeFullState, nil
 }
 
-func removeStateFile(ctx context.Context, statePath string) {
+func RemoveStateFile(ctx context.Context, statePath string) {
 	log.Infof(ctx, "Removing state file: %s", statePath)
 	if err := os.Remove(statePath); err != nil {
 		logrus.Warningf("Failed to remove state file: %v", err)
diff --git a/cmd/remove.go b/cmd/remove.go
index 962d6a3b..fca91221 100644
--- a/cmd/remove.go
+++ b/cmd/remove.go
@@ -166,8 +166,12 @@ func clusterRemoveDind(ctx *cli.Context) error {
 			return err
 		}
 	}
-	localKubeConfigPath := pki.GetLocalKubeConfig(filePath, "")
 	// remove the kube config file
+	localKubeConfigPath := pki.GetLocalKubeConfig(filePath, "")
 	pki.RemoveAdminConfig(context.Background(), localKubeConfigPath)
+
+	// remove cluster state file
+	stateFilePath := cluster.GetStateFilePath(filePath, "")
+	cluster.RemoveStateFile(context.Background(), stateFilePath)
 	return err
 }
diff --git a/dind/dind.go b/dind/dind.go
index a2eb5df6..df116faf 100644
--- a/dind/dind.go
+++ b/dind/dind.go
@@ -120,6 +120,10 @@ func RmoveDindContainer(ctx context.Context, dindAddress string) error {
 	if err := cli.ContainerRemove(ctx, containerName, types.ContainerRemoveOptions{
 		Force:         true,
 		RemoveVolumes: true}); err != nil {
+		if client.IsErrNotFound(err) {
+			logrus.Debugf("[remove/%s] Container doesn't exist on host [%s]", containerName, cli.DaemonHost())
+			return nil
+		}
 		return fmt.Errorf("Failed to remove dind container [%s] on host [%s]: %v", containerName, cli.DaemonHost(), err)
 	}
 	logrus.Infof("[%s] Successfully Removed dind container [%s] on host [%s]", DINDPlane, containerName, cli.DaemonHost())