Make datasource.AvailabilityChanges() able to be dynamic so fail out for configdrive mount and url 404's faster

Signed-off-by: Sven Dowideit <SvenDowideit@home.org.au>

cmd/cloudinitsave/cloudinitsave.go

@@ -67,11 +67,83 @@ func Main() {
 
 func SaveCloudConfig(network bool) error {
 	log.Debugf("SaveCloudConfig")
-	userDataBytes, metadata, err := fetchUserData(network)
+	cfg := rancherConfig.LoadConfig()
+
+	dss := getDatasources(cfg, network)
+	if len(dss) == 0 {
+		log.Errorf("currentDatasource - none found")
+		return nil
+	}
+
+	selectDatasource(dss)
+	return nil
+}
+
+func RequiresNetwork(datasource string) bool {
+	// TODO: move into the datasources (and metadatasources)
+	parts := strings.SplitN(datasource, ":", 2)
+	requiresNetwork, ok := map[string]bool{
+		"ec2":          true,
+		"file":         false,
+		"url":          true,
+		"cmdline":      true,
+		"configdrive":  false,
+		"digitalocean": true,
+		"gce":          true,
+		"packet":       true,
+	}[parts[0]]
+	return ok && requiresNetwork
+}
+
+func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error {
+	os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600)
+
+	if len(scriptBytes) > 0 {
+		log.Infof("Writing to %s", rancherConfig.CloudConfigScriptFile)
+		if err := util.WriteFileAtomic(rancherConfig.CloudConfigScriptFile, scriptBytes, 500); err != nil {
+			log.Errorf("Error while writing file %s: %v", rancherConfig.CloudConfigScriptFile, err)
+			return err
+		}
+	}
+
+	if len(cloudConfigBytes) > 0 {
+		if err := util.WriteFileAtomic(rancherConfig.CloudConfigBootFile, cloudConfigBytes, 400); err != nil {
+			return err
+		}
+		// Don't put secrets into the log
+		//log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigBootFile, string(cloudConfigBytes))
+	}
+
+	metaDataBytes, err := yaml.Marshal(metadata)
 	if err != nil {
 		return err
 	}
 
+	if err = util.WriteFileAtomic(rancherConfig.MetaDataFile, metaDataBytes, 400); err != nil {
+		return err
+	}
+	// Don't put secrets into the log
+	//log.Infof("Written to %s:\n%s", rancherConfig.MetaDataFile, string(metaDataBytes))
+
+	return nil
+}
+
+func fetchAndSave(ds datasource.Datasource) error {
+	var metadata datasource.Metadata
+
+	log.Infof("Fetching user-data from datasource %s", ds)
+	userDataBytes, err := ds.FetchUserdata()
+	if err != nil {
+		log.Errorf("Failed fetching user-data from datasource: %v", err)
+		return err
+	}
+	log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
+	metadata, err = ds.FetchMetadata()
+	if err != nil {
+		log.Errorf("Failed fetching meta-data from datasource: %v", err)
+		return err
+	}
+
 	userData := string(userDataBytes)
 	scriptBytes := []byte{}
 
@@ -101,87 +173,6 @@ func SaveCloudConfig(network bool) error {
 	return saveFiles(userDataBytes, scriptBytes, metadata)
 }
 
-func RequiresNetwork(datasource string) bool {
-	parts := strings.SplitN(datasource, ":", 2)
-	requiresNetwork, ok := map[string]bool{
-		"ec2":          true,
-		"file":         false,
-		"url":          true,
-		"cmdline":      true,
-		"configdrive":  false,
-		"digitalocean": true,
-		"gce":          true,
-		"packet":       true,
-	}[parts[0]]
-	return ok && requiresNetwork
-}
-
-func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error {
-	os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600)
-
-	if len(scriptBytes) > 0 {
-		log.Infof("Writing to %s", rancherConfig.CloudConfigScriptFile)
-		if err := util.WriteFileAtomic(rancherConfig.CloudConfigScriptFile, scriptBytes, 500); err != nil {
-			log.Errorf("Error while writing file %s: %v", rancherConfig.CloudConfigScriptFile, err)
-			return err
-		}
-	}
-
-	if len(cloudConfigBytes) > 0 {
-		if err := util.WriteFileAtomic(rancherConfig.CloudConfigBootFile, cloudConfigBytes, 400); err != nil {
-			return err
-		}
-		log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigBootFile, string(cloudConfigBytes))
-	}
-
-	metaDataBytes, err := yaml.Marshal(metadata)
-	if err != nil {
-		return err
-	}
-
-	if err = util.WriteFileAtomic(rancherConfig.MetaDataFile, metaDataBytes, 400); err != nil {
-		return err
-	}
-	log.Infof("Written to %s:\n%s", rancherConfig.MetaDataFile, string(metaDataBytes))
-
-	return nil
-}
-
-func currentDatasource(network bool) (datasource.Datasource, error) {
-	cfg := rancherConfig.LoadConfig()
-
-	dss := getDatasources(cfg, network)
-	if len(dss) == 0 {
-		log.Errorf("currentDatasource - none found")
-		return nil, nil
-	}
-
-	ds := selectDatasource(dss)
-	return ds, nil
-}
-
-func fetchUserData(network bool) ([]byte, datasource.Metadata, error) {
-	var metadata datasource.Metadata
-	ds, err := currentDatasource(network)
-	if err != nil || ds == nil {
-		log.Errorf("Failed to select datasource: %v", err)
-		return nil, metadata, err
-	}
-	log.Infof("Fetching user-data from datasource %v", ds.Type())
-	userDataBytes, err := ds.FetchUserdata()
-	if err != nil {
-		log.Errorf("Failed fetching user-data from datasource: %v", err)
-		return nil, metadata, err
-	}
-	log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
-	metadata, err = ds.FetchMetadata()
-	if err != nil {
-		log.Errorf("Failed fetching meta-data from datasource: %v", err)
-		return nil, metadata, err
-	}
-	return userDataBytes, metadata, nil
-}
-
 // getDatasources creates a slice of possible Datasources for cloudinit based
 // on the different source command-line flags.
 func getDatasources(cfg *rancherConfig.CloudConfig, network bool) []datasource.Datasource {
@@ -276,14 +267,15 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
 			for {
 				log.Infof("cloud-init: Checking availability of %q\n", s.Type())
 				if s.IsAvailable() {
-					log.Infof("cloud-init: Datasource GOOD: %s", s)
+					log.Infof("cloud-init: Datasource available: %s", s)
 					ds <- s
 					return
 				}
-				log.Errorf("cloud-init: Datasource not ready: %s", s)
 				if !s.AvailabilityChanges() {
+					log.Infof("cloud-init: Datasource unavailable, skipping: %s", s)
 					return
 				}
+				log.Errorf("cloud-init: Datasource not ready, will retry: %s", s)
 				select {
 				case <-stop:
 					return
@@ -303,6 +295,10 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
 	var s datasource.Datasource
 	select {
 	case s = <-ds:
+		err := fetchAndSave(s)
+		if err != nil {
+			log.Errorf("Error fetching cloud-init datasource(%s): %s", s, err)
+		}
 	case <-done:
 	case <-time.After(datasourceTimeout):
 	}

config/cloudinit/datasource/configdrive/configdrive.go

@@ -37,13 +37,14 @@ const (
 )
 
 type ConfigDrive struct {
-	root      string
+	root                string
-	readFile  func(filename string) ([]byte, error)
+	readFile            func(filename string) ([]byte, error)
-	lastError error
+	lastError           error
+	availabilityChanges bool
 }
 
 func NewDatasource(root string) *ConfigDrive {
-	return &ConfigDrive{root, ioutil.ReadFile, nil}
+	return &ConfigDrive{root, ioutil.ReadFile, nil, true}
 }
 
 func (cd *ConfigDrive) IsAvailable() bool {
@@ -51,6 +52,8 @@ func (cd *ConfigDrive) IsAvailable() bool {
 		cd.lastError = MountConfigDrive()
 		if cd.lastError != nil {
 			log.Error(cd.lastError)
+			// Don't keep retrying if we can't mount
+			cd.availabilityChanges = false
 			return false
 		}
 		defer cd.Finish()
@@ -58,6 +61,7 @@ func (cd *ConfigDrive) IsAvailable() bool {
 
 	_, cd.lastError = os.Stat(cd.root)
 	return !os.IsNotExist(cd.lastError)
+	// TODO: consider changing IsNotExists to not-available _and_ does not change
 }
 
 func (cd *ConfigDrive) Finish() error {
@@ -72,7 +76,7 @@ func (cd *ConfigDrive) String() string {
 }
 
 func (cd *ConfigDrive) AvailabilityChanges() bool {
-	return true
+	return cd.availabilityChanges
 }
 
 func (cd *ConfigDrive) ConfigRoot() string {
@@ -130,15 +134,13 @@ func (cd *ConfigDrive) tryReadFile(filename string) ([]byte, error) {
 		}
 		defer cd.Finish()
 	}
-	log.Printf("Attempting to read from %q\n", filename)
+	log.Debugf("Attempting to read from %q\n", filename)
 	data, err := cd.readFile(filename)
 	if os.IsNotExist(err) {
 		err = nil
 	}
 	if err != nil {
 		log.Errorf("ERROR read cloud-config file(%s) - err: %q", filename, err)
-	} else {
-		log.Debugf("SUCCESS read cloud-config file(%s) - date: %s", filename, string(data))
 	}
 	return data, err
 }

config/cloudinit/datasource/configdrive/configdrive_test.go

@@ -57,7 +57,7 @@ func TestFetchMetadata(t *testing.T) {
 			},
 		},
 	} {
-		cd := ConfigDrive{tt.root, tt.files.ReadFile, nil}
+		cd := ConfigDrive{tt.root, tt.files.ReadFile, nil, true}
 		metadata, err := cd.FetchMetadata()
 		if err != nil {
 			t.Fatalf("bad error for %+v: want %v, got %q", tt, nil, err)
@@ -91,7 +91,7 @@ func TestFetchUserdata(t *testing.T) {
 			"userdata",
 		},
 	} {
-		cd := ConfigDrive{tt.root, tt.files.ReadFile, nil}
+		cd := ConfigDrive{tt.root, tt.files.ReadFile, nil, true}
 		userdata, err := cd.FetchUserdata()
 		if err != nil {
 			t.Fatalf("bad error for %+v: want %v, got %q", tt, nil, err)
@@ -116,7 +116,7 @@ func TestConfigRoot(t *testing.T) {
 			"/media/configdrive/openstack",
 		},
 	} {
-		cd := ConfigDrive{tt.root, nil, nil}
+		cd := ConfigDrive{tt.root, nil, nil, true}
 		if configRoot := cd.ConfigRoot(); configRoot != tt.configRoot {
 			t.Fatalf("bad config root for %q: want %q, got %q", tt, tt.configRoot, configRoot)
 		}

config/cloudinit/datasource/url/url.go

@@ -46,7 +46,9 @@ func (f *RemoteFile) String() string {
 
 func (f *RemoteFile) AvailabilityChanges() bool {
 	if f.lastError != nil {
-		if _, ok := f.lastError.(pkg.ErrNotFound); ok {
+		// if we have a Network error, then we should retry.
+		// otherwise, we've made a request to the server, and its said nope.
+		if _, ok := f.lastError.(pkg.ErrNetwork); !ok {
 			return false
 		}
 	}

os-config.tpl.yml

@@ -67,7 +67,6 @@ rancher:
   cloud_init:
     datasources:
     - configdrive:/media/config-2
-    - url:http://example.com/404-error
   repositories:
     core:
       url: {{.OS_SERVICES_REPO}}/{{.REPO_VERSION}}