Support service cache update

cmd/control/console.go

@@ -43,8 +43,14 @@ func consoleSubcommands() []cli.Command {
 			Action: consoleEnable,
 		},
 		{
-			Name:   "list",
-			Usage:  "list available consoles",
+			Name:  "list",
+			Usage: "list available consoles",
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "update, u",
+					Usage: "update console cache",
+				},
+			},
 			Action: consoleList,
 		},
 	}
@@ -127,7 +133,7 @@ func consoleEnable(c *cli.Context) error {
 
 func consoleList(c *cli.Context) error {
 	cfg := config.LoadConfig()
-	consoles := availableConsoles(cfg)
+	consoles := availableConsoles(cfg, c.Bool("update"))
 	CurrentConsole := CurrentConsole()
 
 	for _, console := range consoles {
@@ -144,13 +150,20 @@ func consoleList(c *cli.Context) error {
 }
 
 func validateConsole(console string, cfg *config.CloudConfig) {
-	consoles := availableConsoles(cfg)
+	consoles := availableConsoles(cfg, false)
 	if !service.IsLocalOrURL(console) && !util.Contains(consoles, console) {
 		log.Fatalf("%s is not a valid console", console)
 	}
 }
 
-func availableConsoles(cfg *config.CloudConfig) []string {
+func availableConsoles(cfg *config.CloudConfig, update bool) []string {
+	if update {
+		err := network.UpdateCaches(cfg.Rancher.Repositories.ToArray(), "consoles")
+		if err != nil {
+			log.Debugf("Failed to update console caches: %v", err)
+		}
+
+	}
 	consoles, err := network.GetConsoles(cfg.Rancher.Repositories.ToArray())
 	if err != nil {
 		log.Fatal(err)

cmd/control/engine.go

@@ -108,8 +108,14 @@ func engineSubcommands() []cli.Command {
 			Action: engineEnable,
 		},
 		{
-			Name:   "list",
-			Usage:  "list available Docker engines (include the Dind engines)",
+			Name:  "list",
+			Usage: "list available Docker engines (include the Dind engines)",
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "update, u",
+					Usage: "update engine cache",
+				},
+			},
 			Action: engineList,
 		},
 	}
@@ -272,7 +278,7 @@ func engineEnable(c *cli.Context) error {
 
 func engineList(c *cli.Context) error {
 	cfg := config.LoadConfig()
-	engines := availableEngines(cfg)
+	engines := availableEngines(cfg, c.Bool("update"))
 	currentEngine := CurrentEngine()
 
 	for _, engine := range engines {
@@ -315,13 +321,20 @@ func engineList(c *cli.Context) error {
 }
 
 func validateEngine(engine string, cfg *config.CloudConfig) {
-	engines := availableEngines(cfg)
+	engines := availableEngines(cfg, false)
 	if !service.IsLocalOrURL(engine) && !util.Contains(engines, engine) {
 		log.Fatalf("%s is not a valid engine", engine)
 	}
 }
 
-func availableEngines(cfg *config.CloudConfig) []string {
+func availableEngines(cfg *config.CloudConfig, update bool) []string {
+	if update {
+		err := network.UpdateCaches(cfg.Rancher.Repositories.ToArray(), "engines")
+		if err != nil {
+			log.Debugf("Failed to update engine caches: %v", err)
+		}
+
+	}
 	engines, err := network.GetEngines(cfg.Rancher.Repositories.ToArray())
 	if err != nil {
 		log.Fatal(err)

cmd/control/os.go

@@ -71,8 +71,14 @@ func osSubcommands() []cli.Command {
 			},
 		},
 		{
-			Name:   "list",
-			Usage:  "list the current available versions",
+			Name:  "list",
+			Usage: "list the current available versions",
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "update, u",
+					Usage: "update engine cache",
+				},
+			},
 			Action: osMetaDataGet,
 		},
 		{
@@ -83,7 +89,7 @@ func osSubcommands() []cli.Command {
 	}
 }
 
-func getImages() (*Images, error) {
+func getImages(update bool) (*Images, error) {
 	upgradeURL, err := getUpgradeURL()
 	if err != nil {
 		return nil, err
@@ -110,6 +116,13 @@ func getImages() (*Images, error) {
 		u.RawQuery = q.Encode()
 		upgradeURL = u.String()
 
+		if update {
+			_, err := network.UpdateCache(upgradeURL)
+			if err != nil {
+				log.Errorf("Failed to update os caches: %v", err)
+			}
+		}
+
 		body, err = network.LoadFromNetwork(upgradeURL)
 		if err != nil {
 			return nil, err
@@ -131,7 +144,7 @@ func getImages() (*Images, error) {
 }
 
 func osMetaDataGet(c *cli.Context) error {
-	images, err := getImages()
+	images, err := getImages(c.Bool("update"))
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -172,7 +185,7 @@ func osMetaDataGet(c *cli.Context) error {
 }
 
 func getLatestImage() (string, error) {
-	images, err := getImages()
+	images, err := getImages(false)
 	if err != nil {
 		return "", err
 	}

cmd/control/service/service.go

@@ -78,6 +78,10 @@ func serviceSubCommands() []cli.Command {
 					Name:  "all, a",
 					Usage: "list all services and state",
 				},
+				cli.BoolFlag{
+					Name:  "update, u",
+					Usage: "update service cache",
+				},
 			},
 			Action: list,
 		},
@@ -180,7 +184,7 @@ func list(c *cli.Context) error {
 		clone[service] = enabled
 	}
 
-	services := availableService(cfg)
+	services := availableService(cfg, c.Bool("update"))
 
 	if c.Bool("all") {
 		for service := range cfg.Rancher.Services {
@@ -222,7 +226,7 @@ func IsLocalOrURL(service string) bool {
 
 // ValidService checks to see if the service definition exists
 func ValidService(service string, cfg *config.CloudConfig) bool {
-	services := availableService(cfg)
+	services := availableService(cfg, false)
 	if !IsLocalOrURL(service) && !util.Contains(services, service) {
 		return false
 	}
@@ -235,7 +239,14 @@ func validateService(service string, cfg *config.CloudConfig) {
 	}
 }
 
-func availableService(cfg *config.CloudConfig) []string {
+func availableService(cfg *config.CloudConfig, update bool) []string {
+	if update {
+		err := network.UpdateCaches(cfg.Rancher.Repositories.ToArray(), "services")
+		if err != nil {
+			log.Debugf("Failed to update service caches: %v", err)
+		}
+
+	}
 	services, err := network.GetServices(cfg.Rancher.Repositories.ToArray())
 	if err != nil {
 		log.Fatalf("Failed to get services: %v", err)

pkg/util/network/cache.go

@@ -43,3 +43,8 @@ func cacheAdd(location string, data []byte) {
 	cacheFile := cacheDirectory + locationHash(location)
 	os.Rename(tempFile.Name(), cacheFile)
 }
+
+func cacheRemove(location string) error {
+	cacheFile := cacheDirectory + locationHash(location)
+	return os.Remove(cacheFile)
+}

pkg/util/network/network.go

@@ -193,3 +193,38 @@ func LoadMultiEngineResource(name string) ([]byte, error) {
 
 	return nil, err
 }
+
+func UpdateCaches(urls []string, key string) error {
+	for _, url := range urls {
+		indexURL := fmt.Sprintf("%s/index.yml", url)
+		content, err := UpdateCache(indexURL)
+		if err != nil {
+			return err
+		}
+
+		services := make(map[string][]string)
+		err = yaml.Unmarshal(content, &services)
+		if err != nil {
+			return err
+		}
+
+		list := services[key]
+		for _, name := range list {
+			serviceURL := serviceURL(url, name)
+			// no need to handle error
+			UpdateCache(serviceURL)
+		}
+	}
+	return nil
+}
+
+func UpdateCache(location string) ([]byte, error) {
+	if err := cacheRemove(location); err != nil {
+		return []byte{}, err
+	}
+	content, err := LoadResource(location, true)
+	if err != nil {
+		return []byte{}, err
+	}
+	return content, nil
+}