ros list shows all the active services and any cache available updates

Signed-off-by: Sven Dowideit <SvenDowideit@home.org.au>

.dockerignore

@@ -2,6 +2,7 @@
 .idea
 .trash-cache
 bin
+docs
 state
 build
 images/*/build

cmd/cloudinitsave/cloudinitsave.go

@@ -328,7 +328,8 @@ func isCompose(content string) bool {
 	return strings.HasPrefix(content, "#compose\n")
 }
 
-func composeToCloudConfig(bytes []byte) ([]byte, error) {
+//TODO: move to config?
+func ComposeToCloudConfig(bytes []byte) ([]byte, error) {
 	compose := make(map[interface{}]interface{})
 	err := yaml.Unmarshal(bytes, &compose)
 	if err != nil {

cmd/control/cli.go

@@ -1,12 +1,18 @@
 package control
 
 import (
+	"fmt"
 	"os"
 
+	yaml "github.com/cloudfoundry-incubator/candiedyaml"
+
 	"github.com/codegangsta/cli"
+	libcomposeConfig "github.com/docker/libcompose/config"
+
 	"github.com/rancher/os/cmd/control/service"
 	"github.com/rancher/os/config"
 	"github.com/rancher/os/log"
+	"github.com/rancher/os/util/network"
 )
 
 func Main() {
@@ -27,110 +33,311 @@ func Main() {
 
 	app.Commands = []cli.Command{
 		{
-			Name:            "bootstrap",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          bootstrapAction,
+			Name:      "fetch",
+			ShortName: "f",
+			Usage:     "fetch configs from repos",
+			HideHelp:  true,
+			Action:    fetchServices,
 		},
+		// service
 		{
-			Name:        "config",
-			ShortName:   "c",
-			Usage:       "configure settings",
+			Name:      "list",
+			ShortName: "",
+			Usage:     "list services and states",
+			HideHelp:  true,
+			Action:    listServices,
+		}, {
+			Name:      "install",
+			ShortName: "",
+			Usage:     "install/upgrade service / RancherOS",
+			HideHelp:  true,
+			Action:    dummy,
+		}, {
+			Name:      "remove",
+			ShortName: "",
+			Usage:     "remove service",
+			HideHelp:  true,
+			Action:    dummy,
+		}, {
+			Name:      "logs",
+			ShortName: "",
+			Usage:     "service logs",
+			HideHelp:  true,
+			Action:    dummy,
+		},
+		// settings
+		{
+			Name:      "get",
+			ShortName: "",
+			Usage:     "get config value(s)",
+			HideHelp:  true,
+			Action:    dummy,
+		}, {
+			Name:      "set",
+			ShortName: "",
+			Usage:     "set config value(s)",
+			HideHelp:  true,
+			Action:    dummy,
+		},
+		// complete config
+		{
+			Name:      "export",
+			ShortName: "",
+			Usage:     "export config",
+			HideHelp:  true,
+			Action:    dummy,
+		}, {
+			Name:      "apply",
+			ShortName: "",
+			Usage:     "apply service&config changes",
+			HideHelp:  true,
+			Action:    dummy,
+		}, {
+			Name:      "validate",
+			ShortName: "",
+			Usage:     "validate config / service file",
+			HideHelp:  true,
+			Action:    dummy,
+		},
+		// old..
+		{
+			Name:        "old",
+			ShortName:   "o",
+			Usage:       "old Command line (deprecated, will be removed in future)",
 			HideHelp:    true,
-			Subcommands: configSubcommands(),
+			Subcommands: originalCli,
 		},
-		{
-			Name:        "console",
-			Usage:       "manage which console container is used",
-			HideHelp:    true,
-			Subcommands: consoleSubcommands(),
-		},
-		{
-			Name:            "console-init",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          consoleInitAction,
-		},
-		{
-			Name:            "dev",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          devAction,
-		},
-		{
-			Name:            "docker-init",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          dockerInitAction,
-		},
-		{
-			Name:        "engine",
-			Usage:       "manage which Docker engine is used",
-			HideHelp:    true,
-			Subcommands: engineSubcommands(),
-		},
-		{
-			Name:            "entrypoint",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          entrypointAction,
-		},
-		{
-			Name:            "env",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          envAction,
-		},
-		service.Commands(),
-		{
-			Name:        "os",
-			Usage:       "operating system upgrade/downgrade",
-			HideHelp:    true,
-			Subcommands: osSubcommands(),
-		},
-		{
-			Name:            "preload-images",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          preloadImagesAction,
-		},
-		{
-			Name:            "switch-console",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          switchConsoleAction,
-		},
-		{
-			Name:        "tls",
-			Usage:       "setup tls configuration",
-			HideHelp:    true,
-			Subcommands: tlsConfCommands(),
-		},
-		{
-			Name:            "udev-settle",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          udevSettleAction,
-		},
-		{
-			Name:            "user-docker",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          userDockerAction,
-		},
-		installCommand,
-		selinuxCommand(),
 	}
+	app.Commands = append(app.Commands, hiddenInternalCommands...)
 
 	app.Run(os.Args)
 }
+
+func dummy(c *cli.Context) error {
+	return nil
+}
+
+func fetchServices(c *cli.Context) error {
+	// fetch all the index.yml files, and the service.yml files that they refer to
+	// and put into a cache dir.
+	// Q - should there be one dir per index.yml so that you can have more than one service with the same name..
+
+	//TODO: need a --purge
+	//TODO: also need to fetch the rancher/os choices
+
+	return network.FetchAllServices()
+}
+
+func listServices(c *cli.Context) error {
+	//get the current cfg, and the make a cfg with all cached services
+	//then iterate through, listing all possible services, and what version is running, vs what version they could run
+	//Can't just merge current cfg and cached services, as we lose service.yml version info
+	currentConfig := config.LoadConfig()
+	cachedConfigs := GetAllServices()
+	// TODO: sort them!
+	fmt.Printf("Running\n")
+	for serviceName, serviceConfig := range currentConfig.Rancher.Services {
+		fmt.Printf("\t%s: %s\n", serviceName, serviceConfig.Image)
+		if len(cachedConfigs[serviceName]) > 0 {
+			fmt.Printf("\t\tAlternatives: ")
+			for serviceLongName, _ := range cachedConfigs[serviceName] {
+				fmt.Printf("%s, ", serviceLongName)
+			}
+			fmt.Printf("\n")
+		}
+	}
+	fmt.Printf("Available\n")
+	for serviceName, service := range cachedConfigs {
+		if _, ok := currentConfig.Rancher.Services[serviceName]; ok {
+			continue
+		}
+		for serviceLongName, _ := range service {
+			fmt.Printf("\t%s: %s\n", serviceName, serviceLongName)
+		}
+	}
+
+	return nil
+}
+
+func GetAllServices() map[string]map[string]*libcomposeConfig.ServiceConfigV1 {
+	//result := make(map[string]*libcomposeConfig.ServiceConfig)
+	result := make(map[string]map[string]*libcomposeConfig.ServiceConfigV1)
+
+	cfg := config.LoadConfig()
+	for repoName, _ := range cfg.Rancher.Repositories {
+		indexPath := fmt.Sprintf("%s/index.yml", repoName)
+		//content, err := network.LoadResource(indexPath, false)
+		content, err := network.CacheLookup(indexPath)
+		if err != nil {
+			log.Errorf("Failed to load %s: %v", indexPath, err)
+			continue
+		}
+
+		services := make(map[string][]string)
+		err = yaml.Unmarshal(content, &services)
+		if err != nil {
+			log.Errorf("Failed to unmarshal %s: %v", indexPath, err)
+			continue
+		}
+		for serviceType, serviceList := range services {
+			for _, serviceLongName := range serviceList {
+				servicePath := fmt.Sprintf("%s/%s.yml", repoName, serviceLongName)
+				//log.Infof("loading %s", serviceLongName)
+				content, err := network.CacheLookup(servicePath)
+				if err != nil {
+					log.Errorf("Failed to load %s: %v", servicePath, err)
+					continue
+				}
+				if content, err = ComposeToCloudConfig(content); err != nil {
+					log.Errorf("Failed to convert compose to cloud-config syntax: %v", err)
+					continue
+				}
+
+				p, err := config.ReadConfig(content, true)
+				if err != nil {
+					log.Errorf("Failed to load %s : %v", servicePath, err)
+				}
+
+				// yes, the serviceLongName is really only the yml file name
+				// and each yml file can contain more than one actual service
+				for serviceName, service := range p.Rancher.Services {
+					//service, _ := p.ServiceConfigs.Get(serviceName)
+					n := fmt.Sprintf("%s/%s", repoName, serviceLongName)
+					if result[serviceName] == nil {
+						result[serviceName] = map[string]*libcomposeConfig.ServiceConfigV1{}
+					}
+					result[serviceName][n] = service
+					log.Debugf("loaded %s(%s): %s", n, serviceType, serviceName)
+				}
+
+			}
+		}
+	}
+
+	return result
+}
+
+//TODO: copied from cloudinitsave, move to config.
+func ComposeToCloudConfig(bytes []byte) ([]byte, error) {
+	compose := make(map[interface{}]interface{})
+	err := yaml.Unmarshal(bytes, &compose)
+	if err != nil {
+		return nil, err
+	}
+
+	return yaml.Marshal(map[interface{}]interface{}{
+		"rancher": map[interface{}]interface{}{
+			"services": compose,
+		},
+	})
+}
+
+var originalCli = []cli.Command{
+	{
+		Name:        "config",
+		ShortName:   "c",
+		Usage:       "configure settings",
+		HideHelp:    true,
+		Subcommands: configSubcommands(),
+	},
+	{
+		Name:        "console",
+		Usage:       "manage which console container is used",
+		HideHelp:    true,
+		Subcommands: consoleSubcommands(),
+	},
+	{
+		Name:        "engine",
+		Usage:       "manage which Docker engine is used",
+		HideHelp:    true,
+		Subcommands: engineSubcommands(),
+	},
+	service.Commands(),
+	{
+		Name:        "os",
+		Usage:       "operating system upgrade/downgrade",
+		HideHelp:    true,
+		Subcommands: osSubcommands(),
+	},
+	{
+		Name:        "tls",
+		Usage:       "setup tls configuration",
+		HideHelp:    true,
+		Subcommands: tlsConfCommands(),
+	},
+	installCommand,
+	selinuxCommand(),
+}
+
+var hiddenInternalCommands = []cli.Command{
+	{
+		Name:            "bootstrap",
+		Hidden:          true,
+		HideHelp:        true,
+		SkipFlagParsing: true,
+		Action:          bootstrapAction,
+	},
+	{
+		Name:            "udev-settle",
+		Hidden:          true,
+		HideHelp:        true,
+		SkipFlagParsing: true,
+		Action:          udevSettleAction,
+	},
+	{
+		Name:            "user-docker",
+		Hidden:          true,
+		HideHelp:        true,
+		SkipFlagParsing: true,
+		Action:          userDockerAction,
+	},
+	{
+		Name:            "preload-images",
+		Hidden:          true,
+		HideHelp:        true,
+		SkipFlagParsing: true,
+		Action:          preloadImagesAction,
+	},
+	{
+		Name:            "switch-console",
+		Hidden:          true,
+		HideHelp:        true,
+		SkipFlagParsing: true,
+		Action:          switchConsoleAction,
+	},
+	{
+		Name:            "entrypoint",
+		Hidden:          true,
+		HideHelp:        true,
+		SkipFlagParsing: true,
+		Action:          entrypointAction,
+	},
+	{
+		Name:            "env",
+		Hidden:          true,
+		HideHelp:        true,
+		SkipFlagParsing: true,
+		Action:          envAction,
+	},
+	{
+		Name:            "console-init",
+		Hidden:          true,
+		HideHelp:        true,
+		SkipFlagParsing: true,
+		Action:          consoleInitAction,
+	},
+	{
+		Name:            "dev",
+		Hidden:          true,
+		HideHelp:        true,
+		SkipFlagParsing: true,
+		Action:          devAction,
+	},
+	{
+		Name:            "docker-init",
+		Hidden:          true,
+		HideHelp:        true,
+		SkipFlagParsing: true,
+		Action:          dockerInitAction,
+	},
+}

config/types.go

@@ -45,6 +45,7 @@ const (
 	CloudConfigScriptFile  = "/var/lib/rancher/conf/cloud-config-script"
 	MetaDataFile           = "/var/lib/rancher/conf/metadata"
 	CloudConfigFile        = "/var/lib/rancher/conf/cloud-config.yml"
+	CacheDirectory         = "/var/lib/rancher/cache/"
 )
 
 var (

main.go

@@ -1,11 +1,11 @@
 package main
 
 import (
-	"github.com/containernetworking/cni/plugins/ipam/host-local"
+	hostlocal "github.com/containernetworking/cni/plugins/ipam/host-local"
 	"github.com/containernetworking/cni/plugins/main/bridge"
 	"github.com/docker/docker/docker"
 	"github.com/docker/docker/pkg/reexec"
-	"github.com/rancher/cniglue"
+	glue "github.com/rancher/cniglue"
 	"github.com/rancher/os/cmd/cloudinitexecute"
 	"github.com/rancher/os/cmd/cloudinitsave"
 	"github.com/rancher/os/cmd/control"

scripts/tar-images

@@ -3,7 +3,7 @@ set -e
 
 cd $(dirname $0)/..
 
-IMAGES=$(bin/host_ros c images -i build/initrd/usr/share/ros/os-config.yml)
+IMAGES=$(bin/host_ros old c images -i build/initrd/usr/share/ros/os-config.yml)
 for i in $IMAGES; do
     if [ "${FORCE_PULL}" = "1" ] || ! docker inspect $i >/dev/null 2>&1; then
         docker pull $i

scripts/template

@@ -7,4 +7,4 @@ cd $(dirname $0)/..
 
 OUTPUT=build/initrd/usr/share/ros
 mkdir -p $OUTPUT
-./bin/host_ros c generate < os-config.tpl.yml > $OUTPUT/os-config.yml
+./bin/host_ros old c generate < os-config.tpl.yml > $OUTPUT/os-config.yml

util/network/cache.go

@@ -5,41 +5,45 @@ import (
 	"encoding/hex"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
+	"github.com/rancher/os/config"
 	"github.com/rancher/os/log"
 )
 
-const (
-	cacheDirectory = "/var/lib/rancher/cache/"
-)
-
 func locationHash(location string) string {
 	sum := md5.Sum([]byte(location))
 	return hex.EncodeToString(sum[:])
 }
 
-func cacheLookup(location string) []byte {
-	cacheFile := cacheDirectory + locationHash(location)
+func CacheLookup(location string) ([]byte, error) {
+	cacheFile := filepath.Join(config.CacheDirectory, location)
 	bytes, err := ioutil.ReadFile(cacheFile)
 	if err == nil {
 		log.Debugf("Using cached file: %s", cacheFile)
-		return bytes
+		return bytes, nil
 	}
-	return nil
+	log.Debugf("Cached file not found: %s", cacheFile)
+	return nil, err
 }
 
-func cacheAdd(location string, data []byte) {
-	tempFile, err := ioutil.TempFile(cacheDirectory, "")
+func cacheAdd(location string, data []byte) error {
+	os.MkdirAll(config.CacheDirectory, 0755)
+	tempFile, err := ioutil.TempFile(config.CacheDirectory, "")
 	if err != nil {
-		return
+		return err
 	}
 	defer os.Remove(tempFile.Name())
 
 	_, err = tempFile.Write(data)
 	if err != nil {
-		return
+		return err
 	}
 
-	cacheFile := cacheDirectory + locationHash(location)
+	cacheFile := filepath.Join(config.CacheDirectory, location)
+	cacheDir := filepath.Dir(cacheFile)
+	log.Debugf("writing %s to %s", cacheFile, cacheDir)
+	os.MkdirAll(cacheDir, 0755)
 	os.Rename(tempFile.Name(), cacheFile)
+	return nil
 }

util/network/network.go

@@ -32,6 +32,48 @@ func GetEngines(urls []string) ([]string, error) {
 	return getServices(urls, "engines")
 }
 
+func FetchAllServices() error {
+	cfg := config.LoadConfig()
+	for name, url := range cfg.Rancher.Repositories {
+		log.Infof("repo index %s: %v", name, url.URL)
+		indexURL := fmt.Sprintf("%s/index.yml", url.URL)
+		content, err := loadFromNetwork(indexURL)
+		if err != nil {
+			log.Errorf("Failed to load %s: %v", indexURL, err)
+			continue
+		}
+		// save the index file to the cache dir
+		cacheAdd(fmt.Sprintf("%s/index.yml", name), content)
+		// load it, and then download each service file and cache too
+		services := make(map[string][]string)
+		err = yaml.Unmarshal(content, &services)
+		if err != nil {
+			log.Errorf("Failed to unmarshal %s: %v", indexURL, err)
+			continue
+		}
+
+		for serviceType, serviceList := range services {
+			for _, serviceName := range serviceList {
+				fmt.Printf("\t%s is type %s from %s\n", serviceName, serviceType, name)
+				serviceURL := serviceURL(url.URL, serviceName)
+				content, err := loadFromNetwork(serviceURL)
+				if err != nil {
+					log.Errorf("Failed to load %s: %v", serviceURL, err)
+					continue
+				}
+				// save the service file to the cache dir
+				if err = cacheAdd(fmt.Sprintf("%s/%s.yml", name, serviceName), content); err != nil {
+					log.Errorf("cacheAdd: %s", err)
+				}
+				//display which services are new, and which are updated from previous cache
+				//and `listServices` will need to compare the cached servcies with the version that was enabled/up
+				//also list the services that are no longer updated (ie, purge candidates)
+			}
+		}
+	}
+	return nil
+}
+
 func getServices(urls []string, key string) ([]string, error) {
 	result := []string{}
 
@@ -80,11 +122,11 @@ func SetProxyEnvironmentVariables(cfg *config.CloudConfig) {
 }
 
 func loadFromNetwork(location string) ([]byte, error) {
-	bytes := cacheLookup(location)
-	if bytes != nil {
-		return bytes, nil
-	}
-
+	/*	bytes := cacheLookup(location)
+		if bytes != nil {
+			return bytes, nil
+		}
+	*/
 	cfg := config.LoadConfig()
 	SetProxyEnvironmentVariables(cfg)