diff --git a/.dockerignore b/.dockerignore index 6cbff056..e3633ee7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,7 @@ .idea .trash-cache bin +docs state build images/*/build diff --git a/cmd/cloudinitsave/cloudinitsave.go b/cmd/cloudinitsave/cloudinitsave.go index 887cfc48..b2f135b6 100755 --- a/cmd/cloudinitsave/cloudinitsave.go +++ b/cmd/cloudinitsave/cloudinitsave.go @@ -328,7 +328,8 @@ func isCompose(content string) bool { return strings.HasPrefix(content, "#compose\n") } -func composeToCloudConfig(bytes []byte) ([]byte, error) { +//TODO: move to config? +func ComposeToCloudConfig(bytes []byte) ([]byte, error) { compose := make(map[interface{}]interface{}) err := yaml.Unmarshal(bytes, &compose) if err != nil { diff --git a/cmd/control/cli.go b/cmd/control/cli.go old mode 100644 new mode 100755 index 048ba747..ecff8d42 --- a/cmd/control/cli.go +++ b/cmd/control/cli.go @@ -1,12 +1,18 @@ package control import ( + "fmt" "os" + yaml "github.com/cloudfoundry-incubator/candiedyaml" + "github.com/codegangsta/cli" + libcomposeConfig "github.com/docker/libcompose/config" + "github.com/rancher/os/cmd/control/service" "github.com/rancher/os/config" "github.com/rancher/os/log" + "github.com/rancher/os/util/network" ) func Main() { @@ -27,110 +33,311 @@ func Main() { app.Commands = []cli.Command{ { - Name: "bootstrap", - Hidden: true, - HideHelp: true, - SkipFlagParsing: true, - Action: bootstrapAction, + Name: "fetch", + ShortName: "f", + Usage: "fetch configs from repos", + HideHelp: true, + Action: fetchServices, }, + // service { - Name: "config", - ShortName: "c", - Usage: "configure settings", + Name: "list", + ShortName: "", + Usage: "list services and states", + HideHelp: true, + Action: listServices, + }, { + Name: "install", + ShortName: "", + Usage: "install/upgrade service / RancherOS", + HideHelp: true, + Action: dummy, + }, { + Name: "remove", + ShortName: "", + Usage: "remove service", + HideHelp: true, + Action: dummy, + }, { + Name: "logs", + ShortName: "", + Usage: "service logs", + HideHelp: true, + Action: dummy, + }, + // settings + { + Name: "get", + ShortName: "", + Usage: "get config value(s)", + HideHelp: true, + Action: dummy, + }, { + Name: "set", + ShortName: "", + Usage: "set config value(s)", + HideHelp: true, + Action: dummy, + }, + // complete config + { + Name: "export", + ShortName: "", + Usage: "export config", + HideHelp: true, + Action: dummy, + }, { + Name: "apply", + ShortName: "", + Usage: "apply service&config changes", + HideHelp: true, + Action: dummy, + }, { + Name: "validate", + ShortName: "", + Usage: "validate config / service file", + HideHelp: true, + Action: dummy, + }, + // old.. + { + Name: "old", + ShortName: "o", + Usage: "old Command line (deprecated, will be removed in future)", HideHelp: true, - Subcommands: configSubcommands(), + Subcommands: originalCli, }, - { - Name: "console", - Usage: "manage which console container is used", - HideHelp: true, - Subcommands: consoleSubcommands(), - }, - { - Name: "console-init", - Hidden: true, - HideHelp: true, - SkipFlagParsing: true, - Action: consoleInitAction, - }, - { - Name: "dev", - Hidden: true, - HideHelp: true, - SkipFlagParsing: true, - Action: devAction, - }, - { - Name: "docker-init", - Hidden: true, - HideHelp: true, - SkipFlagParsing: true, - Action: dockerInitAction, - }, - { - Name: "engine", - Usage: "manage which Docker engine is used", - HideHelp: true, - Subcommands: engineSubcommands(), - }, - { - Name: "entrypoint", - Hidden: true, - HideHelp: true, - SkipFlagParsing: true, - Action: entrypointAction, - }, - { - Name: "env", - Hidden: true, - HideHelp: true, - SkipFlagParsing: true, - Action: envAction, - }, - service.Commands(), - { - Name: "os", - Usage: "operating system upgrade/downgrade", - HideHelp: true, - Subcommands: osSubcommands(), - }, - { - Name: "preload-images", - Hidden: true, - HideHelp: true, - SkipFlagParsing: true, - Action: preloadImagesAction, - }, - { - Name: "switch-console", - Hidden: true, - HideHelp: true, - SkipFlagParsing: true, - Action: switchConsoleAction, - }, - { - Name: "tls", - Usage: "setup tls configuration", - HideHelp: true, - Subcommands: tlsConfCommands(), - }, - { - Name: "udev-settle", - Hidden: true, - HideHelp: true, - SkipFlagParsing: true, - Action: udevSettleAction, - }, - { - Name: "user-docker", - Hidden: true, - HideHelp: true, - SkipFlagParsing: true, - Action: userDockerAction, - }, - installCommand, - selinuxCommand(), } + app.Commands = append(app.Commands, hiddenInternalCommands...) app.Run(os.Args) } + +func dummy(c *cli.Context) error { + return nil +} + +func fetchServices(c *cli.Context) error { + // fetch all the index.yml files, and the service.yml files that they refer to + // and put into a cache dir. + // Q - should there be one dir per index.yml so that you can have more than one service with the same name.. + + //TODO: need a --purge + //TODO: also need to fetch the rancher/os choices + + return network.FetchAllServices() +} + +func listServices(c *cli.Context) error { + //get the current cfg, and the make a cfg with all cached services + //then iterate through, listing all possible services, and what version is running, vs what version they could run + //Can't just merge current cfg and cached services, as we lose service.yml version info + currentConfig := config.LoadConfig() + cachedConfigs := GetAllServices() + // TODO: sort them! + fmt.Printf("Running\n") + for serviceName, serviceConfig := range currentConfig.Rancher.Services { + fmt.Printf("\t%s: %s\n", serviceName, serviceConfig.Image) + if len(cachedConfigs[serviceName]) > 0 { + fmt.Printf("\t\tAlternatives: ") + for serviceLongName, _ := range cachedConfigs[serviceName] { + fmt.Printf("%s, ", serviceLongName) + } + fmt.Printf("\n") + } + } + fmt.Printf("Available\n") + for serviceName, service := range cachedConfigs { + if _, ok := currentConfig.Rancher.Services[serviceName]; ok { + continue + } + for serviceLongName, _ := range service { + fmt.Printf("\t%s: %s\n", serviceName, serviceLongName) + } + } + + return nil +} + +func GetAllServices() map[string]map[string]*libcomposeConfig.ServiceConfigV1 { + //result := make(map[string]*libcomposeConfig.ServiceConfig) + result := make(map[string]map[string]*libcomposeConfig.ServiceConfigV1) + + cfg := config.LoadConfig() + for repoName, _ := range cfg.Rancher.Repositories { + indexPath := fmt.Sprintf("%s/index.yml", repoName) + //content, err := network.LoadResource(indexPath, false) + content, err := network.CacheLookup(indexPath) + if err != nil { + log.Errorf("Failed to load %s: %v", indexPath, err) + continue + } + + services := make(map[string][]string) + err = yaml.Unmarshal(content, &services) + if err != nil { + log.Errorf("Failed to unmarshal %s: %v", indexPath, err) + continue + } + for serviceType, serviceList := range services { + for _, serviceLongName := range serviceList { + servicePath := fmt.Sprintf("%s/%s.yml", repoName, serviceLongName) + //log.Infof("loading %s", serviceLongName) + content, err := network.CacheLookup(servicePath) + if err != nil { + log.Errorf("Failed to load %s: %v", servicePath, err) + continue + } + if content, err = ComposeToCloudConfig(content); err != nil { + log.Errorf("Failed to convert compose to cloud-config syntax: %v", err) + continue + } + + p, err := config.ReadConfig(content, true) + if err != nil { + log.Errorf("Failed to load %s : %v", servicePath, err) + } + + // yes, the serviceLongName is really only the yml file name + // and each yml file can contain more than one actual service + for serviceName, service := range p.Rancher.Services { + //service, _ := p.ServiceConfigs.Get(serviceName) + n := fmt.Sprintf("%s/%s", repoName, serviceLongName) + if result[serviceName] == nil { + result[serviceName] = map[string]*libcomposeConfig.ServiceConfigV1{} + } + result[serviceName][n] = service + log.Debugf("loaded %s(%s): %s", n, serviceType, serviceName) + } + + } + } + } + + return result +} + +//TODO: copied from cloudinitsave, move to config. +func ComposeToCloudConfig(bytes []byte) ([]byte, error) { + compose := make(map[interface{}]interface{}) + err := yaml.Unmarshal(bytes, &compose) + if err != nil { + return nil, err + } + + return yaml.Marshal(map[interface{}]interface{}{ + "rancher": map[interface{}]interface{}{ + "services": compose, + }, + }) +} + +var originalCli = []cli.Command{ + { + Name: "config", + ShortName: "c", + Usage: "configure settings", + HideHelp: true, + Subcommands: configSubcommands(), + }, + { + Name: "console", + Usage: "manage which console container is used", + HideHelp: true, + Subcommands: consoleSubcommands(), + }, + { + Name: "engine", + Usage: "manage which Docker engine is used", + HideHelp: true, + Subcommands: engineSubcommands(), + }, + service.Commands(), + { + Name: "os", + Usage: "operating system upgrade/downgrade", + HideHelp: true, + Subcommands: osSubcommands(), + }, + { + Name: "tls", + Usage: "setup tls configuration", + HideHelp: true, + Subcommands: tlsConfCommands(), + }, + installCommand, + selinuxCommand(), +} + +var hiddenInternalCommands = []cli.Command{ + { + Name: "bootstrap", + Hidden: true, + HideHelp: true, + SkipFlagParsing: true, + Action: bootstrapAction, + }, + { + Name: "udev-settle", + Hidden: true, + HideHelp: true, + SkipFlagParsing: true, + Action: udevSettleAction, + }, + { + Name: "user-docker", + Hidden: true, + HideHelp: true, + SkipFlagParsing: true, + Action: userDockerAction, + }, + { + Name: "preload-images", + Hidden: true, + HideHelp: true, + SkipFlagParsing: true, + Action: preloadImagesAction, + }, + { + Name: "switch-console", + Hidden: true, + HideHelp: true, + SkipFlagParsing: true, + Action: switchConsoleAction, + }, + { + Name: "entrypoint", + Hidden: true, + HideHelp: true, + SkipFlagParsing: true, + Action: entrypointAction, + }, + { + Name: "env", + Hidden: true, + HideHelp: true, + SkipFlagParsing: true, + Action: envAction, + }, + { + Name: "console-init", + Hidden: true, + HideHelp: true, + SkipFlagParsing: true, + Action: consoleInitAction, + }, + { + Name: "dev", + Hidden: true, + HideHelp: true, + SkipFlagParsing: true, + Action: devAction, + }, + { + Name: "docker-init", + Hidden: true, + HideHelp: true, + SkipFlagParsing: true, + Action: dockerInitAction, + }, +} diff --git a/config/types.go b/config/types.go index db9267c9..b914ea5b 100755 --- a/config/types.go +++ b/config/types.go @@ -45,6 +45,7 @@ const ( CloudConfigScriptFile = "/var/lib/rancher/conf/cloud-config-script" MetaDataFile = "/var/lib/rancher/conf/metadata" CloudConfigFile = "/var/lib/rancher/conf/cloud-config.yml" + CacheDirectory = "/var/lib/rancher/cache/" ) var ( diff --git a/main.go b/main.go old mode 100644 new mode 100755 index fc84e4fe..e5707615 --- a/main.go +++ b/main.go @@ -1,11 +1,11 @@ package main import ( - "github.com/containernetworking/cni/plugins/ipam/host-local" + hostlocal "github.com/containernetworking/cni/plugins/ipam/host-local" "github.com/containernetworking/cni/plugins/main/bridge" "github.com/docker/docker/docker" "github.com/docker/docker/pkg/reexec" - "github.com/rancher/cniglue" + glue "github.com/rancher/cniglue" "github.com/rancher/os/cmd/cloudinitexecute" "github.com/rancher/os/cmd/cloudinitsave" "github.com/rancher/os/cmd/control" diff --git a/scripts/tar-images b/scripts/tar-images index 93b3afdd..634c0512 100755 --- a/scripts/tar-images +++ b/scripts/tar-images @@ -3,7 +3,7 @@ set -e cd $(dirname $0)/.. -IMAGES=$(bin/host_ros c images -i build/initrd/usr/share/ros/os-config.yml) +IMAGES=$(bin/host_ros old c images -i build/initrd/usr/share/ros/os-config.yml) for i in $IMAGES; do if [ "${FORCE_PULL}" = "1" ] || ! docker inspect $i >/dev/null 2>&1; then docker pull $i diff --git a/scripts/template b/scripts/template index ad41a5ab..3f878aa1 100755 --- a/scripts/template +++ b/scripts/template @@ -7,4 +7,4 @@ cd $(dirname $0)/.. OUTPUT=build/initrd/usr/share/ros mkdir -p $OUTPUT -./bin/host_ros c generate < os-config.tpl.yml > $OUTPUT/os-config.yml +./bin/host_ros old c generate < os-config.tpl.yml > $OUTPUT/os-config.yml diff --git a/util/network/cache.go b/util/network/cache.go old mode 100644 new mode 100755 index cba17518..267a5528 --- a/util/network/cache.go +++ b/util/network/cache.go @@ -5,41 +5,45 @@ import ( "encoding/hex" "io/ioutil" "os" + "path/filepath" + "github.com/rancher/os/config" "github.com/rancher/os/log" ) -const ( - cacheDirectory = "/var/lib/rancher/cache/" -) - func locationHash(location string) string { sum := md5.Sum([]byte(location)) return hex.EncodeToString(sum[:]) } -func cacheLookup(location string) []byte { - cacheFile := cacheDirectory + locationHash(location) +func CacheLookup(location string) ([]byte, error) { + cacheFile := filepath.Join(config.CacheDirectory, location) bytes, err := ioutil.ReadFile(cacheFile) if err == nil { log.Debugf("Using cached file: %s", cacheFile) - return bytes + return bytes, nil } - return nil + log.Debugf("Cached file not found: %s", cacheFile) + return nil, err } -func cacheAdd(location string, data []byte) { - tempFile, err := ioutil.TempFile(cacheDirectory, "") +func cacheAdd(location string, data []byte) error { + os.MkdirAll(config.CacheDirectory, 0755) + tempFile, err := ioutil.TempFile(config.CacheDirectory, "") if err != nil { - return + return err } defer os.Remove(tempFile.Name()) _, err = tempFile.Write(data) if err != nil { - return + return err } - cacheFile := cacheDirectory + locationHash(location) + cacheFile := filepath.Join(config.CacheDirectory, location) + cacheDir := filepath.Dir(cacheFile) + log.Debugf("writing %s to %s", cacheFile, cacheDir) + os.MkdirAll(cacheDir, 0755) os.Rename(tempFile.Name(), cacheFile) + return nil } diff --git a/util/network/network.go b/util/network/network.go old mode 100644 new mode 100755 index 6223c339..c54e2c1d --- a/util/network/network.go +++ b/util/network/network.go @@ -32,6 +32,48 @@ func GetEngines(urls []string) ([]string, error) { return getServices(urls, "engines") } +func FetchAllServices() error { + cfg := config.LoadConfig() + for name, url := range cfg.Rancher.Repositories { + log.Infof("repo index %s: %v", name, url.URL) + indexURL := fmt.Sprintf("%s/index.yml", url.URL) + content, err := loadFromNetwork(indexURL) + if err != nil { + log.Errorf("Failed to load %s: %v", indexURL, err) + continue + } + // save the index file to the cache dir + cacheAdd(fmt.Sprintf("%s/index.yml", name), content) + // load it, and then download each service file and cache too + services := make(map[string][]string) + err = yaml.Unmarshal(content, &services) + if err != nil { + log.Errorf("Failed to unmarshal %s: %v", indexURL, err) + continue + } + + for serviceType, serviceList := range services { + for _, serviceName := range serviceList { + fmt.Printf("\t%s is type %s from %s\n", serviceName, serviceType, name) + serviceURL := serviceURL(url.URL, serviceName) + content, err := loadFromNetwork(serviceURL) + if err != nil { + log.Errorf("Failed to load %s: %v", serviceURL, err) + continue + } + // save the service file to the cache dir + if err = cacheAdd(fmt.Sprintf("%s/%s.yml", name, serviceName), content); err != nil { + log.Errorf("cacheAdd: %s", err) + } + //display which services are new, and which are updated from previous cache + //and `listServices` will need to compare the cached servcies with the version that was enabled/up + //also list the services that are no longer updated (ie, purge candidates) + } + } + } + return nil +} + func getServices(urls []string, key string) ([]string, error) { result := []string{} @@ -80,11 +122,11 @@ func SetProxyEnvironmentVariables(cfg *config.CloudConfig) { } func loadFromNetwork(location string) ([]byte, error) { - bytes := cacheLookup(location) - if bytes != nil { - return bytes, nil - } - + /* bytes := cacheLookup(location) + if bytes != nil { + return bytes, nil + } + */ cfg := config.LoadConfig() SetProxyEnvironmentVariables(cfg)