From 69fe4bb61917e35ee6335dc6059171b6d2eff084 Mon Sep 17 00:00:00 2001 From: Josh Curl Date: Wed, 25 May 2016 09:40:28 -0700 Subject: [PATCH] Cache remote repos and service definitions --- cmd/control/service.go | 4 +- compose/project.go | 15 ++--- os-config.tpl.yml | 1 + util/network/cache.go | 45 +++++++++++++ util/network/network.go | 122 +++++++++++++++++++++++++++++++++++ util/network/network_test.go | 23 +++++++ util/util.go | 77 ---------------------- util/util_test.go | 19 +----- 8 files changed, 201 insertions(+), 105 deletions(-) create mode 100644 util/network/cache.go create mode 100644 util/network/network.go create mode 100644 util/network/network_test.go diff --git a/cmd/control/service.go b/cmd/control/service.go index 99cea16c..36f21264 100644 --- a/cmd/control/service.go +++ b/cmd/control/service.go @@ -11,7 +11,7 @@ import ( "github.com/docker/libcompose/project" "github.com/rancher/os/compose" "github.com/rancher/os/config" - "github.com/rancher/os/util" + "github.com/rancher/os/util/network" ) type projectFactory struct { @@ -178,7 +178,7 @@ func list(c *cli.Context) error { clone[service] = enabled } - services, err := util.GetServices(cfg.Rancher.Repositories.ToArray()) + services, err := network.GetServices(cfg.Rancher.Repositories.ToArray()) if err != nil { logrus.Fatalf("Failed to get services: %v", err) } diff --git a/compose/project.go b/compose/project.go index 28d305a8..61ae0cac 100644 --- a/compose/project.go +++ b/compose/project.go @@ -17,6 +17,7 @@ import ( "github.com/rancher/os/config" rosDocker "github.com/rancher/os/docker" "github.com/rancher/os/util" + "github.com/rancher/os/util/network" ) func CreateService(cfg *config.CloudConfig, name string, serviceConfig *composeConfig.ServiceConfigV1) (project.Service, error) { @@ -209,9 +210,9 @@ func newCoreServiceProject(cfg *config.CloudConfig, useNetwork bool) (*project.P continue } - bytes, err := LoadServiceResource(service, useNetwork, cfg) + bytes, err := network.LoadServiceResource(service, useNetwork, cfg) if err != nil { - if err == util.ErrNoNetwork { + if err == network.ErrNoNetwork { log.Debugf("Can not load %s, networking not enabled", service) } else { log.Errorf("Failed to load %s : %v", service, err) @@ -265,7 +266,7 @@ func StageServices(cfg *config.CloudConfig, services ...string) error { } for _, service := range services { - bytes, err := LoadServiceResource(service, true, cfg) + bytes, err := network.LoadServiceResource(service, true, cfg) if err != nil { return fmt.Errorf("Failed to load %s : %v", service, err) } @@ -277,12 +278,12 @@ func StageServices(cfg *config.CloudConfig, services ...string) error { bytes, err = yaml.Marshal(config.StringifyValues(m)) if err != nil { - fmt.Errorf("Failed to marshal YAML configuration: %s : %v", service, err) + return fmt.Errorf("Failed to marshal YAML configuration: %s : %v", service, err) } err = p.Load(bytes) if err != nil { - fmt.Errorf("Failed to load %s : %v", service, err) + return fmt.Errorf("Failed to load %s : %v", service, err) } } @@ -297,7 +298,3 @@ func StageServices(cfg *config.CloudConfig, services ...string) error { return p.Pull(context.Background()) } - -func LoadServiceResource(name string, network bool, cfg *config.CloudConfig) ([]byte, error) { - return util.LoadResource(name, network, cfg.Rancher.Repositories.ToArray()) -} diff --git a/os-config.tpl.yml b/os-config.tpl.yml index f76675b6..396f464c 100644 --- a/os-config.tpl.yml +++ b/os-config.tpl.yml @@ -297,6 +297,7 @@ rancher: - /lib/modules:/lib/modules - /run:/run - /usr/share/ros:/usr/share/ros + - /var/lib/rancher/cache:/var/lib/rancher/cache - /var/lib/rancher/conf:/var/lib/rancher/conf - /var/lib/rancher:/var/lib/rancher - /var/log:/var/log diff --git a/util/network/cache.go b/util/network/cache.go new file mode 100644 index 00000000..fcf5c22e --- /dev/null +++ b/util/network/cache.go @@ -0,0 +1,45 @@ +package network + +import ( + "crypto/md5" + "encoding/hex" + "io/ioutil" + "os" + + log "github.com/Sirupsen/logrus" +) + +const ( + cacheDirectory = "/var/lib/rancher/cache/" +) + +func locationHash(location string) string { + sum := md5.Sum([]byte(location)) + return hex.EncodeToString(sum[:]) +} + +func cacheLookup(location string) []byte { + cacheFile := cacheDirectory + locationHash(location) + bytes, err := ioutil.ReadFile(cacheFile) + if err == nil { + log.Debugf("Using cached file: %s", cacheFile) + return bytes + } + return nil +} + +func cacheAdd(location string, data []byte) { + tempFile, err := ioutil.TempFile(cacheDirectory, "") + if err != nil { + return + } + defer os.Remove(tempFile.Name()) + + _, err = tempFile.Write(data) + if err != nil { + return + } + + cacheFile := cacheDirectory + locationHash(location) + os.Rename(tempFile.Name(), cacheFile) +} diff --git a/util/network/network.go b/util/network/network.go new file mode 100644 index 00000000..6940c0e7 --- /dev/null +++ b/util/network/network.go @@ -0,0 +1,122 @@ +package network + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + "strings" + + yaml "github.com/cloudfoundry-incubator/candiedyaml" + + log "github.com/Sirupsen/logrus" + "github.com/rancher/os/config" +) + +var ( + ErrNoNetwork = errors.New("Networking not available to load resource") + ErrNotFound = errors.New("Failed to find resource") +) + +func GetServices(urls []string) ([]string, error) { + result := []string{} + + for _, url := range urls { + indexUrl := fmt.Sprintf("%s/index.yml", url) + content, err := LoadResource(indexUrl, true) + if err != nil { + log.Errorf("Failed to load %s: %v", indexUrl, err) + continue + } + + services := make(map[string][]string) + err = yaml.Unmarshal(content, &services) + if err != nil { + log.Errorf("Failed to unmarshal %s: %v", indexUrl, err) + continue + } + + if list, ok := services["services"]; ok { + result = append(result, list...) + } + } + + return result, nil +} + +func retryHttp(f func() (*http.Response, error), times int) (resp *http.Response, err error) { + for i := 0; i < times; i++ { + if resp, err = f(); err == nil { + return + } + log.Warnf("Error making HTTP request: %s. Retrying", err) + } + return +} + +func loadFromNetwork(location string, network bool) ([]byte, error) { + bytes := cacheLookup(location) + if bytes != nil { + return bytes, nil + } + + resp, err := retryHttp(func() (*http.Response, error) { + return http.Get(location) + }, 8) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("non-200 http response: %d", resp.StatusCode) + } + defer resp.Body.Close() + + bytes, err = ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + cacheAdd(location, bytes) + + return bytes, nil +} + +func LoadResource(location string, network bool) ([]byte, error) { + if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") { + if !network { + return nil, ErrNoNetwork + } + return loadFromNetwork(location, network) + } else if strings.HasPrefix(location, "/") { + return ioutil.ReadFile(location) + } + + return nil, ErrNotFound +} + +func serviceUrl(url, name string) string { + return fmt.Sprintf("%s/%s/%s.yml", url, name[0:1], name) +} + +func LoadServiceResource(name string, useNetwork bool, cfg *config.CloudConfig) ([]byte, error) { + bytes, err := LoadResource(name, useNetwork) + if err == nil { + log.Debugf("Loaded %s from %s", name, name) + return bytes, nil + } + if err == ErrNoNetwork || !useNetwork { + return nil, ErrNoNetwork + } + + urls := cfg.Rancher.Repositories.ToArray() + for _, url := range urls { + serviceUrl := serviceUrl(url, name) + bytes, err := LoadResource(serviceUrl, useNetwork) + if err == nil { + log.Debugf("Loaded %s from %s", name, serviceUrl) + return bytes, nil + } + } + + return nil, ErrNotFound +} diff --git a/util/network/network_test.go b/util/network/network_test.go new file mode 100644 index 00000000..4dddbd03 --- /dev/null +++ b/util/network/network_test.go @@ -0,0 +1,23 @@ +package network + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func NoTestLoadResourceSimple(t *testing.T) { + assert := require.New(t) + + expected := `services: +- debian-console +- ubuntu-console +` + expected = strings.TrimSpace(expected) + + b, e := LoadResource("https://raw.githubusercontent.com/rancher/os-services/v0.3.4/index.yml", true) + + assert.Nil(e) + assert.Equal(expected, strings.TrimSpace(string(b))) +} diff --git a/util/util.go b/util/util.go index c6b7365a..ee1d0ffb 100644 --- a/util/util.go +++ b/util/util.go @@ -2,11 +2,8 @@ package util import ( "bytes" - "errors" - "fmt" "io" "io/ioutil" - "net/http" "os" "strings" @@ -17,11 +14,6 @@ import ( "reflect" ) -var ( - ErrNoNetwork = errors.New("Networking not available to load resource") - ErrNotFound = errors.New("Failed to find resource") -) - type AnyMap map[interface{}]interface{} func Contains(values []string, value string) bool { @@ -256,32 +248,6 @@ func ToStrings(data []interface{}) []string { return result } -func GetServices(urls []string) ([]string, error) { - result := []string{} - - for _, url := range urls { - indexUrl := fmt.Sprintf("%s/index.yml", url) - content, err := LoadResource(indexUrl, true, []string{}) - if err != nil { - log.Errorf("Failed to load %s: %v", indexUrl, err) - continue - } - - services := make(map[string][]string) - err = yaml.Unmarshal(content, &services) - if err != nil { - log.Errorf("Failed to unmarshal %s: %v", indexUrl, err) - continue - } - - if list, ok := services["services"]; ok { - result = append(result, list...) - } - } - - return result, nil -} - func DirLs(dir string) ([]interface{}, error) { result := []interface{}{} files, err := ioutil.ReadDir(dir) @@ -294,49 +260,6 @@ func DirLs(dir string) ([]interface{}, error) { return result, nil } -func retryHttp(f func() (*http.Response, error), times int) (resp *http.Response, err error) { - for i := 0; i < times; i++ { - if resp, err = f(); err == nil { - return - } - log.Warnf("Error making HTTP request: %s. Retrying", err) - } - return -} - -func LoadResource(location string, network bool, urls []string) ([]byte, error) { - var bytes []byte - err := ErrNotFound - - if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") { - if !network { - return nil, ErrNoNetwork - } - resp, err := retryHttp(func() (*http.Response, error) { return http.Get(location) }, 8) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("non-200 http response: %d", resp.StatusCode) - } - defer resp.Body.Close() - return ioutil.ReadAll(resp.Body) - } else if strings.HasPrefix(location, "/") { - return ioutil.ReadFile(location) - } else if len(location) > 0 { - for _, url := range urls { - ymlUrl := fmt.Sprintf("%s/%s/%s.yml", url, location[0:1], location) - bytes, err = LoadResource(ymlUrl, network, []string{}) - if err == nil { - log.Debugf("Loaded %s from %s", location, ymlUrl) - return bytes, nil - } - } - } - - return nil, err -} - func Map2KVPairs(m map[string]string) []string { r := make([]string, 0, len(m)) for k, v := range m { diff --git a/util/util_test.go b/util/util_test.go index 53c0679d..37327081 100644 --- a/util/util_test.go +++ b/util/util_test.go @@ -1,9 +1,9 @@ package util import ( - "github.com/stretchr/testify/require" - "strings" "testing" + + "github.com/stretchr/testify/require" ) type testCloudConfig struct { @@ -180,18 +180,3 @@ func TestMapsUnion(t *testing.T) { } assert.Equal(expected, MapsUnion(m0, m1)) } - -func NoTestLoadResourceSimple(t *testing.T) { - assert := require.New(t) - - expected := `services: -- debian-console -- ubuntu-console -` - expected = strings.TrimSpace(expected) - - b, e := LoadResource("https://raw.githubusercontent.com/rancher/os-services/v0.3.4/index.yml", true, []string{}) - - assert.Nil(e) - assert.Equal(expected, strings.TrimSpace(string(b))) -}