installer: Repositoris now support multiple uris

pkg/installer/client/http.go

@@ -39,49 +39,73 @@ func NewHttpClient(r RepoData) *HttpClient {
 }
 
 func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+	var file *os.File = nil
+	var u *url.URL = nil
+
 	artifactName := path.Base(artifact.GetPath())
-	Info("Downloading artifact", artifactName, "from", c.RepoData.Uri)
+	ok := false
 
 	temp, err := ioutil.TempDir(os.TempDir(), "tree")
 	if err != nil {
 		return nil, err
 	}
 
-	file, err := ioutil.TempFile(temp, "HttpClient")
+	for _, uri := range c.RepoData.Urls {
+		Info("Downloading artifact", artifactName, "from", uri)
+
+		file, err = ioutil.TempFile(temp, "HttpClient")
 		if err != nil {
-		return nil, err
+			continue
 		}
 
-	u, err := url.Parse(c.RepoData.Uri)
+		u, err = url.Parse(uri)
 		if err != nil {
-		return nil, err
+			continue
 		}
 		u.Path = path.Join(u.Path, artifactName)
 
 		_, err = grab.Get(temp, u.String())
 		if err != nil {
-		return nil, err
+			continue
 		}
 
 		err = helpers.CopyFile(filepath.Join(temp, artifactName), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
+
+	if !ok {
+		return nil, err
+	}
+
 	newart := artifact
 	newart.SetPath(file.Name())
 	return newart, nil
 }
 
 func (c *HttpClient) DownloadFile(name string) (string, error) {
+	var file *os.File = nil
+	var u *url.URL = nil
+	ok := false
+
 	temp, err := ioutil.TempDir(os.TempDir(), "tree")
 	if err != nil {
 		return "", err
 	}
-	file, err := ioutil.TempFile(os.TempDir(), "HttpClient")
+
+	for _, uri := range c.RepoData.Urls {
+
+		file, err = ioutil.TempFile(os.TempDir(), "HttpClient")
 		if err != nil {
-		return "", err
+			continue
 		}
 		//defer os.Remove(file.Name())
-	u, err := url.Parse(c.RepoData.Uri)
+		u, err = url.Parse(uri)
 		if err != nil {
-		return "", err
+			continue
 		}
 		u.Path = path.Join(u.Path, name)
 
@@ -89,10 +113,20 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 
 		_, err = grab.Get(temp, u.String())
 		if err != nil {
-		return "", err
+			continue
 		}
 
 		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
+
+	if !ok {
+		return "", err
+	}
 
 	return file.Name(), err
 }

pkg/installer/client/http_test.go

@@ -44,7 +44,7 @@ var _ = Describe("Http client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewHttpClient(RepoData{Uri: ts.URL})
+			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
 			path, err := c.DownloadFile("test.txt")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path)).To(Equal("test"))
@@ -62,7 +62,7 @@ var _ = Describe("Http client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewHttpClient(RepoData{Uri: ts.URL})
+			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
 			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path.GetPath())).To(Equal("test"))

pkg/installer/client/interface.go

@@ -16,5 +16,5 @@
 package client
 
 type RepoData struct {
-	Uri string
+	Urls []string
 }

pkg/installer/client/local.go

@@ -36,29 +36,58 @@ func NewLocalClient(r RepoData) *LocalClient {
 }
 
 func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+	var err error
+	var file *os.File = nil
+
 	artifactName := path.Base(artifact.GetPath())
-	Info("Downloading artifact", artifactName, "from", c.RepoData.Uri)
-	file, err := ioutil.TempFile(os.TempDir(), "localclient")
+	ok := false
+	for _, uri := range c.RepoData.Urls {
+		Info("Downloading artifact", artifactName, "from", uri)
+		file, err = ioutil.TempFile(os.TempDir(), "localclient")
 		if err != nil {
-		return nil, err
+			continue
 		}
 		//defer os.Remove(file.Name())
+		err = helpers.CopyFile(filepath.Join(uri, artifactName), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
+
+	if !ok {
+		return nil, err
+	}
 
-	err = helpers.CopyFile(filepath.Join(c.RepoData.Uri, artifactName), file.Name())
 	newart := artifact
 	newart.SetPath(file.Name())
 	return newart, nil
 }
 func (c *LocalClient) DownloadFile(name string) (string, error) {
-	Info("Downloading file", name, "from", c.RepoData.Uri)
+	var err error
+	var file *os.File = nil
 
-	file, err := ioutil.TempFile(os.TempDir(), "localclient")
+	ok := false
+	for _, uri := range c.RepoData.Urls {
+		Info("Downloading file", name, "from", uri)
+		file, err = ioutil.TempFile(os.TempDir(), "localclient")
 		if err != nil {
-		return "", err
+			continue
 		}
 		//defer os.Remove(file.Name())
 
-	err = helpers.CopyFile(filepath.Join(c.RepoData.Uri, name), file.Name())
+		err = helpers.CopyFile(filepath.Join(uri, name), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
 
-	return file.Name(), err
+	if ok {
+		return file.Name(), nil
+	}
+
+	return "", err
 }

pkg/installer/client/local_test.go

@@ -39,7 +39,7 @@ var _ = Describe("Local client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewLocalClient(RepoData{Uri: tmpdir})
+			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
 			path, err := c.DownloadFile("test.txt")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path)).To(Equal("test"))
@@ -55,7 +55,7 @@ var _ = Describe("Local client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewLocalClient(RepoData{Uri: tmpdir})
+			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
 			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path.GetPath())).To(Equal("test"))

pkg/installer/installer_test.go

@@ -16,6 +16,7 @@
 package installer_test
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -92,7 +93,7 @@ var _ = Describe("Installer", func() {
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
-			Expect(repo.GetUri()).To(Equal(tmpdir))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
@@ -103,16 +104,18 @@ var _ = Describe("Installer", func() {
 			repo2, err := NewLuetRepositoryFromYaml([]byte(`
 name: "test"
 type: "disk"
-uri: "`+tmpdir+`"
+urls:
+  - "`+tmpdir+`"
 `), pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 
 			inst.Repositories(Repositories{repo2})
-			Expect(repo.GetUri()).To(Equal(tmpdir))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 			systemDB := pkg.NewInMemoryDatabase(false)
 			system := &System{Database: systemDB, Target: fakeroot}
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			fmt.Println("ERR ", err)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
@@ -204,7 +207,7 @@ uri: "`+tmpdir+`"
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
-			Expect(repo.GetUri()).To(Equal(tmpdir))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
@@ -215,12 +218,13 @@ uri: "`+tmpdir+`"
 			repo2, err := NewLuetRepositoryFromYaml([]byte(`
 name: "test"
 type: "disk"
-uri: "`+tmpdir+`"
+urls:
+  - "`+tmpdir+`"
 `), pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 
 			inst.Repositories(Repositories{repo2})
-			Expect(repo.GetUri()).To(Equal(tmpdir))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
 			bolt, err := ioutil.TempDir("", "db")
@@ -311,7 +315,7 @@ uri: "`+tmpdir+`"
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
-			Expect(repo.GetUri()).To(Equal(tmpdir))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
@@ -322,12 +326,13 @@ uri: "`+tmpdir+`"
 			repo2, err := NewLuetRepositoryFromYaml([]byte(`
 name: "test"
 type: "disk"
-uri: "`+tmpdir+`"
+urls:
+  - "`+tmpdir+`"
 `), pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 
 			inst.Repositories(Repositories{repo2})
-			Expect(repo.GetUri()).To(Equal(tmpdir))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
 			bolt, err := ioutil.TempDir("", "db")
@@ -425,7 +430,7 @@ uri: "`+tmpdir+`"
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
-			Expect(repo.GetUri()).To(Equal(tmpdir))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
@@ -436,12 +441,13 @@ uri: "`+tmpdir+`"
 			repo2, err := NewLuetRepositoryFromYaml([]byte(`
 name: "test"
 type: "disk"
-uri: "`+tmpdir+`"
+urls:
+  - "`+tmpdir+`"
 `), pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 
 			inst.Repositories(Repositories{repo2})
-			Expect(repo.GetUri()).To(Equal(tmpdir))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
 			bolt, err := ioutil.TempDir("", "db")

pkg/installer/interface.go

@@ -38,8 +38,9 @@ type Repositories []Repository
 
 type Repository interface {
 	GetName() string
-	GetUri() string
-	SetUri(string)
+	GetUrls() []string
+	SetUrls([]string)
+	AddUrl(string)
 	GetPriority() int
 	GetIndex() compiler.ArtifactIndex
 	GetTree() tree.Builder

pkg/installer/repository.go

@@ -35,7 +35,7 @@ import (
 
 type LuetRepository struct {
 	Name     string                 `json:"name"`
-	Uri      string                 `json:"uri"`
+	Urls     []string               `json:"urls"`
 	Priority int                    `json:"priority"`
 	Index    compiler.ArtifactIndex `json:"index"`
 	Tree     tree.Builder           `json:"-"`
@@ -45,7 +45,7 @@ type LuetRepository struct {
 
 type LuetRepositorySerialized struct {
 	Name     string                      `json:"name"`
-	Uri      string                      `json:"uri"`
+	Urls     []string                    `json:"urls"`
 	Priority int                         `json:"priority"`
 	Index    []*compiler.PackageArtifact `json:"index"`
 	Type     string                      `json:"type"`
@@ -67,7 +67,7 @@ func GenerateRepository(name, uri, t string, priority int, src, treeDir string,
 }
 
 func NewLuetRepository(name, uri, t string, priority int, art []compiler.Artifact, builder tree.Builder) Repository {
-	return &LuetRepository{Index: art, Type: t, Tree: builder, Name: name, Uri: uri, Priority: priority}
+	return &LuetRepository{Index: art, Type: t, Tree: builder, Name: name, Urls: []string{uri}, Priority: priority}
 }
 
 func NewLuetRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, error) {
@@ -78,7 +78,7 @@ func NewLuetRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository,
 		return nil, err
 	}
 	r.Name = p.Name
-	r.Uri = p.Uri
+	r.Urls = p.Urls
 	r.Priority = p.Priority
 	r.Type = p.Type
 	i := compiler.ArtifactIndex{}
@@ -143,11 +143,14 @@ func (r *LuetRepository) SetType(p string) {
 	r.Type = p
 }
 
-func (r *LuetRepository) SetUri(p string) {
-	r.Uri = p
+func (r *LuetRepository) AddUrl(p string) {
+	r.Urls = append(r.Urls, p)
 }
-func (r *LuetRepository) GetUri() string {
-	return r.Uri
+func (r *LuetRepository) GetUrls() []string {
+	return r.Urls
+}
+func (r *LuetRepository) SetUrls(urls []string) {
+	r.Urls = urls
 }
 func (r *LuetRepository) GetPriority() int {
 	return r.Priority
@@ -191,9 +194,9 @@ func (r *LuetRepository) Write(dst string) error {
 func (r *LuetRepository) Client() Client {
 	switch r.GetType() {
 	case "disk":
-		return client.NewLocalClient(client.RepoData{Uri: r.GetUri()})
+		return client.NewLocalClient(client.RepoData{Urls: r.GetUrls()})
 	case "http":
-		return client.NewHttpClient(client.RepoData{Uri: r.GetUri()})
+		return client.NewHttpClient(client.RepoData{Urls: r.GetUrls()})
 	}
 
 	return nil
@@ -205,7 +208,7 @@ func (r *LuetRepository) Sync() (Repository, error) {
 	}
 	file, err := c.DownloadFile("repository.yaml")
 	if err != nil {
-		return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri())
+		return nil, errors.Wrap(err, "While downloading repository.yaml")
 	}
 	dat, err := ioutil.ReadFile(file)
 	if err != nil {
@@ -222,7 +225,7 @@ func (r *LuetRepository) Sync() (Repository, error) {
 
 	archivetree, err := c.DownloadFile("tree.tar")
 	if err != nil {
-		return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri())
+		return nil, errors.Wrap(err, "While downloading repository.yaml")
 	}
 	defer os.RemoveAll(archivetree) // clean up
 
@@ -251,7 +254,7 @@ func (r *LuetRepository) Sync() (Repository, error) {
 	}
 	repo.SetTree(reciper)
 	repo.SetTreePath(treefs)
-	repo.SetUri(r.GetUri())
+	repo.SetUrls(r.GetUrls())
 
 	return repo, nil
 }