diff --git a/pkg/installer/client/http.go b/pkg/installer/client/http.go index b39db3c1..1068294e 100644 --- a/pkg/installer/client/http.go +++ b/pkg/installer/client/http.go @@ -39,60 +39,94 @@ func NewHttpClient(r RepoData) *HttpClient { } func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) { + var file *os.File = nil + var u *url.URL = nil + artifactName := path.Base(artifact.GetPath()) - Info("Downloading artifact", artifactName, "from", c.RepoData.Uri) + ok := false temp, err := ioutil.TempDir(os.TempDir(), "tree") if err != nil { return nil, err } - file, err := ioutil.TempFile(temp, "HttpClient") - if err != nil { + for _, uri := range c.RepoData.Urls { + Info("Downloading artifact", artifactName, "from", uri) + + file, err = ioutil.TempFile(temp, "HttpClient") + if err != nil { + continue + } + + u, err = url.Parse(uri) + if err != nil { + continue + } + u.Path = path.Join(u.Path, artifactName) + + _, err = grab.Get(temp, u.String()) + if err != nil { + continue + } + + err = helpers.CopyFile(filepath.Join(temp, artifactName), file.Name()) + if err != nil { + continue + } + ok = true + break + } + + if !ok { return nil, err } - u, err := url.Parse(c.RepoData.Uri) - if err != nil { - return nil, err - } - u.Path = path.Join(u.Path, artifactName) - - _, err = grab.Get(temp, u.String()) - if err != nil { - return nil, err - } - - err = helpers.CopyFile(filepath.Join(temp, artifactName), file.Name()) newart := artifact newart.SetPath(file.Name()) return newart, nil } func (c *HttpClient) DownloadFile(name string) (string, error) { + var file *os.File = nil + var u *url.URL = nil + ok := false + temp, err := ioutil.TempDir(os.TempDir(), "tree") if err != nil { return "", err } - file, err := ioutil.TempFile(os.TempDir(), "HttpClient") - if err != nil { - return "", err - } - //defer os.Remove(file.Name()) - u, err := url.Parse(c.RepoData.Uri) - if err != nil { - return "", err - } - u.Path = path.Join(u.Path, name) - Info("Downloading", u.String()) + for _, uri := range c.RepoData.Urls { - _, err = grab.Get(temp, u.String()) - if err != nil { - return "", err + file, err = ioutil.TempFile(os.TempDir(), "HttpClient") + if err != nil { + continue + } + //defer os.Remove(file.Name()) + u, err = url.Parse(uri) + if err != nil { + continue + } + u.Path = path.Join(u.Path, name) + + Info("Downloading", u.String()) + + _, err = grab.Get(temp, u.String()) + if err != nil { + continue + } + + err = helpers.CopyFile(filepath.Join(temp, name), file.Name()) + if err != nil { + continue + } + ok = true + break } - err = helpers.CopyFile(filepath.Join(temp, name), file.Name()) + if !ok { + return "", err + } return file.Name(), err } diff --git a/pkg/installer/client/http_test.go b/pkg/installer/client/http_test.go index ab124f32..59057754 100644 --- a/pkg/installer/client/http_test.go +++ b/pkg/installer/client/http_test.go @@ -44,7 +44,7 @@ var _ = Describe("Http client", func() { err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm) Expect(err).ToNot(HaveOccurred()) - c := NewHttpClient(RepoData{Uri: ts.URL}) + c := NewHttpClient(RepoData{Urls: []string{ts.URL}}) path, err := c.DownloadFile("test.txt") Expect(err).ToNot(HaveOccurred()) Expect(helpers.Read(path)).To(Equal("test")) @@ -62,7 +62,7 @@ var _ = Describe("Http client", func() { err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm) Expect(err).ToNot(HaveOccurred()) - c := NewHttpClient(RepoData{Uri: ts.URL}) + c := NewHttpClient(RepoData{Urls: []string{ts.URL}}) path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"}) Expect(err).ToNot(HaveOccurred()) Expect(helpers.Read(path.GetPath())).To(Equal("test")) diff --git a/pkg/installer/client/interface.go b/pkg/installer/client/interface.go index 7085b3af..1d142b3c 100644 --- a/pkg/installer/client/interface.go +++ b/pkg/installer/client/interface.go @@ -16,5 +16,5 @@ package client type RepoData struct { - Uri string -} \ No newline at end of file + Urls []string +} diff --git a/pkg/installer/client/local.go b/pkg/installer/client/local.go index 478a4cb4..888cf6b5 100644 --- a/pkg/installer/client/local.go +++ b/pkg/installer/client/local.go @@ -36,29 +36,58 @@ func NewLocalClient(r RepoData) *LocalClient { } func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) { + var err error + var file *os.File = nil + artifactName := path.Base(artifact.GetPath()) - Info("Downloading artifact", artifactName, "from", c.RepoData.Uri) - file, err := ioutil.TempFile(os.TempDir(), "localclient") - if err != nil { + ok := false + for _, uri := range c.RepoData.Urls { + Info("Downloading artifact", artifactName, "from", uri) + file, err = ioutil.TempFile(os.TempDir(), "localclient") + if err != nil { + continue + } + //defer os.Remove(file.Name()) + err = helpers.CopyFile(filepath.Join(uri, artifactName), file.Name()) + if err != nil { + continue + } + ok = true + break + } + + if !ok { return nil, err } - //defer os.Remove(file.Name()) - err = helpers.CopyFile(filepath.Join(c.RepoData.Uri, artifactName), file.Name()) newart := artifact newart.SetPath(file.Name()) return newart, nil } func (c *LocalClient) DownloadFile(name string) (string, error) { - Info("Downloading file", name, "from", c.RepoData.Uri) + var err error + var file *os.File = nil - file, err := ioutil.TempFile(os.TempDir(), "localclient") - if err != nil { - return "", err + ok := false + for _, uri := range c.RepoData.Urls { + Info("Downloading file", name, "from", uri) + file, err = ioutil.TempFile(os.TempDir(), "localclient") + if err != nil { + continue + } + //defer os.Remove(file.Name()) + + err = helpers.CopyFile(filepath.Join(uri, name), file.Name()) + if err != nil { + continue + } + ok = true + break } - //defer os.Remove(file.Name()) - err = helpers.CopyFile(filepath.Join(c.RepoData.Uri, name), file.Name()) + if ok { + return file.Name(), nil + } - return file.Name(), err + return "", err } diff --git a/pkg/installer/client/local_test.go b/pkg/installer/client/local_test.go index 517c0718..3d1521d2 100644 --- a/pkg/installer/client/local_test.go +++ b/pkg/installer/client/local_test.go @@ -39,7 +39,7 @@ var _ = Describe("Local client", func() { err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm) Expect(err).ToNot(HaveOccurred()) - c := NewLocalClient(RepoData{Uri: tmpdir}) + c := NewLocalClient(RepoData{Urls: []string{tmpdir}}) path, err := c.DownloadFile("test.txt") Expect(err).ToNot(HaveOccurred()) Expect(helpers.Read(path)).To(Equal("test")) @@ -55,7 +55,7 @@ var _ = Describe("Local client", func() { err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm) Expect(err).ToNot(HaveOccurred()) - c := NewLocalClient(RepoData{Uri: tmpdir}) + c := NewLocalClient(RepoData{Urls: []string{tmpdir}}) path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"}) Expect(err).ToNot(HaveOccurred()) Expect(helpers.Read(path.GetPath())).To(Equal("test")) diff --git a/pkg/installer/installer_test.go b/pkg/installer/installer_test.go index 8fcbc728..ec46b6df 100644 --- a/pkg/installer/installer_test.go +++ b/pkg/installer/installer_test.go @@ -16,6 +16,7 @@ package installer_test import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -92,7 +93,7 @@ var _ = Describe("Installer", func() { Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue()) - Expect(repo.GetUri()).To(Equal(tmpdir)) + Expect(repo.GetUrls()[0]).To(Equal(tmpdir)) Expect(repo.GetType()).To(Equal("disk")) fakeroot, err := ioutil.TempDir("", "fakeroot") @@ -103,16 +104,18 @@ var _ = Describe("Installer", func() { repo2, err := NewLuetRepositoryFromYaml([]byte(` name: "test" type: "disk" -uri: "`+tmpdir+`" +urls: + - "`+tmpdir+`" `), pkg.NewInMemoryDatabase(false)) Expect(err).ToNot(HaveOccurred()) inst.Repositories(Repositories{repo2}) - Expect(repo.GetUri()).To(Equal(tmpdir)) + Expect(repo.GetUrls()[0]).To(Equal(tmpdir)) Expect(repo.GetType()).To(Equal("disk")) systemDB := pkg.NewInMemoryDatabase(false) system := &System{Database: systemDB, Target: fakeroot} err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system) + fmt.Println("ERR ", err) Expect(err).ToNot(HaveOccurred()) Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue()) @@ -204,7 +207,7 @@ uri: "`+tmpdir+`" Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue()) - Expect(repo.GetUri()).To(Equal(tmpdir)) + Expect(repo.GetUrls()[0]).To(Equal(tmpdir)) Expect(repo.GetType()).To(Equal("disk")) fakeroot, err := ioutil.TempDir("", "fakeroot") @@ -215,12 +218,13 @@ uri: "`+tmpdir+`" repo2, err := NewLuetRepositoryFromYaml([]byte(` name: "test" type: "disk" -uri: "`+tmpdir+`" +urls: + - "`+tmpdir+`" `), pkg.NewInMemoryDatabase(false)) Expect(err).ToNot(HaveOccurred()) inst.Repositories(Repositories{repo2}) - Expect(repo.GetUri()).To(Equal(tmpdir)) + Expect(repo.GetUrls()[0]).To(Equal(tmpdir)) Expect(repo.GetType()).To(Equal("disk")) bolt, err := ioutil.TempDir("", "db") @@ -311,7 +315,7 @@ uri: "`+tmpdir+`" Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue()) - Expect(repo.GetUri()).To(Equal(tmpdir)) + Expect(repo.GetUrls()[0]).To(Equal(tmpdir)) Expect(repo.GetType()).To(Equal("disk")) fakeroot, err := ioutil.TempDir("", "fakeroot") @@ -322,12 +326,13 @@ uri: "`+tmpdir+`" repo2, err := NewLuetRepositoryFromYaml([]byte(` name: "test" type: "disk" -uri: "`+tmpdir+`" +urls: + - "`+tmpdir+`" `), pkg.NewInMemoryDatabase(false)) Expect(err).ToNot(HaveOccurred()) inst.Repositories(Repositories{repo2}) - Expect(repo.GetUri()).To(Equal(tmpdir)) + Expect(repo.GetUrls()[0]).To(Equal(tmpdir)) Expect(repo.GetType()).To(Equal("disk")) bolt, err := ioutil.TempDir("", "db") @@ -425,7 +430,7 @@ uri: "`+tmpdir+`" Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue()) - Expect(repo.GetUri()).To(Equal(tmpdir)) + Expect(repo.GetUrls()[0]).To(Equal(tmpdir)) Expect(repo.GetType()).To(Equal("disk")) fakeroot, err := ioutil.TempDir("", "fakeroot") @@ -436,12 +441,13 @@ uri: "`+tmpdir+`" repo2, err := NewLuetRepositoryFromYaml([]byte(` name: "test" type: "disk" -uri: "`+tmpdir+`" +urls: + - "`+tmpdir+`" `), pkg.NewInMemoryDatabase(false)) Expect(err).ToNot(HaveOccurred()) inst.Repositories(Repositories{repo2}) - Expect(repo.GetUri()).To(Equal(tmpdir)) + Expect(repo.GetUrls()[0]).To(Equal(tmpdir)) Expect(repo.GetType()).To(Equal("disk")) bolt, err := ioutil.TempDir("", "db") diff --git a/pkg/installer/interface.go b/pkg/installer/interface.go index 374b23d2..f3b5f95e 100644 --- a/pkg/installer/interface.go +++ b/pkg/installer/interface.go @@ -38,8 +38,9 @@ type Repositories []Repository type Repository interface { GetName() string - GetUri() string - SetUri(string) + GetUrls() []string + SetUrls([]string) + AddUrl(string) GetPriority() int GetIndex() compiler.ArtifactIndex GetTree() tree.Builder diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go index d0b81533..96bcc057 100644 --- a/pkg/installer/repository.go +++ b/pkg/installer/repository.go @@ -35,7 +35,7 @@ import ( type LuetRepository struct { Name string `json:"name"` - Uri string `json:"uri"` + Urls []string `json:"urls"` Priority int `json:"priority"` Index compiler.ArtifactIndex `json:"index"` Tree tree.Builder `json:"-"` @@ -45,7 +45,7 @@ type LuetRepository struct { type LuetRepositorySerialized struct { Name string `json:"name"` - Uri string `json:"uri"` + Urls []string `json:"urls"` Priority int `json:"priority"` Index []*compiler.PackageArtifact `json:"index"` Type string `json:"type"` @@ -67,7 +67,7 @@ func GenerateRepository(name, uri, t string, priority int, src, treeDir string, } func NewLuetRepository(name, uri, t string, priority int, art []compiler.Artifact, builder tree.Builder) Repository { - return &LuetRepository{Index: art, Type: t, Tree: builder, Name: name, Uri: uri, Priority: priority} + return &LuetRepository{Index: art, Type: t, Tree: builder, Name: name, Urls: []string{uri}, Priority: priority} } func NewLuetRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, error) { @@ -78,7 +78,7 @@ func NewLuetRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, return nil, err } r.Name = p.Name - r.Uri = p.Uri + r.Urls = p.Urls r.Priority = p.Priority r.Type = p.Type i := compiler.ArtifactIndex{} @@ -143,11 +143,14 @@ func (r *LuetRepository) SetType(p string) { r.Type = p } -func (r *LuetRepository) SetUri(p string) { - r.Uri = p +func (r *LuetRepository) AddUrl(p string) { + r.Urls = append(r.Urls, p) } -func (r *LuetRepository) GetUri() string { - return r.Uri +func (r *LuetRepository) GetUrls() []string { + return r.Urls +} +func (r *LuetRepository) SetUrls(urls []string) { + r.Urls = urls } func (r *LuetRepository) GetPriority() int { return r.Priority @@ -191,9 +194,9 @@ func (r *LuetRepository) Write(dst string) error { func (r *LuetRepository) Client() Client { switch r.GetType() { case "disk": - return client.NewLocalClient(client.RepoData{Uri: r.GetUri()}) + return client.NewLocalClient(client.RepoData{Urls: r.GetUrls()}) case "http": - return client.NewHttpClient(client.RepoData{Uri: r.GetUri()}) + return client.NewHttpClient(client.RepoData{Urls: r.GetUrls()}) } return nil @@ -205,7 +208,7 @@ func (r *LuetRepository) Sync() (Repository, error) { } file, err := c.DownloadFile("repository.yaml") if err != nil { - return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri()) + return nil, errors.Wrap(err, "While downloading repository.yaml") } dat, err := ioutil.ReadFile(file) if err != nil { @@ -222,7 +225,7 @@ func (r *LuetRepository) Sync() (Repository, error) { archivetree, err := c.DownloadFile("tree.tar") if err != nil { - return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri()) + return nil, errors.Wrap(err, "While downloading repository.yaml") } defer os.RemoveAll(archivetree) // clean up @@ -251,7 +254,7 @@ func (r *LuetRepository) Sync() (Repository, error) { } repo.SetTree(reciper) repo.SetTreePath(treefs) - repo.SetUri(r.GetUri()) + repo.SetUrls(r.GetUrls()) return repo, nil }