diff --git a/cmd/build.go b/cmd/build.go index baa1b565..2fd03e95 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -22,8 +22,8 @@ import ( "github.com/ghodss/yaml" helpers "github.com/mudler/luet/cmd/helpers" "github.com/mudler/luet/cmd/util" + "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/compiler" - "github.com/mudler/luet/pkg/compiler/types/artifact" compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" "github.com/mudler/luet/pkg/installer" diff --git a/cmd/database/create.go b/cmd/database/create.go index 796d7880..c938165c 100644 --- a/cmd/database/create.go +++ b/cmd/database/create.go @@ -19,7 +19,7 @@ import ( "io/ioutil" "github.com/mudler/luet/cmd/util" - artifact "github.com/mudler/luet/pkg/compiler/types/artifact" + artifact "github.com/mudler/luet/pkg/api/core/types/artifact" . "github.com/mudler/luet/pkg/logger" pkg "github.com/mudler/luet/pkg/package" diff --git a/cmd/pack.go b/cmd/pack.go index 7d3452a2..7a1b307e 100644 --- a/cmd/pack.go +++ b/cmd/pack.go @@ -20,7 +20,7 @@ import ( "time" helpers "github.com/mudler/luet/cmd/helpers" - "github.com/mudler/luet/pkg/compiler/types/artifact" + "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/compiler/types/compression" compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" . "github.com/mudler/luet/pkg/config" diff --git a/go.mod b/go.mod index 63b04a3d..8d7250a1 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( github.com/pelletier/go-toml v1.9.4 // indirect github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f github.com/pkg/errors v0.9.1 + github.com/rancher-sandbox/gofilecache v0.0.0-20210330135715-becdeff5df15 github.com/schollz/progressbar/v3 v3.7.1 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cast v1.4.1 // indirect diff --git a/pkg/compiler/types/artifact/artifact.go b/pkg/api/core/types/artifact/artifact.go similarity index 99% rename from pkg/compiler/types/artifact/artifact.go rename to pkg/api/core/types/artifact/artifact.go index eccc4fcf..f46b6fb6 100644 --- a/pkg/compiler/types/artifact/artifact.go +++ b/pkg/api/core/types/artifact/artifact.go @@ -50,7 +50,7 @@ import ( yaml "gopkg.in/yaml.v2" ) -// When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository +// When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository // which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact. // In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages. type PackageArtifact struct { diff --git a/pkg/compiler/types/artifact/artifact_suite_test.go b/pkg/api/core/types/artifact/artifact_suite_test.go similarity index 100% rename from pkg/compiler/types/artifact/artifact_suite_test.go rename to pkg/api/core/types/artifact/artifact_suite_test.go diff --git a/pkg/compiler/types/artifact/artifact_test.go b/pkg/api/core/types/artifact/artifact_test.go similarity index 99% rename from pkg/compiler/types/artifact/artifact_test.go rename to pkg/api/core/types/artifact/artifact_test.go index 4139d83d..a7979643 100644 --- a/pkg/compiler/types/artifact/artifact_test.go +++ b/pkg/api/core/types/artifact/artifact_test.go @@ -20,10 +20,10 @@ import ( "os" "path/filepath" + . "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/compiler" . "github.com/mudler/luet/pkg/compiler/backend" backend "github.com/mudler/luet/pkg/compiler/backend" - . "github.com/mudler/luet/pkg/compiler/types/artifact" compression "github.com/mudler/luet/pkg/compiler/types/compression" compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" diff --git a/pkg/api/core/types/artifact/cache.go b/pkg/api/core/types/artifact/cache.go new file mode 100644 index 00000000..09aa694b --- /dev/null +++ b/pkg/api/core/types/artifact/cache.go @@ -0,0 +1,63 @@ +// Copyright © 2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package artifact + +import ( + "crypto/sha512" + "fmt" + "os" + "path/filepath" + + "github.com/pkg/errors" + "github.com/rancher-sandbox/gofilecache" +) + +type ArtifactCache struct { + gofilecache.Cache +} + +func NewCache(dir string) *ArtifactCache { + return &ArtifactCache{Cache: *gofilecache.InitCache(dir)} +} + +func (c *ArtifactCache) cacheID(a *PackageArtifact) [64]byte { + fingerprint := filepath.Base(a.Path) + if a.CompileSpec != nil && a.CompileSpec.Package != nil { + fingerprint = a.CompileSpec.Package.GetFingerPrint() + } + if len(a.Checksums) > 0 { + for _, cs := range a.Checksums.List() { + t := cs[0] + result := cs[1] + fingerprint += fmt.Sprintf("+%s:%s", t, result) + } + } + return sha512.Sum512([]byte(fingerprint)) +} + +func (c *ArtifactCache) Get(a *PackageArtifact) (string, error) { + fileName, _, err := c.Cache.GetFile(c.cacheID(a)) + return fileName, err +} + +func (c *ArtifactCache) Put(a *PackageArtifact) (gofilecache.OutputID, int64, error) { + file, err := os.Open(a.Path) + if err != nil { + return [64]byte{}, 0, errors.Wrapf(err, "failed opening %s", a.Path) + } + defer file.Close() + return c.Cache.Put(c.cacheID(a), file) +} diff --git a/pkg/api/core/types/artifact/cache_test.go b/pkg/api/core/types/artifact/cache_test.go new file mode 100644 index 00000000..38a7e3a1 --- /dev/null +++ b/pkg/api/core/types/artifact/cache_test.go @@ -0,0 +1,89 @@ +// Copyright © 2019 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package artifact_test + +import ( + "io/ioutil" + "os" + "path/filepath" + + . "github.com/mudler/luet/pkg/api/core/types/artifact" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" + fileHelper "github.com/mudler/luet/pkg/helpers/file" + pkg "github.com/mudler/luet/pkg/package" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Cache", func() { + Context("CacheID", func() { + It("Get and retrieve files", func() { + tmpdir, err := ioutil.TempDir(os.TempDir(), "test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + + tmpdirartifact, err := ioutil.TempDir(os.TempDir(), "testartifact") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdirartifact) // clean up + + err = ioutil.WriteFile(filepath.Join(tmpdirartifact, "foo"), []byte(string("foo")), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + a := NewPackageArtifact(filepath.Join(tmpdir, "foo.tar.gz")) + err = a.Compress(tmpdirartifact, 1) + Expect(err).ToNot(HaveOccurred()) + + cache := NewCache(tmpdir) + + // Put an artifact in the cache and retrieve it later + // the artifact is NOT hashed so it is referenced just by the path in the cache + _, _, err = cache.Put(a) + Expect(err).ToNot(HaveOccurred()) + + path, err := cache.Get(a) + Expect(err).ToNot(HaveOccurred()) + + b := NewPackageArtifact(path) + + err = b.Unpack(tmpdir, false) + Expect(err).ToNot(HaveOccurred()) + + Expect(fileHelper.Exists(filepath.Join(tmpdir, "foo"))).To(BeTrue()) + + bb, err := ioutil.ReadFile(filepath.Join(tmpdir, "foo")) + Expect(err).ToNot(HaveOccurred()) + + Expect(string(bb)).To(Equal("foo")) + + // After the artifact is hashed, the fingerprint mutates so the cache doesn't see it hitting again + // the test we did above fails as we expect to. + a.Hash() + _, err = cache.Get(a) + Expect(err).To(HaveOccurred()) + + a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "bar"}} + _, _, err = cache.Put(a) + Expect(err).ToNot(HaveOccurred()) + + c := NewPackageArtifact(filepath.Join(tmpdir, "foo.tar.gz")) + c.Hash() + c.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "bar"}} + _, err = cache.Get(c) + Expect(err).ToNot(HaveOccurred()) + }) + }) +}) diff --git a/pkg/compiler/types/artifact/checksum.go b/pkg/api/core/types/artifact/checksum.go similarity index 88% rename from pkg/compiler/types/artifact/checksum.go rename to pkg/api/core/types/artifact/checksum.go index 6d213143..5db70e2a 100644 --- a/pkg/compiler/types/artifact/checksum.go +++ b/pkg/api/core/types/artifact/checksum.go @@ -24,6 +24,7 @@ import ( "hash" "io" "os" + "sort" // . "github.com/mudler/luet/pkg/logger" "github.com/pkg/errors" @@ -42,6 +43,18 @@ type HashOptions struct { Type HashImplementation } +func (c Checksums) List() (res [][]string) { + keys := make([]string, 0) + for k, _ := range c { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + res = append(res, []string{k, c[k]}) + } + return +} + // Generate generates all Checksums supported for the artifact func (c *Checksums) Generate(a *PackageArtifact) error { return c.generateSHA256(a) diff --git a/pkg/compiler/types/artifact/checksum_test.go b/pkg/api/core/types/artifact/checksum_test.go similarity index 97% rename from pkg/compiler/types/artifact/checksum_test.go rename to pkg/api/core/types/artifact/checksum_test.go index 2219452d..ccac7bf1 100644 --- a/pkg/compiler/types/artifact/checksum_test.go +++ b/pkg/api/core/types/artifact/checksum_test.go @@ -19,7 +19,7 @@ import ( "io/ioutil" "os" - . "github.com/mudler/luet/pkg/compiler/types/artifact" + . "github.com/mudler/luet/pkg/api/core/types/artifact" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/compiler/backend.go b/pkg/compiler/backend.go index 7cbb844a..93ddaf54 100644 --- a/pkg/compiler/backend.go +++ b/pkg/compiler/backend.go @@ -7,7 +7,7 @@ import ( "path/filepath" "strings" - artifact "github.com/mudler/luet/pkg/compiler/types/artifact" + artifact "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/compiler/backend" "github.com/mudler/luet/pkg/config" diff --git a/pkg/compiler/backend/simpledocker_test.go b/pkg/compiler/backend/simpledocker_test.go index ff9ececb..3ba17857 100644 --- a/pkg/compiler/backend/simpledocker_test.go +++ b/pkg/compiler/backend/simpledocker_test.go @@ -16,11 +16,11 @@ package backend_test import ( + "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/compiler" . "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/compiler/backend" . "github.com/mudler/luet/pkg/compiler/backend" - "github.com/mudler/luet/pkg/compiler/types/artifact" fileHelper "github.com/mudler/luet/pkg/helpers/file" "io/ioutil" diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 0f844b7a..33f1308b 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -30,9 +30,9 @@ import ( "time" "github.com/imdario/mergo" + artifact "github.com/mudler/luet/pkg/api/core/types/artifact" bus "github.com/mudler/luet/pkg/bus" "github.com/mudler/luet/pkg/compiler/backend" - artifact "github.com/mudler/luet/pkg/compiler/types/artifact" "github.com/mudler/luet/pkg/compiler/types/options" compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" "github.com/mudler/luet/pkg/helpers" diff --git a/pkg/installer/client/docker.go b/pkg/installer/client/docker.go index 89b076f0..d2c68a99 100644 --- a/pkg/installer/client/docker.go +++ b/pkg/installer/client/docker.go @@ -1,4 +1,4 @@ -// Copyright © 2020 Ettore Di Giacinto +// Copyright © 2020-2021 Ettore Di Giacinto // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -26,7 +26,7 @@ import ( "github.com/docker/go-units" "github.com/pkg/errors" - "github.com/mudler/luet/pkg/compiler/types/artifact" + "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers/docker" fileHelper "github.com/mudler/luet/pkg/helpers/file" @@ -41,6 +41,7 @@ type DockerClient struct { RepoData RepoData auth *types.AuthConfig verify bool + Cache *artifact.ArtifactCache } func NewDockerClient(r RepoData) *DockerClient { @@ -49,25 +50,22 @@ func NewDockerClient(r RepoData) *DockerClient { dat, _ := json.Marshal(r.Authentication) json.Unmarshal(dat, auth) - return &DockerClient{RepoData: r, auth: auth} + return &DockerClient{RepoData: r, auth: auth, + Cache: artifact.NewCache(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()), + } } func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) { //var u *url.URL = nil var err error - var temp string Spinner(22) defer SpinnerStop() - var resultingArtifact *artifact.PackageArtifact + resultingArtifact := a.ShallowCopy() artifactName := path.Base(a.Path) - cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName) - Debug("Cache file", cacheFile) - if err := fileHelper.EnsureDir(cacheFile); err != nil { - return nil, errors.Wrapf(err, "could not create cache folder %s for %s", config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), cacheFile) - } - ok := false + + downloaded := false // TODO: // Files are in URI/packagename:version (GetPackageImageName() method) @@ -77,19 +75,26 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact. // is done in such cases (see repository.go) // Check if file is already in cache - if fileHelper.Exists(cacheFile) { - Debug("Cache hit for artifact", artifactName) + fileName, err := c.Cache.Get(a) + // Check if file is already in cache + if err == nil { resultingArtifact = a - resultingArtifact.Path = cacheFile + resultingArtifact.Path = fileName resultingArtifact.Checksums = artifact.Checksums{} + Debug("Use artifact", artifactName, "from cache.") } else { - temp, err = config.LuetCfg.GetSystem().TempDir("tree") + temp, err := config.LuetCfg.GetSystem().TempDir("image") if err != nil { return nil, err } defer os.RemoveAll(temp) + tempArtifact, err := config.LuetCfg.GetSystem().TempFile("artifact") + if err != nil { + return nil, err + } + defer os.RemoveAll(tempArtifact.Name()) for _, uri := range c.RepoData.Urls { imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID()) @@ -110,26 +115,37 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact. Info(fmt.Sprintf("Pulled: %s", info.Target.Digest)) Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size)))) - Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile) + Debug("\nCompressing result ", filepath.Join(temp), "to", tempArtifact.Name()) - newart := a // We discard checksum, that are checked while during pull and unpack - newart.Checksums = artifact.Checksums{} - newart.Path = cacheFile // First set to cache file - newart.Path = newart.GetUncompressedName() // Calculate the real path from cacheFile - err = newart.Compress(temp, 1) + resultingArtifact.Checksums = artifact.Checksums{} + resultingArtifact.Path = tempArtifact.Name() // First set to cache file + err = resultingArtifact.Compress(temp, 1) if err != nil { Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error())) continue } - resultingArtifact = newart - ok = true + _, _, err = c.Cache.Put(resultingArtifact) + if err != nil { + Error(fmt.Sprintf("Failed storing package %s from cache: %s", imageName, err.Error())) + continue + } + + fileName, err := c.Cache.Get(resultingArtifact) + if err != nil { + Error(fmt.Sprintf("Failed getting package %s from cache: %s", imageName, err.Error())) + continue + } + + resultingArtifact.Path = fileName // Cache is persistent. tempArtifact is not + + downloaded = true break } - if !ok { - return nil, err + if !downloaded { + return nil, errors.Wrap(err, "no image available from repositories") } } diff --git a/pkg/installer/client/docker_test.go b/pkg/installer/client/docker_test.go index 6b0e0c2c..c3d8a4d4 100644 --- a/pkg/installer/client/docker_test.go +++ b/pkg/installer/client/docker_test.go @@ -20,7 +20,7 @@ import ( "os" "path/filepath" - "github.com/mudler/luet/pkg/compiler/types/artifact" + "github.com/mudler/luet/pkg/api/core/types/artifact" compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" fileHelper "github.com/mudler/luet/pkg/helpers/file" diff --git a/pkg/installer/client/http.go b/pkg/installer/client/http.go index 12d24b98..22f2abc5 100644 --- a/pkg/installer/client/http.go +++ b/pkg/installer/client/http.go @@ -1,4 +1,4 @@ -// Copyright © 2019 Ettore Di Giacinto +// Copyright © 2019-2021 Ettore Di Giacinto // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -26,9 +26,9 @@ import ( "strconv" "time" - "github.com/mudler/luet/pkg/compiler/types/artifact" - fileHelper "github.com/mudler/luet/pkg/helpers/file" + "github.com/mudler/luet/pkg/api/core/types/artifact" . "github.com/mudler/luet/pkg/logger" + "github.com/pkg/errors" "github.com/cavaliercoder/grab" "github.com/mudler/luet/pkg/config" @@ -38,14 +38,18 @@ import ( type HttpClient struct { RepoData RepoData + Cache *artifact.ArtifactCache } func NewHttpClient(r RepoData) *HttpClient { - return &HttpClient{RepoData: r} + return &HttpClient{ + RepoData: r, + Cache: artifact.NewCache(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()), + } } func NewGrabClient() *grab.Client { - httpTimeout := 120 + httpTimeout := 360 timeout := os.Getenv("HTTP_TIMEOUT") if timeout != "" { timeoutI, err := strconv.Atoi(timeout) @@ -65,7 +69,7 @@ func NewGrabClient() *grab.Client { } } -func (c *HttpClient) PrepareReq(dst, url string) (*grab.Request, error) { +func (c *HttpClient) prepareReq(dst, url string) (*grab.Request, error) { req, err := grab.NewRequest(dst, url) if err != nil { @@ -88,169 +92,114 @@ func Round(input float64) float64 { return math.Floor(input + 0.5) } -func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) { - var u *url.URL = nil - var err error - var req *grab.Request - var temp string - - artifactName := path.Base(a.Path) - cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName) - ok := false - - // Check if file is already in cache - if fileHelper.Exists(cacheFile) { - Debug("Use artifact", artifactName, "from cache.") - } else { - - temp, err = config.LuetCfg.GetSystem().TempDir("tree") - if err != nil { - return nil, err - } - defer os.RemoveAll(temp) - - client := NewGrabClient() - - for _, uri := range c.RepoData.Urls { - Debug("Downloading artifact", artifactName, "from", uri) - - u, err = url.Parse(uri) - if err != nil { - continue - } - u.Path = path.Join(u.Path, artifactName) - - req, err = c.PrepareReq(temp, u.String()) - if err != nil { - continue - } - - resp := client.Do(req) - - bar := progressbar.NewOptions64( - resp.Size(), - progressbar.OptionSetDescription( - fmt.Sprintf("[cyan] %s - [reset]", - filepath.Base(resp.Request.HTTPRequest.URL.RequestURI()))), - progressbar.OptionSetRenderBlankState(true), - progressbar.OptionEnableColorCodes(config.LuetCfg.GetLogging().Color), - progressbar.OptionClearOnFinish(), - progressbar.OptionShowBytes(true), - progressbar.OptionShowCount(), - progressbar.OptionSetPredictTime(true), - progressbar.OptionFullWidth(), - progressbar.OptionSetTheme(progressbar.Theme{ - Saucer: "[white]=[reset]", - SaucerHead: "[white]>[reset]", - SaucerPadding: " ", - BarStart: "[", - BarEnd: "]", - })) - - bar.Reset() - // start download loop - t := time.NewTicker(500 * time.Millisecond) - defer t.Stop() - - download_loop: - - for { - select { - case <-t.C: - bar.Set64(resp.BytesComplete()) - - case <-resp.Done: - // download is complete - break download_loop - } - } - - if err = resp.Err(); err != nil { - continue - } - - if err != nil { - continue - } - - Info("\nDownloaded", artifactName, "of", - fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (", - fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )") - - Debug("\nCopying file ", filepath.Join(temp, artifactName), "to", cacheFile) - err = fileHelper.CopyFile(filepath.Join(temp, artifactName), cacheFile) - - bar.Finish() - ok = true - break - } - - if !ok { - return nil, err - } - } - - newart := a - newart.Path = cacheFile - return newart, nil -} - -func (c *HttpClient) DownloadFile(name string) (string, error) { +func (c *HttpClient) DownloadFile(p string) (string, error) { var file *os.File = nil - var u *url.URL = nil - var err error - var req *grab.Request - var temp string - - ok := false - - temp, err = config.LuetCfg.GetSystem().TempDir("tree") + var downloaded bool + temp, err := config.LuetCfg.GetSystem().TempDir("download") if err != nil { return "", err } + defer os.RemoveAll(temp) client := NewGrabClient() for _, uri := range c.RepoData.Urls { - file, err = config.LuetCfg.GetSystem().TempFile("HttpClient") if err != nil { + Debug("Failed downloading", p, "from", uri) + continue } + Debug("Downloading artifact", p, "from", uri) - u, err = url.Parse(uri) + u, err := url.Parse(uri) if err != nil { continue } - u.Path = path.Join(u.Path, name) + u.Path = path.Join(u.Path, p) - Debug("Downloading", u.String()) - - req, err = c.PrepareReq(temp, u.String()) + req, err := c.prepareReq(file.Name(), u.String()) if err != nil { continue } resp := client.Do(req) + + bar := progressbar.NewOptions64( + resp.Size(), + progressbar.OptionSetDescription( + fmt.Sprintf("[cyan] %s - [reset]", + filepath.Base(resp.Request.HTTPRequest.URL.RequestURI()))), + progressbar.OptionSetRenderBlankState(true), + progressbar.OptionEnableColorCodes(config.LuetCfg.GetLogging().Color), + progressbar.OptionClearOnFinish(), + progressbar.OptionShowBytes(true), + progressbar.OptionShowCount(), + progressbar.OptionSetPredictTime(true), + progressbar.OptionFullWidth(), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "[white]=[reset]", + SaucerHead: "[white]>[reset]", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + })) + + bar.Reset() + // start download loop + t := time.NewTicker(500 * time.Millisecond) + defer t.Stop() + + download_loop: + + for { + select { + case <-t.C: + bar.Set64(resp.BytesComplete()) + + case <-resp.Done: + // download is complete + break download_loop + } + } + if err = resp.Err(); err != nil { continue } - Info("Downloaded", filepath.Base(resp.Filename), "of", + Info("\nDownloaded", p, "of", fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (", fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )") - err = fileHelper.CopyFile(filepath.Join(temp, name), file.Name()) - if err != nil { - continue - } - ok = true + bar.Finish() + downloaded = true break } - if !ok { - return "", err + if !downloaded { + return "", errors.Wrap(err, "artifact not available in any of the specified url locations") + } + return file.Name(), nil +} + +func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) { + newart := a.ShallowCopy() + artifactName := path.Base(a.Path) + + fileName, err := c.Cache.Get(a) + // Check if file is already in cache + if err == nil { + newart.Path = fileName + Debug("Use artifact", artifactName, "from cache.") + } else { + d, err := c.DownloadFile(artifactName) + if err != nil { + return nil, errors.Wrapf(err, "failed downloading %s", artifactName) + } + + newart.Path = d + c.Cache.Put(newart) } - return file.Name(), err + return newart, nil } diff --git a/pkg/installer/client/http_test.go b/pkg/installer/client/http_test.go index ee8ecfc2..6bf7b24e 100644 --- a/pkg/installer/client/http_test.go +++ b/pkg/installer/client/http_test.go @@ -22,7 +22,7 @@ import ( "os" "path/filepath" - "github.com/mudler/luet/pkg/compiler/types/artifact" + "github.com/mudler/luet/pkg/api/core/types/artifact" fileHelper "github.com/mudler/luet/pkg/helpers/file" . "github.com/mudler/luet/pkg/installer/client" . "github.com/onsi/ginkgo" diff --git a/pkg/installer/client/local.go b/pkg/installer/client/local.go index ce6c29d8..9c93618f 100644 --- a/pkg/installer/client/local.go +++ b/pkg/installer/client/local.go @@ -1,4 +1,4 @@ -// Copyright © 2019 Ettore Di Giacinto +// Copyright © 2020-2021 Ettore Di Giacinto // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -20,61 +20,46 @@ import ( "path" "path/filepath" - "github.com/mudler/luet/pkg/compiler/types/artifact" + "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/config" fileHelper "github.com/mudler/luet/pkg/helpers/file" . "github.com/mudler/luet/pkg/logger" + "github.com/pkg/errors" ) type LocalClient struct { RepoData RepoData + Cache *artifact.ArtifactCache } func NewLocalClient(r RepoData) *LocalClient { - return &LocalClient{RepoData: r} + return &LocalClient{ + Cache: artifact.NewCache(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()), + RepoData: r, + } } func (c *LocalClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) { var err error - rootfs := "" artifactName := path.Base(a.Path) - cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName) - - if !config.LuetCfg.ConfigFromHost { - rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs() - if err != nil { - return nil, err - } - } + newart := a.ShallowCopy() + fileName, err := c.Cache.Get(a) // Check if file is already in cache - if fileHelper.Exists(cacheFile) { + if err == nil { + newart.Path = fileName Debug("Use artifact", artifactName, "from cache.") } else { - ok := false - for _, uri := range c.RepoData.Urls { - - uri = filepath.Join(rootfs, uri) - - Info("Downloading artifact", artifactName, "from", uri) - - //defer os.Remove(file.Name()) - err = fileHelper.CopyFile(filepath.Join(uri, artifactName), cacheFile) - if err != nil { - continue - } - ok = true - break + d, err := c.DownloadFile(artifactName) + if err != nil { + return nil, errors.Wrapf(err, "failed downloading %s", artifactName) } - if !ok { - return nil, err - } + newart.Path = d + c.Cache.Put(newart) } - newart := a - newart.Path = cacheFile return newart, nil } diff --git a/pkg/installer/client/local_test.go b/pkg/installer/client/local_test.go index de9cb19e..486d75df 100644 --- a/pkg/installer/client/local_test.go +++ b/pkg/installer/client/local_test.go @@ -20,7 +20,7 @@ import ( "os" "path/filepath" - "github.com/mudler/luet/pkg/compiler/types/artifact" + "github.com/mudler/luet/pkg/api/core/types/artifact" fileHelper "github.com/mudler/luet/pkg/helpers/file" . "github.com/mudler/luet/pkg/installer/client" . "github.com/onsi/ginkgo" diff --git a/pkg/installer/installer.go b/pkg/installer/installer.go index 9bb2e3bd..17a0c4b6 100644 --- a/pkg/installer/installer.go +++ b/pkg/installer/installer.go @@ -25,8 +25,8 @@ import ( "sync" "github.com/mudler/luet/pkg/api/core/types" + artifact "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/bus" - artifact "github.com/mudler/luet/pkg/compiler/types/artifact" "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" fileHelper "github.com/mudler/luet/pkg/helpers/file" diff --git a/pkg/installer/interface.go b/pkg/installer/interface.go index b7293982..7391bb9f 100644 --- a/pkg/installer/interface.go +++ b/pkg/installer/interface.go @@ -16,7 +16,7 @@ package installer import ( - artifact "github.com/mudler/luet/pkg/compiler/types/artifact" + artifact "github.com/mudler/luet/pkg/api/core/types/artifact" //"github.com/mudler/luet/pkg/solver" ) diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go index 3562938b..cdf4fe7f 100644 --- a/pkg/installer/repository.go +++ b/pkg/installer/repository.go @@ -27,7 +27,7 @@ import ( "strings" "time" - artifact "github.com/mudler/luet/pkg/compiler/types/artifact" + artifact "github.com/mudler/luet/pkg/api/core/types/artifact" compression "github.com/mudler/luet/pkg/compiler/types/compression" fileHelper "github.com/mudler/luet/pkg/helpers/file" "go.uber.org/multierr" diff --git a/pkg/installer/repository_docker.go b/pkg/installer/repository_docker.go index 043cf614..2d03972a 100644 --- a/pkg/installer/repository_docker.go +++ b/pkg/installer/repository_docker.go @@ -24,10 +24,10 @@ import ( "strings" "time" + artifact "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/bus" compiler "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/compiler/backend" - artifact "github.com/mudler/luet/pkg/compiler/types/artifact" "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" "github.com/mudler/luet/pkg/helpers/docker" diff --git a/pkg/installer/repository_local.go b/pkg/installer/repository_local.go index 8390bc01..04930367 100644 --- a/pkg/installer/repository_local.go +++ b/pkg/installer/repository_local.go @@ -24,7 +24,7 @@ import ( "strings" "time" - artifact "github.com/mudler/luet/pkg/compiler/types/artifact" + artifact "github.com/mudler/luet/pkg/api/core/types/artifact" . "github.com/mudler/luet/pkg/logger" pkg "github.com/mudler/luet/pkg/package" diff --git a/pkg/installer/repository_test.go b/pkg/installer/repository_test.go index fe1a5ac6..da26e55a 100644 --- a/pkg/installer/repository_test.go +++ b/pkg/installer/repository_test.go @@ -25,9 +25,9 @@ import ( "path/filepath" "github.com/mudler/luet/pkg/api/core/types" + artifact "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/compiler" backend "github.com/mudler/luet/pkg/compiler/backend" - artifact "github.com/mudler/luet/pkg/compiler/types/artifact" compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" "github.com/mudler/luet/pkg/helpers" fileHelper "github.com/mudler/luet/pkg/helpers/file"