Do implement a real cache

Instead of merely storing files into disk use a real cache.

This also makes possible finally to reference artifacts in the cache with the
package checksum, which solves the cache hit checksum failures we had
previously.

cmd/build.go

@@ -22,8 +22,8 @@ import (
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/installer"
 

cmd/database/create.go

@@ -19,7 +19,7 @@ import (
 	"io/ioutil"
 
 	"github.com/mudler/luet/cmd/util"
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"

cmd/pack.go

@@ -20,7 +20,7 @@ import (
 	"time"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler/types/compression"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	. "github.com/mudler/luet/pkg/config"

go.mod

@@ -62,6 +62,7 @@ require (
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
 	github.com/pkg/errors v0.9.1
+	github.com/rancher-sandbox/gofilecache v0.0.0-20210330135715-becdeff5df15
 	github.com/schollz/progressbar/v3 v3.7.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cast v1.4.1 // indirect

pkg/api/core/types/artifact/artifact.go

@@ -50,7 +50,7 @@ import (
 	yaml "gopkg.in/yaml.v2"
 )
 
-//  When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository
+// When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository
 // which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact.
 // In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages.
 type PackageArtifact struct {

pkg/api/core/types/artifact/artifact_test.go

@@ -20,10 +20,10 @@ import (
 	"os"
 	"path/filepath"
 
+	. "github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
-	. "github.com/mudler/luet/pkg/compiler/types/artifact"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 

pkg/api/core/types/artifact/cache.go

@@ -0,0 +1,63 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact
+
+import (
+	"crypto/sha512"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/rancher-sandbox/gofilecache"
+)
+
+type ArtifactCache struct {
+	gofilecache.Cache
+}
+
+func NewCache(dir string) *ArtifactCache {
+	return &ArtifactCache{Cache: *gofilecache.InitCache(dir)}
+}
+
+func (c *ArtifactCache) cacheID(a *PackageArtifact) [64]byte {
+	fingerprint := filepath.Base(a.Path)
+	if a.CompileSpec != nil && a.CompileSpec.Package != nil {
+		fingerprint = a.CompileSpec.Package.GetFingerPrint()
+	}
+	if len(a.Checksums) > 0 {
+		for _, cs := range a.Checksums.List() {
+			t := cs[0]
+			result := cs[1]
+			fingerprint += fmt.Sprintf("+%s:%s", t, result)
+		}
+	}
+	return sha512.Sum512([]byte(fingerprint))
+}
+
+func (c *ArtifactCache) Get(a *PackageArtifact) (string, error) {
+	fileName, _, err := c.Cache.GetFile(c.cacheID(a))
+	return fileName, err
+}
+
+func (c *ArtifactCache) Put(a *PackageArtifact) (gofilecache.OutputID, int64, error) {
+	file, err := os.Open(a.Path)
+	if err != nil {
+		return [64]byte{}, 0, errors.Wrapf(err, "failed opening %s", a.Path)
+	}
+	defer file.Close()
+	return c.Cache.Put(c.cacheID(a), file)
+}

pkg/api/core/types/artifact/cache_test.go

@@ -0,0 +1,89 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	. "github.com/mudler/luet/pkg/api/core/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Cache", func() {
+	Context("CacheID", func() {
+		It("Get and retrieve files", func() {
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpdirartifact, err := ioutil.TempDir(os.TempDir(), "testartifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdirartifact) // clean up
+
+			err = ioutil.WriteFile(filepath.Join(tmpdirartifact, "foo"), []byte(string("foo")), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			a := NewPackageArtifact(filepath.Join(tmpdir, "foo.tar.gz"))
+			err = a.Compress(tmpdirartifact, 1)
+			Expect(err).ToNot(HaveOccurred())
+
+			cache := NewCache(tmpdir)
+
+			// Put an artifact in the cache and retrieve it later
+			// the artifact is NOT hashed so it is referenced just by the path in the cache
+			_, _, err = cache.Put(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			path, err := cache.Get(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			b := NewPackageArtifact(path)
+
+			err = b.Unpack(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "foo"))).To(BeTrue())
+
+			bb, err := ioutil.ReadFile(filepath.Join(tmpdir, "foo"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(bb)).To(Equal("foo"))
+
+			// After the artifact is hashed, the fingerprint mutates so the cache doesn't see it hitting again
+			// the test we did above fails as we expect to.
+			a.Hash()
+			_, err = cache.Get(a)
+			Expect(err).To(HaveOccurred())
+
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "bar"}}
+			_, _, err = cache.Put(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			c := NewPackageArtifact(filepath.Join(tmpdir, "foo.tar.gz"))
+			c.Hash()
+			c.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "bar"}}
+			_, err = cache.Get(c)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})

pkg/api/core/types/artifact/checksum.go

@@ -24,6 +24,7 @@ import (
 	"hash"
 	"io"
 	"os"
+	"sort"
 
 	//	. "github.com/mudler/luet/pkg/logger"
 	"github.com/pkg/errors"
@@ -42,6 +43,18 @@ type HashOptions struct {
 	Type   HashImplementation
 }
 
+func (c Checksums) List() (res [][]string) {
+	keys := make([]string, 0)
+	for k, _ := range c {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		res = append(res, []string{k, c[k]})
+	}
+	return
+}
+
 // Generate generates all Checksums supported for the artifact
 func (c *Checksums) Generate(a *PackageArtifact) error {
 	return c.generateSHA256(a)

pkg/api/core/types/artifact/checksum_test.go

@@ -19,7 +19,7 @@ import (
 	"io/ioutil"
 	"os"
 
-	. "github.com/mudler/luet/pkg/compiler/types/artifact"
+	. "github.com/mudler/luet/pkg/api/core/types/artifact"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

pkg/compiler/backend.go

@@ -7,7 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/config"

pkg/compiler/backend/simpledocker_test.go

@@ -16,11 +16,11 @@
 package backend_test
 
 import (
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	"io/ioutil"

pkg/compiler/compiler.go

@@ -30,9 +30,9 @@ import (
 	"time"
 
 	"github.com/imdario/mergo"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	bus "github.com/mudler/luet/pkg/bus"
 	"github.com/mudler/luet/pkg/compiler/backend"
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/helpers"

pkg/installer/client/docker.go

@@ -1,4 +1,4 @@
-// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
+// Copyright © 2020-2021 Ettore Di Giacinto <mudler@mocaccino.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -26,7 +26,7 @@ import (
 	"github.com/docker/go-units"
 	"github.com/pkg/errors"
 
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers/docker"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
@@ -41,6 +41,7 @@ type DockerClient struct {
 	RepoData RepoData
 	auth     *types.AuthConfig
 	verify   bool
+	Cache    *artifact.ArtifactCache
 }
 
 func NewDockerClient(r RepoData) *DockerClient {
@@ -49,25 +50,22 @@ func NewDockerClient(r RepoData) *DockerClient {
 	dat, _ := json.Marshal(r.Authentication)
 	json.Unmarshal(dat, auth)
 
-	return &DockerClient{RepoData: r, auth: auth}
+	return &DockerClient{RepoData: r, auth: auth,
+		Cache: artifact.NewCache(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()),
+	}
 }
 
 func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	//var u *url.URL = nil
 	var err error
-	var temp string
 
 	Spinner(22)
 	defer SpinnerStop()
 
-	var resultingArtifact *artifact.PackageArtifact
+	resultingArtifact := a.ShallowCopy()
 	artifactName := path.Base(a.Path)
-	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
-	Debug("Cache file", cacheFile)
-	if err := fileHelper.EnsureDir(cacheFile); err != nil {
-		return nil, errors.Wrapf(err, "could not create cache folder %s for %s", config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), cacheFile)
-	}
-	ok := false
+
+	downloaded := false
 
 	// TODO:
 	// Files are in URI/packagename:version (GetPackageImageName() method)
@@ -77,19 +75,26 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
 	// is done in such cases (see repository.go)
 
 	// Check if file is already in cache
-	if fileHelper.Exists(cacheFile) {
-		Debug("Cache hit for artifact", artifactName)
+	fileName, err := c.Cache.Get(a)
+	// Check if file is already in cache
+	if err == nil {
 		resultingArtifact = a
-		resultingArtifact.Path = cacheFile
+		resultingArtifact.Path = fileName
 		resultingArtifact.Checksums = artifact.Checksums{}
+		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 
-		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
+		temp, err := config.LuetCfg.GetSystem().TempDir("image")
 		if err != nil {
 			return nil, err
 		}
 		defer os.RemoveAll(temp)
 
+		tempArtifact, err := config.LuetCfg.GetSystem().TempFile("artifact")
+		if err != nil {
+			return nil, err
+		}
+		defer os.RemoveAll(tempArtifact.Name())
 		for _, uri := range c.RepoData.Urls {
 
 			imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID())
@@ -110,26 +115,37 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
 
 			Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
 			Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size))))
-			Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile)
+			Debug("\nCompressing result ", filepath.Join(temp), "to", tempArtifact.Name())
 
-			newart := a
 			// We discard checksum, that are checked while during pull and unpack
-			newart.Checksums = artifact.Checksums{}
-			newart.Path = cacheFile                    // First set to cache file
-			newart.Path = newart.GetUncompressedName() // Calculate the real path from cacheFile
-			err = newart.Compress(temp, 1)
+			resultingArtifact.Checksums = artifact.Checksums{}
+			resultingArtifact.Path = tempArtifact.Name() // First set to cache file
+			err = resultingArtifact.Compress(temp, 1)
 			if err != nil {
 				Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))
 				continue
 			}
-			resultingArtifact = newart
 
-			ok = true
+			_, _, err = c.Cache.Put(resultingArtifact)
+			if err != nil {
+				Error(fmt.Sprintf("Failed storing package %s from cache: %s", imageName, err.Error()))
+				continue
+			}
+
+			fileName, err := c.Cache.Get(resultingArtifact)
+			if err != nil {
+				Error(fmt.Sprintf("Failed getting package %s from cache: %s", imageName, err.Error()))
+				continue
+			}
+
+			resultingArtifact.Path = fileName // Cache is persistent. tempArtifact is not
+
+			downloaded = true
 			break
 		}
 
-		if !ok {
-			return nil, err
+		if !downloaded {
+			return nil, errors.Wrap(err, "no image available from repositories")
 		}
 	}
 

pkg/installer/client/docker_test.go

@@ -20,7 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 

pkg/installer/client/http.go

@@ -1,4 +1,4 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@gentoo.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -26,9 +26,9 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
-	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	. "github.com/mudler/luet/pkg/logger"
+	"github.com/pkg/errors"
 
 	"github.com/cavaliercoder/grab"
 	"github.com/mudler/luet/pkg/config"
@@ -38,14 +38,18 @@ import (
 
 type HttpClient struct {
 	RepoData RepoData
+	Cache    *artifact.ArtifactCache
 }
 
 func NewHttpClient(r RepoData) *HttpClient {
-	return &HttpClient{RepoData: r}
+	return &HttpClient{
+		RepoData: r,
+		Cache:    artifact.NewCache(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()),
+	}
 }
 
 func NewGrabClient() *grab.Client {
-	httpTimeout := 120
+	httpTimeout := 360
 	timeout := os.Getenv("HTTP_TIMEOUT")
 	if timeout != "" {
 		timeoutI, err := strconv.Atoi(timeout)
@@ -65,7 +69,7 @@ func NewGrabClient() *grab.Client {
 	}
 }
 
-func (c *HttpClient) PrepareReq(dst, url string) (*grab.Request, error) {
+func (c *HttpClient) prepareReq(dst, url string) (*grab.Request, error) {
 
 	req, err := grab.NewRequest(dst, url)
 	if err != nil {
@@ -88,169 +92,114 @@ func Round(input float64) float64 {
 	return math.Floor(input + 0.5)
 }
 
-func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
-	var u *url.URL = nil
-	var err error
-	var req *grab.Request
-	var temp string
-
-	artifactName := path.Base(a.Path)
-	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
-	ok := false
-
-	// Check if file is already in cache
-	if fileHelper.Exists(cacheFile) {
-		Debug("Use artifact", artifactName, "from cache.")
-	} else {
-
-		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
-		if err != nil {
-			return nil, err
-		}
-		defer os.RemoveAll(temp)
-
-		client := NewGrabClient()
-
-		for _, uri := range c.RepoData.Urls {
-			Debug("Downloading artifact", artifactName, "from", uri)
-
-			u, err = url.Parse(uri)
-			if err != nil {
-				continue
-			}
-			u.Path = path.Join(u.Path, artifactName)
-
-			req, err = c.PrepareReq(temp, u.String())
-			if err != nil {
-				continue
-			}
-
-			resp := client.Do(req)
-
-			bar := progressbar.NewOptions64(
-				resp.Size(),
-				progressbar.OptionSetDescription(
-					fmt.Sprintf("[cyan] %s - [reset]",
-						filepath.Base(resp.Request.HTTPRequest.URL.RequestURI()))),
-				progressbar.OptionSetRenderBlankState(true),
-				progressbar.OptionEnableColorCodes(config.LuetCfg.GetLogging().Color),
-				progressbar.OptionClearOnFinish(),
-				progressbar.OptionShowBytes(true),
-				progressbar.OptionShowCount(),
-				progressbar.OptionSetPredictTime(true),
-				progressbar.OptionFullWidth(),
-				progressbar.OptionSetTheme(progressbar.Theme{
-					Saucer:        "[white]=[reset]",
-					SaucerHead:    "[white]>[reset]",
-					SaucerPadding: " ",
-					BarStart:      "[",
-					BarEnd:        "]",
-				}))
-
-			bar.Reset()
-			// start download loop
-			t := time.NewTicker(500 * time.Millisecond)
-			defer t.Stop()
-
-		download_loop:
-
-			for {
-				select {
-				case <-t.C:
-					bar.Set64(resp.BytesComplete())
-
-				case <-resp.Done:
-					// download is complete
-					break download_loop
-				}
-			}
-
-			if err = resp.Err(); err != nil {
-				continue
-			}
-
-			if err != nil {
-				continue
-			}
-
-			Info("\nDownloaded", artifactName, "of",
-				fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
-				fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
-
-			Debug("\nCopying file ", filepath.Join(temp, artifactName), "to", cacheFile)
-			err = fileHelper.CopyFile(filepath.Join(temp, artifactName), cacheFile)
-
-			bar.Finish()
-			ok = true
-			break
-		}
-
-		if !ok {
-			return nil, err
-		}
-	}
-
-	newart := a
-	newart.Path = cacheFile
-	return newart, nil
-}
-
-func (c *HttpClient) DownloadFile(name string) (string, error) {
+func (c *HttpClient) DownloadFile(p string) (string, error) {
 	var file *os.File = nil
-	var u *url.URL = nil
-	var err error
-	var req *grab.Request
-	var temp string
-
-	ok := false
-
-	temp, err = config.LuetCfg.GetSystem().TempDir("tree")
+	var downloaded bool
+	temp, err := config.LuetCfg.GetSystem().TempDir("download")
 	if err != nil {
 		return "", err
 	}
+	defer os.RemoveAll(temp)
 
 	client := NewGrabClient()
 
 	for _, uri := range c.RepoData.Urls {
-
 		file, err = config.LuetCfg.GetSystem().TempFile("HttpClient")
 		if err != nil {
+			Debug("Failed downloading", p, "from", uri)
+
 			continue
 		}
+		Debug("Downloading artifact", p, "from", uri)
 
-		u, err = url.Parse(uri)
+		u, err := url.Parse(uri)
 		if err != nil {
 			continue
 		}
-		u.Path = path.Join(u.Path, name)
+		u.Path = path.Join(u.Path, p)
 
-		Debug("Downloading", u.String())
-
-		req, err = c.PrepareReq(temp, u.String())
+		req, err := c.prepareReq(file.Name(), u.String())
 		if err != nil {
 			continue
 		}
 
 		resp := client.Do(req)
+
+		bar := progressbar.NewOptions64(
+			resp.Size(),
+			progressbar.OptionSetDescription(
+				fmt.Sprintf("[cyan] %s - [reset]",
+					filepath.Base(resp.Request.HTTPRequest.URL.RequestURI()))),
+			progressbar.OptionSetRenderBlankState(true),
+			progressbar.OptionEnableColorCodes(config.LuetCfg.GetLogging().Color),
+			progressbar.OptionClearOnFinish(),
+			progressbar.OptionShowBytes(true),
+			progressbar.OptionShowCount(),
+			progressbar.OptionSetPredictTime(true),
+			progressbar.OptionFullWidth(),
+			progressbar.OptionSetTheme(progressbar.Theme{
+				Saucer:        "[white]=[reset]",
+				SaucerHead:    "[white]>[reset]",
+				SaucerPadding: " ",
+				BarStart:      "[",
+				BarEnd:        "]",
+			}))
+
+		bar.Reset()
+		// start download loop
+		t := time.NewTicker(500 * time.Millisecond)
+		defer t.Stop()
+
+	download_loop:
+
+		for {
+			select {
+			case <-t.C:
+				bar.Set64(resp.BytesComplete())
+
+			case <-resp.Done:
+				// download is complete
+				break download_loop
+			}
+		}
+
 		if err = resp.Err(); err != nil {
 			continue
 		}
 
-		Info("Downloaded", filepath.Base(resp.Filename), "of",
+		Info("\nDownloaded", p, "of",
 			fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
 			fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
 
-		err = fileHelper.CopyFile(filepath.Join(temp, name), file.Name())
-		if err != nil {
-			continue
-		}
-		ok = true
+		bar.Finish()
+		downloaded = true
 		break
 	}
 
-	if !ok {
-		return "", err
+	if !downloaded {
+		return "", errors.Wrap(err, "artifact not available in any of the specified url locations")
+	}
+	return file.Name(), nil
+}
+
+func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
+	newart := a.ShallowCopy()
+	artifactName := path.Base(a.Path)
+
+	fileName, err := c.Cache.Get(a)
+	// Check if file is already in cache
+	if err == nil {
+		newart.Path = fileName
+		Debug("Use artifact", artifactName, "from cache.")
+	} else {
+		d, err := c.DownloadFile(artifactName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed downloading %s", artifactName)
+		}
+
+		newart.Path = d
+		c.Cache.Put(newart)
 	}
 
-	return file.Name(), err
+	return newart, nil
 }

pkg/installer/client/http_test.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer/client"
 	. "github.com/onsi/ginkgo"

pkg/installer/client/local.go

@@ -1,4 +1,4 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+// Copyright © 2020-2021 Ettore Di Giacinto <mudler@gentoo.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -20,61 +20,46 @@ import (
 	"path"
 	"path/filepath"
 
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/config"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
+	"github.com/pkg/errors"
 )
 
 type LocalClient struct {
 	RepoData RepoData
+	Cache    *artifact.ArtifactCache
 }
 
 func NewLocalClient(r RepoData) *LocalClient {
-	return &LocalClient{RepoData: r}
+	return &LocalClient{
+		Cache:    artifact.NewCache(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()),
+		RepoData: r,
+	}
 }
 
 func (c *LocalClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	var err error
 
-	rootfs := ""
 	artifactName := path.Base(a.Path)
-	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
-
-	if !config.LuetCfg.ConfigFromHost {
-		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
-		if err != nil {
-			return nil, err
-		}
-	}
+	newart := a.ShallowCopy()
 
+	fileName, err := c.Cache.Get(a)
 	// Check if file is already in cache
-	if fileHelper.Exists(cacheFile) {
+	if err == nil {
+		newart.Path = fileName
 		Debug("Use artifact", artifactName, "from cache.")
 	} else {
-		ok := false
-		for _, uri := range c.RepoData.Urls {
-
-			uri = filepath.Join(rootfs, uri)
-
-			Info("Downloading artifact", artifactName, "from", uri)
-
-			//defer os.Remove(file.Name())
-			err = fileHelper.CopyFile(filepath.Join(uri, artifactName), cacheFile)
-			if err != nil {
-				continue
-			}
-			ok = true
-			break
+		d, err := c.DownloadFile(artifactName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed downloading %s", artifactName)
 		}
 
-		if !ok {
-			return nil, err
-		}
+		newart.Path = d
+		c.Cache.Put(newart)
 	}
 
-	newart := a
-	newart.Path = cacheFile
 	return newart, nil
 }
 

pkg/installer/client/local_test.go

@@ -20,7 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer/client"
 	. "github.com/onsi/ginkgo"

pkg/installer/installer.go

@@ -25,8 +25,8 @@ import (
 	"sync"
 
 	"github.com/mudler/luet/pkg/api/core/types"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/bus"
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"

pkg/installer/interface.go

@@ -16,7 +16,7 @@
 package installer
 
 import (
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	//"github.com/mudler/luet/pkg/solver"
 )
 

pkg/installer/repository.go

@@ -27,7 +27,7 @@ import (
 	"strings"
 	"time"
 
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	"go.uber.org/multierr"

pkg/installer/repository_docker.go

@@ -24,10 +24,10 @@ import (
 	"strings"
 	"time"
 
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/bus"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	"github.com/mudler/luet/pkg/helpers/docker"

pkg/installer/repository_local.go

@@ -24,7 +24,7 @@ import (
 	"strings"
 	"time"
 
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 

pkg/installer/repository_test.go

@@ -25,9 +25,9 @@ import (
 	"path/filepath"
 
 	"github.com/mudler/luet/pkg/api/core/types"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"