installer: Add support for cached repository

pkg/helpers/archive.go

@@ -18,8 +18,11 @@ package helpers
 import (
 	"io"
 	"os"
+	//"os/user"
+	//"strconv"
 
 	"github.com/docker/docker/pkg/archive"
+	//"github.com/docker/docker/pkg/idtools"
 )
 
 func Tar(src, dest string) error {
@@ -55,8 +58,25 @@ func Untar(src, dest string, sameOwner bool) error {
 	}
 	defer in.Close()
 
-	return archive.Untar(in, dest, &archive.TarOptions{
-		NoLchown:        !sameOwner,
+	opts := &archive.TarOptions{
+		// NOTE: NoLchown boolean is used for chmod of the symlink
+		// Probably it's needed set this always to true.
+		NoLchown:        true,
 		ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
-	})
+	}
+
+	/*
+	u, err := user.Current()
+	if err != nil {
+		return err
+	}
+	// TODO: This seems not sufficient for untar with normal user.
+	if u.Uid != "0" {
+		uid, _ := strconv.Atoi(u.Uid)
+		gid, _ := strconv.Atoi(u.Gid)
+		opts.ChownOpts = &idtools.Identity{UID: uid, GID: gid}
+	}
+	*/
+
+	return archive.Untar(in, dest, opts)
 }

pkg/installer/client/http.go

@@ -49,15 +49,16 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 	if err != nil {
 		return nil, err
 	}
+	defer os.RemoveAll(temp)
+
+	file, err = ioutil.TempFile(temp, "HttpClient")
+	if err != nil {
+		return nil, err
+	}
 
 	for _, uri := range c.RepoData.Urls {
 		Info("Downloading artifact", artifactName, "from", uri)
 
-		file, err = ioutil.TempFile(temp, "HttpClient")
-		if err != nil {
-			continue
-		}
-
 		u, err = url.Parse(uri)
 		if err != nil {
 			continue
@@ -103,7 +104,6 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 		if err != nil {
 			continue
 		}
-		//defer os.Remove(file.Name())
 		u, err = url.Parse(uri)
 		if err != nil {
 			continue

pkg/installer/installer.go

@@ -128,7 +128,7 @@ func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
 	defer SpinnerStop()
 	syncedRepos := Repositories{}
 	for _, r := range l.PackageRepositories {
-		repo, err := r.Sync()
+		repo, err := r.Sync(false)
 		if err != nil {
 			return nil, errors.Wrap(err, "Failed syncing repository: "+r.GetName())
 		}

pkg/installer/interface.go

@@ -48,7 +48,7 @@ type Repository interface {
 	GetTree() tree.Builder
 	SetTree(tree.Builder)
 	Write(path string, resetRevision bool) error
-	Sync() (Repository, error)
+	Sync(bool) (Repository, error)
 	GetTreePath() string
 	SetTreePath(string)
 	GetType() string

pkg/installer/repository.go

@@ -26,15 +26,16 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mudler/luet/pkg/installer/client"
-
-	"github.com/ghodss/yaml"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/installer/client"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/ghodss/yaml"
+	. "github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 )
 
@@ -74,7 +75,7 @@ func GenerateRepository(name, descr, t string, urls []string, priority int, src,
 	}
 
 	return NewLuetSystemRepository(
-		config.NewLuetRepository(name, t, descr, urls, priority, true),
+		config.NewLuetRepository(name, t, descr, urls, priority, true, false),
 		art, tr), nil
 }
 
@@ -106,6 +107,7 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 			p.Urls,
 			p.Priority,
 			true,
+			false,
 		),
 	}
 	if p.Revision > 0 {
@@ -288,43 +290,88 @@ func (r *LuetSystemRepository) Client() Client {
 
 	return nil
 }
-func (r *LuetSystemRepository) Sync() (Repository, error) {
-	Debug("Sync of the repository", r.Name, "in progress..")
+func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
+	var repoUpdated bool = false
+	var treefs string
+
+	Debug("Sync of the repository", r.Name, "in progress...")
 	c := r.Client()
 	if c == nil {
 		return nil, errors.New("No client could be generated from repository.")
 	}
+
+	// Retrieve remote repository.yaml for retrieve revision and date
 	file, err := c.DownloadFile(REPOSITORY_SPECFILE)
 	if err != nil {
 		return nil, errors.Wrap(err, "While downloading "+REPOSITORY_SPECFILE)
 	}
-	repo, err := r.ReadSpecFile(file, true)
+
+	repobasedir := helpers.GetRepoDatabaseDirPath(r.GetName())
+	repo, err := r.ReadSpecFile(file, false)
 	if err != nil {
 		return nil, err
 	}
+	// Remove temporary file that contains repository.html.
+	// Example: /tmp/HttpClient236052003
+	defer os.RemoveAll(file)
 
-	archivetree, err := c.DownloadFile(TREE_TARBALL)
-	if err != nil {
-		return nil, errors.Wrap(err, "While downloading "+TREE_TARBALL)
+	if r.Cached {
+		if !force {
+			localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir, REPOSITORY_SPECFILE), false)
+			if localRepo != nil {
+				if localRepo.GetRevision() == repo.GetRevision() &&
+					localRepo.GetLastUpdate() == repo.GetLastUpdate() {
+					repoUpdated = true
+				}
+			}
+		}
+		if r.GetTreePath() == "" {
+			treefs = filepath.Join(repobasedir, "treefs")
+		} else {
+			treefs = r.GetTreePath()
+		}
+
+	} else {
+		treefs, err = ioutil.TempDir(os.TempDir(), "treefs")
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
 	}
-	defer os.RemoveAll(archivetree) // clean up
 
-	treefs, err := ioutil.TempDir(os.TempDir(), "treefs")
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	//defer os.RemoveAll(treefs) // clean up
+	if !repoUpdated {
+		archivetree, err := c.DownloadFile(TREE_TARBALL)
+		if err != nil {
+			return nil, errors.Wrap(err, "While downloading "+TREE_TARBALL)
+		}
+		defer os.RemoveAll(archivetree) // clean up
 
-	// TODO: Following as option if archive as output?
-	// archive, err := ioutil.TempDir(os.TempDir(), "archive")
-	// if err != nil {
-	// 	return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	// }
-	// defer os.RemoveAll(archive) // clean up
+		Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
 
-	err = helpers.Untar(archivetree, treefs, false)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while unpacking rootfs")
+		if r.Cached {
+			// Copy updated repository.yaml file to repo dir now that the tree is synced.
+			err = helpers.CopyFile(file, filepath.Join(repobasedir, REPOSITORY_SPECFILE))
+			if err != nil {
+				return nil, errors.Wrap(err, "Error on update "+REPOSITORY_SPECFILE)
+			}
+			// Remove previous tree
+			os.RemoveAll(treefs)
+		}
+		Debug("Untar tree of the repository " + r.Name + "...")
+		err = helpers.Untar(archivetree, treefs, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while unpacking rootfs")
+		}
+
+		tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64)
+
+		InfoC(
+			Bold(Red(":house: Repository "+r.GetName()+" revision: ")).String() +
+				Bold(Green(repo.GetRevision())).String() + " - " +
+				Bold(Green(time.Unix(tsec, 0).String())).String(),
+		)
+
+	} else {
+		Info("Repository", r.GetName(), "is already up to date.")
 	}
 
 	reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
@@ -332,6 +379,7 @@ func (r *LuetSystemRepository) Sync() (Repository, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while unpacking rootfs")
 	}
+
 	repo.SetTree(reciper)
 	repo.SetTreePath(treefs)
 	repo.SetUrls(r.GetUrls())