Allow to pull specfiles from published repositories

- Interpolates values from the repositories compilespec if present
- Automatically merge cache images coming from specified repository when
  necessary

Fixes #194

cmd/build.go

@@ -16,7 +16,6 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -25,6 +24,7 @@ import (
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	"github.com/mudler/luet/pkg/installer"
 
 	"github.com/mudler/luet/pkg/compiler/types/compression"
 	"github.com/mudler/luet/pkg/compiler/types/options"
@@ -44,17 +44,17 @@ var buildCmd = &cobra.Command{
 	Long: `Builds one or more packages from a tree (current directory is implied):
 
 	$ luet build utils/busybox utils/yq ...
-	
+
 Builds all packages
-	
+
 	$ luet build --all
-	
+
 Builds only the leaf packages:
-	
+
 	$ luet build --full
 
 Build package revdeps:
-	
+
 	$ luet build --revdeps utils/yq
 
 Build package without dependencies (needs the images already in the host, or either need to be available online):
@@ -69,7 +69,6 @@ Build packages specifying multiple definition trees:
 		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
 		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
 		viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged"))
-		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
 		viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
 		viper.BindPFlag("all", cmd.Flags().Lookup("all"))
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
@@ -101,7 +100,6 @@ Build packages specifying multiple definition trees:
 		privileged := viper.GetBool("privileged")
 		revdeps := viper.GetBool("revdeps")
 		all := viper.GetBool("all")
-		databaseType := viper.GetString("database")
 		compressionType := viper.GetString("compression")
 		imageRepository := viper.GetString("image-repository")
 		values := viper.GetStringSlice("values")
@@ -122,26 +120,25 @@ Build packages specifying multiple definition trees:
 			LuetCfg.GetLogging().SetLogLevel("error")
 		}
 		pretend, _ := cmd.Flags().GetBool("pretend")
+		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
+
 		compilerSpecs := compilerspec.NewLuetCompilationspecs()
 		var db pkg.PackageDatabase
 
 		compilerBackend, err := compiler.NewBackend(backendType)
 		helpers.CheckErr(err)
 
-		switch databaseType {
+		db = pkg.NewInMemoryDatabase(false)
-		case "memory":
-			db = pkg.NewInMemoryDatabase(false)
-
-		case "boltdb":
-			tmpdir, err := ioutil.TempDir("", "package")
-			helpers.CheckErr(err)
-			db = pkg.NewBoltDatabase(tmpdir)
-
-		}
 		defer db.Clean()
 
 		generalRecipe := tree.NewCompilerRecipe(db)
 
+		if fromRepo {
+			if err := installer.LoadBuildTree(generalRecipe, db, LuetCfg); err != nil {
+				Warning("errors while loading trees from repositories", err.Error())
+			}
+		}
+
 		for _, src := range treePaths {
 			Info("Loading tree", src)
 			helpers.CheckErr(generalRecipe.Load(src))
@@ -202,7 +199,6 @@ Build packages specifying multiple definition trees:
 			}
 		} else if !all {
 			for _, a := range args {
-
 				pack, err := helpers.ParsePackageStr(a)
 				if err != nil {
 					Fatal("Invalid package string ", a, ": ", err.Error())
@@ -310,7 +306,6 @@ func init() {
 	buildCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the tree to use.")
 	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
 	buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)")
-	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
@@ -333,6 +328,7 @@ func init() {
 	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	buildCmd.Flags().Bool("live-output", LuetCfg.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
+	buildCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
 
 	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
 	buildCmd.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")

cmd/install.go

@@ -70,16 +70,6 @@ To force install a package:
 			toInstall = append(toInstall, pack)
 		}
 
-		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
-		repos := installer.Repositories{}
-		for _, repo := range LuetCfg.SystemRepositories {
-			if !repo.Enable {
-				continue
-			}
-			r := installer.NewSystemRepository(repo)
-			repos = append(repos, r)
-		}
-
 		stype := LuetCfg.Viper.GetString("solver.type")
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
@@ -111,6 +101,7 @@ To force install a package:
 		}
 
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		repos := installer.SystemRepositories(LuetCfg)
 
 		// Load config protect configs
 		installer.LoadConfigProtectConfs(LuetCfg)

go.mod

@@ -20,8 +20,10 @@ require (
 	github.com/genuinetools/img v0.5.11
 	github.com/ghodss/yaml v1.0.0
 	github.com/google/go-containerregistry v0.2.1
+	github.com/google/renameio v1.0.0
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/hashicorp/go-version v1.2.1
+	github.com/imdario/mergo v0.3.8
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/jedib0t/go-pretty/v6 v6.0.5
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
@@ -51,7 +53,7 @@ require (
 	github.com/theupdateframework/notary v0.7.0
 	go.etcd.io/bbolt v1.3.5
 	go.uber.org/atomic v1.5.1 // indirect
-	go.uber.org/multierr v1.4.0 // indirect
+	go.uber.org/multierr v1.4.0
 	go.uber.org/zap v1.13.0
 	google.golang.org/grpc v1.29.1
 	gopkg.in/yaml.v2 v2.3.0

go.sum

@@ -493,6 +493,8 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/renameio v1.0.0 h1:xhp2CnJmgQmpJU4RY8chagahUq5mbPPAbiSQstKpVMA=
+github.com/google/renameio v1.0.0/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk=
 github.com/google/shlex v0.0.0-20150127133951-6f45313302b9 h1:JM174NTeGNJ2m/oLH3UOWOvWQQKd+BoL3hcSCUWFLt0=
 github.com/google/shlex v0.0.0-20150127133951-6f45313302b9/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

pkg/compiler/compiler.go

@@ -27,6 +27,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/imdario/mergo"
 	bus "github.com/mudler/luet/pkg/bus"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
@@ -37,6 +38,7 @@ import (
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
 )
 
 const BuildFile = "build.yaml"
@@ -48,19 +50,11 @@ type ArtifactIndex []*artifact.PackageArtifact
 func (i ArtifactIndex) CleanPath() ArtifactIndex {
 	newIndex := ArtifactIndex{}
 	for _, art := range i {
-		// FIXME: This is a dup and makes difficult to add attributes to artifacts
+		copy := art.ShallowCopy()
-		newIndex = append(newIndex, &artifact.PackageArtifact{
+		copy.Path = path.Base(art.Path)
-			Path:            path.Base(art.Path),
+		newIndex = append(newIndex, copy)
-			SourceAssertion: art.SourceAssertion,
-			CompileSpec:     art.CompileSpec,
-			Dependencies:    art.Dependencies,
-			CompressionType: art.CompressionType,
-			Checksums:       art.Checksums,
-			Files:           art.Files,
-		})
 	}
 	return newIndex
-	//Update if exists, otherwise just create
 }
 
 type LuetCompiler struct {
@@ -544,7 +538,7 @@ func (cs *LuetCompiler) resolveExistingImageHash(imageHash string) string {
 }
 
 func LoadArtifactFromYaml(spec *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml"
+	metaFile := spec.GetPackage().GetMetadataFilePath()
 	dat, err := ioutil.ReadFile(spec.Rel(metaFile))
 	if err != nil {
 		return nil, errors.Wrap(err, "Error reading file "+metaFile)
@@ -731,6 +725,12 @@ func genImageList(refs []string, hash string) []string {
 }
 
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
+	if len(p.BuildOptions.PullImageRepository) != 0 {
+		orig := cs.Options.PullImageRepository
+		cs.Options.PullImageRepository = append(orig, p.BuildOptions.PullImageRepository...)
+		defer func() { cs.Options.PullImageRepository = orig }()
+	}
+
 	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
 	Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
@@ -866,7 +866,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 
 type templatedata map[string]interface{}
 
-func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) {
+func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.Package) ([]byte, error) {
 
 	var dataresult []byte
 	val := pack.Rel(DefinitionFile)
@@ -876,7 +876,7 @@ func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "unmarshalling values")
 	}
-	cs.Options.BuildValues = []map[string]interface{}{(map[string]interface{})(dst)}
+	cs.Options.BuildValues = append(vals, (map[string]interface{})(dst))
 
 	if _, err := os.Stat(pack.Rel(CollectionFile)); err == nil {
 		val = pack.Rel(CollectionFile)
@@ -897,14 +897,46 @@ func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) {
 		}
 
 		raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
+		td := templatedata{}
+		if len(vals) > 0 {
+			for _, bv := range vals {
+				current := templatedata(bv)
+				if err := mergo.Merge(&td, current); err != nil {
+					return nil, errors.Wrap(err, "merging values maps")
+				}
+			}
+		}
 
-		dat, err := helpers.RenderHelm(string(dataBuild), raw, dst)
+		if err := mergo.Merge(&td, raw); err != nil {
+			return nil, errors.Wrap(err, "merging values maps")
+		}
+
+		dat, err := helpers.RenderHelm(string(dataBuild), td, dst)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}
 		dataresult = []byte(dat)
 	} else {
-		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, cs.Options.BuildValuesFile...)
+		bv := cs.Options.BuildValuesFile
+		if len(vals) > 0 {
+			valuesdir, err := ioutil.TempDir("", "genvalues")
+			if err != nil {
+				return nil, errors.Wrap(err, "Could not create tempdir")
+			}
+			defer os.RemoveAll(valuesdir) // clean up
+			for _, b := range vals {
+				out, err := yaml.Marshal(b)
+				if err != nil {
+					return nil, errors.Wrap(err, "while marshalling values file")
+				}
+				f := filepath.Join(valuesdir, helpers.RandStringRunes(20))
+				if err := ioutil.WriteFile(f, out, os.ModePerm); err != nil {
+					return nil, errors.Wrap(err, "while writing temporary values file")
+				}
+				bv = append([]string{f}, bv...)
+			}
+		}
+		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, bv...)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}
@@ -922,12 +954,43 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio
 		return nil, err
 	}
 
-	bytes, err := cs.templatePackage(pack)
+	opts := options.Compiler{}
+
+	artifactMetadataFile := filepath.Join(p.GetTreeDir(), p.GetMetadataFilePath())
+
+	if fi, err := os.Stat(artifactMetadataFile); err == nil {
+		f, err := os.Open(fi.Name())
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not open %s", fi.Name())
+		}
+		dat, err := ioutil.ReadAll(f)
+		if err != nil {
+			return nil, err
+		}
+		art, err := artifact.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not decode package from yaml")
+		}
+
+		opts = art.CompileSpec.BuildOptions
+		opts.PushImageRepository = ""
+
+	} else if !os.IsNotExist(err) {
+		Debug("error reading already existing artifact metadata file: ", err.Error())
+	}
+
+	bytes, err := cs.templatePackage(opts.BuildValues, pack)
 	if err != nil {
 		return nil, errors.Wrap(err, "while rendering package template")
 	}
 
-	return compilerspec.NewLuetCompilationSpec(bytes, pack)
+	newSpec, err := compilerspec.NewLuetCompilationSpec(bytes, pack)
+	if err != nil {
+		return nil, err
+	}
+	newSpec.BuildOptions = opts
+
+	return newSpec, err
 }
 
 // GetBackend returns the current compilation backend

pkg/compiler/types/artifact/artifact.go

@@ -62,6 +62,11 @@ type PackageArtifact struct {
 	Files           []string                          `json:"files"`
 }
 
+func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
+	copy := *p
+	return &copy
+}
+
 func NewPackageArtifact(path string) *PackageArtifact {
 	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: compression.None}
 }
@@ -129,7 +134,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
 	}
 
-	err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm)
+	err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetMetadataFilePath()), data, os.ModePerm)
 	if err != nil {
 		return errors.Wrap(err, "While writing PackageArtifact YAML")
 	}

pkg/helpers/file.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"sort"
@@ -26,10 +27,41 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/google/renameio"
 	copy "github.com/otiai10/copy"
 	"github.com/pkg/errors"
 )
 
+var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+func RandStringRunes(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letterRunes[rand.Intn(len(letterRunes))]
+	}
+	return string(b)
+}
+
+func Move(src, dst string) error {
+	f, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	t, err := renameio.TempFile("", dst)
+	if err != nil {
+		return err
+	}
+	defer t.Cleanup()
+
+	_, err = io.Copy(t, f)
+	if err != nil {
+		return err
+	}
+	return t.CloseAtomicallyReplace()
+}
+
 func OrderFiles(target string, files []string) ([]string, []string) {
 
 	var newFiles []string

pkg/installer/repository.go

@@ -16,6 +16,7 @@
 package installer
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
@@ -27,6 +28,7 @@ import (
 
 	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	"go.uber.org/multierr"
 
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
@@ -100,6 +102,40 @@ func NewLuetSystemRepositoryMetadata(file string, removeFile bool) (*LuetSystemR
 	return ans, nil
 }
 
+// SystemRepositories returns the repositories from the local configuration file
+func SystemRepositories(c *config.LuetConfig) Repositories {
+	repos := Repositories{}
+	for _, repo := range c.SystemRepositories {
+		if !repo.Enable {
+			continue
+		}
+		r := NewSystemRepository(repo)
+		repos = append(repos, r)
+	}
+	return repos
+}
+
+// LoadBuildTree loads to the tree the compilation specs from the system repositories
+func LoadBuildTree(t tree.Builder, db pkg.PackageDatabase, c *config.LuetConfig) error {
+	var reserr error
+	repos := SystemRepositories(c)
+	for _, r := range repos {
+		repodir, err := config.LuetCfg.GetSystem().TempDir(r.Name)
+		if err != nil {
+			reserr = multierr.Append(reserr, err)
+		}
+		if err := r.SyncBuildMetadata(repodir); err != nil {
+			reserr = multierr.Append(reserr, err)
+		}
+		if err := t.Load(filepath.Join(repodir, "tree")); err != nil {
+			reserr = multierr.Append(reserr, err)
+		}
+	}
+	repos.SyncDatabase(db)
+
+	return reserr
+}
+
 func (m *LuetSystemRepositoryMetadata) WriteFile(path string) error {
 	data, err := yaml.Marshal(m)
 	if err != nil {
@@ -471,7 +507,7 @@ func (r *LuetSystemRepository) AddRepositoryFile(src, fileKey, repositoryRoot st
 	treeFile, err := r.GetRepositoryFile(fileKey)
 	if err != nil {
 		treeFile = defaults
-		r.SetRepositoryFile(fileKey, treeFile)
+		//	r.SetRepositoryFile(fileKey, treeFile)
 	}
 
 	a := artifact.NewPackageArtifact(filepath.Join(repositoryRoot, treeFile.GetFileName()))
@@ -594,6 +630,70 @@ func (r *LuetSystemRepository) SearchArtefact(p pkg.Package) (*artifact.PackageA
 	return nil, errors.New("Not found")
 }
 
+func (r *LuetSystemRepository) getRepoFile(c Client, key string) (*artifact.PackageArtifact, error) {
+
+	treeFile, err := r.GetRepositoryFile(key)
+	if err != nil {
+		return nil, errors.Wrapf(err, "key %s not present in the repository", key)
+	}
+
+	// Get Tree
+	downloadedTreeFile, err := c.DownloadFile(treeFile.GetFileName())
+	if err != nil {
+		return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName())
+	}
+	//defer os.Remove(downloadedTreeFile)
+
+	treeFileArtifact := artifact.NewPackageArtifact(downloadedTreeFile)
+	treeFileArtifact.Checksums = treeFile.GetChecksums()
+	treeFileArtifact.CompressionType = treeFile.GetCompressionType()
+
+	err = treeFileArtifact.Verify()
+	if err != nil {
+		return nil, errors.Wrap(err, "file integrity check failure")
+	}
+
+	return treeFileArtifact, nil
+
+}
+
+func (r *LuetSystemRepository) SyncBuildMetadata(path string) error {
+
+	repo, err := r.Sync(false)
+	if err != nil {
+		return errors.Wrap(err, "while syncronizing repository")
+	}
+
+	c := repo.Client()
+	if c == nil {
+		return errors.New("no client could be generated from repository")
+	}
+
+	a, err := repo.getRepoFile(c, REPOFILE_COMPILER_TREE_KEY)
+	if err != nil {
+		return fmt.Errorf("failed while getting: %s", REPOFILE_COMPILER_TREE_KEY)
+	}
+
+	defer os.RemoveAll(a.Path)
+
+	if err := a.Unpack(filepath.Join(path, "tree"), false); err != nil {
+		return errors.Wrapf(err, "while unpacking: %s", REPOFILE_COMPILER_TREE_KEY)
+	}
+
+	for _, ai := range repo.GetTree().GetDatabase().World() {
+		// Retrieve remote repository.yaml for retrieve revision and date
+		file, err := c.DownloadFile(ai.GetMetadataFilePath())
+		if err != nil {
+			return errors.Wrapf(err, "while downloading metadata for %s", ai.HumanReadableString())
+		}
+		if err := helpers.Move(file, filepath.Join(path, ai.GetMetadataFilePath())); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
 	var repoUpdated bool = false
 	var treefs, metafs string
@@ -612,7 +712,7 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
 	}
 
 	repobasedir := config.LuetCfg.GetSystem().GetRepoDatabaseDirPath(r.GetName())
-	repo, err := r.ReadSpecFile(file)
+	downloadedRepoMeta, err := r.ReadSpecFile(file)
 	if err != nil {
 		return nil, err
 	}
@@ -624,8 +724,8 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
 		if !force {
 			localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir, REPOSITORY_SPECFILE))
 			if localRepo != nil {
-				if localRepo.GetRevision() == repo.GetRevision() &&
+				if localRepo.GetRevision() == downloadedRepoMeta.GetRevision() &&
-					localRepo.GetLastUpdate() == repo.GetLastUpdate() {
+					localRepo.GetLastUpdate() == downloadedRepoMeta.GetLastUpdate() {
 					repoUpdated = true
 				}
 			}
@@ -652,47 +752,22 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
 		}
 	}
 
-	// POST: treeFile and metaFile are present. I check this inside
+	// treeFile and metaFile must be present, they aren't optional
-	// ReadSpecFile and NewLuetSystemRepositoryFromYaml
-	treeFile, _ := repo.GetRepositoryFile(REPOFILE_TREE_KEY)
-	metaFile, _ := repo.GetRepositoryFile(REPOFILE_META_KEY)
-
 	if !repoUpdated {
 
-		// Get Tree
+		treeFileArtifact, err := downloadedRepoMeta.getRepoFile(c, REPOFILE_TREE_KEY)
-		downloadedTreeFile, err := c.DownloadFile(treeFile.GetFileName())
 		if err != nil {
-			return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName())
+			return nil, errors.Wrapf(err, "while fetching '%s'", REPOFILE_TREE_KEY)
-		}
-		defer os.Remove(downloadedTreeFile)
-
-		// Treat the file as artifact, in order to verify it
-		treeFileArtifact := artifact.NewPackageArtifact(downloadedTreeFile)
-		treeFileArtifact.Checksums = treeFile.GetChecksums()
-		treeFileArtifact.CompressionType = treeFile.GetCompressionType()
-
-		err = treeFileArtifact.Verify()
-		if err != nil {
-			return nil, errors.Wrap(err, "Tree integrity check failure")
 		}
+		defer os.Remove(treeFileArtifact.Path)
 
 		Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
 
-		// Get Repository Metadata
+		metaFileArtifact, err := downloadedRepoMeta.getRepoFile(c, REPOFILE_META_KEY)
-		downloadedMeta, err := c.DownloadFile(metaFile.GetFileName())
 		if err != nil {
-			return nil, errors.Wrap(err, "While downloading "+metaFile.GetFileName())
+			return nil, errors.Wrapf(err, "while fetching '%s'", REPOFILE_META_KEY)
-		}
-		defer os.Remove(downloadedMeta)
-
-		metaFileArtifact := artifact.NewPackageArtifact(downloadedMeta)
-		metaFileArtifact.Checksums = metaFile.GetChecksums()
-		metaFileArtifact.CompressionType = metaFile.GetCompressionType()
-
-		err = metaFileArtifact.Verify()
-		if err != nil {
-			return nil, errors.Wrap(err, "Metadata integrity check failure")
 		}
+		defer os.Remove(metaFileArtifact.Path)
 
 		Debug("Metadata tarball for the repository " + r.GetName() + " downloaded correctly.")
 
@@ -722,17 +797,17 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
 			return nil, errors.Wrap(err, "Error met while unpacking metadata")
 		}
 
-		tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64)
+		tsec, _ := strconv.ParseInt(downloadedRepoMeta.GetLastUpdate(), 10, 64)
 
 		InfoC(
 			aurora.Bold(
-				aurora.Red(":house: Repository "+repo.GetName()+" revision: ")).String() +
+				aurora.Red(":house: Repository "+downloadedRepoMeta.GetName()+" revision: ")).String() +
-				aurora.Bold(aurora.Green(repo.GetRevision())).String() + " - " +
+				aurora.Bold(aurora.Green(downloadedRepoMeta.GetRevision())).String() + " - " +
 				aurora.Bold(aurora.Green(time.Unix(tsec, 0).String())).String(),
 		)
 
 	} else {
-		Info("Repository", repo.GetName(), "is already up to date.")
+		Info("Repository", downloadedRepoMeta.GetName(), "is already up to date.")
 	}
 
 	meta, err := NewLuetSystemRepositoryMetadata(
@@ -741,7 +816,7 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "While processing "+REPOSITORY_METAFILE)
 	}
-	repo.SetIndex(meta.ToArtifactIndex())
+	downloadedRepoMeta.SetIndex(meta.ToArtifactIndex())
 
 	reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
 	err = reciper.Load(treefs)
@@ -749,29 +824,33 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
 		return nil, errors.Wrap(err, "Error met while unpacking rootfs")
 	}
 
-	repo.SetTree(reciper)
+	downloadedRepoMeta.SetTree(reciper)
-	repo.SetTreePath(treefs)
+	downloadedRepoMeta.SetTreePath(treefs)
 
 	// Copy the local available data to the one which was synced
 	// e.g. locally we can override the type (disk), or priority
 	// while remotely it could be advertized differently
-	repo.SetUrls(r.GetUrls())
+	r.fill(downloadedRepoMeta)
-	repo.SetAuthentication(r.GetAuthentication())
-	repo.SetType(r.GetType())
-	repo.SetPriority(r.GetPriority())
-	repo.SetName(r.GetName())
-	repo.SetVerify(r.GetVerify())
 
 	InfoC(
 		aurora.Yellow(":information_source:").String() +
 			aurora.Magenta("Repository: ").String() +
-			aurora.Green(aurora.Bold(repo.GetName()).String()).String() +
+			aurora.Green(aurora.Bold(downloadedRepoMeta.GetName()).String()).String() +
 			aurora.Magenta(" Priority: ").String() +
-			aurora.Bold(aurora.Green(repo.GetPriority())).String() +
+			aurora.Bold(aurora.Green(downloadedRepoMeta.GetPriority())).String() +
 			aurora.Magenta(" Type: ").String() +
-			aurora.Bold(aurora.Green(repo.GetType())).String(),
+			aurora.Bold(aurora.Green(downloadedRepoMeta.GetType())).String(),
 	)
-	return repo, nil
+	return downloadedRepoMeta, nil
+}
+
+func (r *LuetSystemRepository) fill(r2 *LuetSystemRepository) {
+	r2.SetUrls(r.GetUrls())
+	r2.SetAuthentication(r.GetAuthentication())
+	r2.SetType(r.GetType())
+	r2.SetPriority(r.GetPriority())
+	r2.SetName(r.GetName())
+	r2.SetVerify(r.GetVerify())
 }
 
 func (r *LuetSystemRepository) Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepository) {

pkg/installer/repository_docker.go

@@ -44,23 +44,22 @@ type dockerRepositoryGenerator struct {
 }
 
 func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
-	return generatePackageImages(l.b, l.imagePrefix, path, db, l.imagePush, l.force)
-}
 
-func pushImage(b compiler.CompilerBackend, image string, force bool) error {
+	Info("Generating docker images for packages in", l.imagePrefix)
-	if b.ImageAvailable(image) && !force {
-		Debug("Image", image, "already present, skipping")
-		return nil
-	}
-	return b.Push(backend.Options{ImageName: image})
-}
-
-func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]*artifact.PackageArtifact, error) {
-	Info("Generating docker images for packages in", imagePrefix)
 	var art []*artifact.PackageArtifact
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			Debug("Skipping", info.Name(), err.Error())
+			return nil
+		}
 
-		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+		if strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			a := artifact.NewPackageArtifact(info.Name())
+			imageRepo := fmt.Sprintf("%s:%s", l.imagePrefix, filepath.Base(info.Name()))
+
+			if err := l.pushFileFromArtifact(a, imageRepo); err != nil {
+				return errors.Wrap(err, "while pushing file from artifact")
+			}
 			return nil // Skip with no errors
 		}
 
@@ -85,19 +84,19 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string,
 			return nil
 		}
 
-		packageImage := fmt.Sprintf("%s:%s", imagePrefix, a.CompileSpec.GetPackage().ImageID())
+		packageImage := fmt.Sprintf("%s:%s", l.imagePrefix, a.CompileSpec.GetPackage().ImageID())
 
-		if imagePush && b.ImageAvailable(packageImage) && !force {
+		if l.imagePush && l.b.ImageAvailable(packageImage) && !l.force {
 			Info("Image", packageImage, "already present, skipping. use --force-push to override")
 		} else {
 			Info("Generating final image", packageImage,
 				"for package ", a.CompileSpec.GetPackage().HumanReadableString())
-			if opts, err := a.GenerateFinalImage(packageImage, b, true); err != nil {
+			if opts, err := a.GenerateFinalImage(packageImage, l.b, true); err != nil {
 				return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
 			}
 		}
-		if imagePush {
+		if l.imagePush {
-			if err := pushImage(b, packageImage, force); err != nil {
+			if err := pushImage(l.b, packageImage, l.force); err != nil {
 				return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage)
 			}
 		}
@@ -115,13 +114,21 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string,
 	return art, nil
 }
 
-func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string, r *LuetSystemRepository) error {
+func pushImage(b compiler.CompilerBackend, image string, force bool) error {
+	if b.ImageAvailable(image) && !force {
+		Debug("Image", image, "already present, skipping")
+		return nil
+	}
+	return b.Push(backend.Options{ImageName: image})
+}
+
+func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string) error {
 	Debug("Generating image", imageTree)
-	if opts, err := a.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil {
+	if opts, err := a.GenerateFinalImage(imageTree, d.b, false); err != nil {
 		return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
 	}
-	if r.PushImages {
+	if d.imagePush {
-		if err := pushImage(r.GetBackend(), imageTree, true); err != nil {
+		if err := pushImage(d.b, imageTree, true); err != nil {
 			return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree)
 		}
 	}
@@ -144,13 +151,13 @@ func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec string, r *LuetSys
 	a := artifact.NewPackageArtifact(tempRepoFile)
 	imageRepo := fmt.Sprintf("%s:%s", d.imagePrefix, REPOSITORY_SPECFILE)
 
-	if err := d.pushFileFromArtifact(a, imageRepo, r); err != nil {
+	if err := d.pushFileFromArtifact(a, imageRepo); err != nil {
 		return errors.Wrap(err, "while pushing file from artifact")
 	}
 	return nil
 }
 
-func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, r *LuetSystemRepository) error {
+func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, b compiler.CompilerBackend) error {
 	// we generate a new archive containing the required compressed file.
 	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
 	treeArchive, err := artifact.CreateArtifactForFile(a.Path)
@@ -159,7 +166,7 @@ func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArt
 	}
 	imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, a.GetFileName())
 
-	return d.pushFileFromArtifact(treeArchive, imageTree, r)
+	return d.pushFileFromArtifact(treeArchive, imageTree)
 }
 
 // Generate creates a Docker luet repository
@@ -216,7 +223,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 
 	// we generate a new archive containing the required compressed file.
 	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
-	if err := d.pushImageFromArtifact(a, r); err != nil {
+	if err := d.pushImageFromArtifact(a, d.b); err != nil {
 		return errors.Wrap(err, "error met while pushing runtime tree")
 	}
 
@@ -226,7 +233,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 	}
 	// we generate a new archive containing the required compressed file.
 	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
-	if err := d.pushImageFromArtifact(a, r); err != nil {
+	if err := d.pushImageFromArtifact(a, d.b); err != nil {
 		return errors.Wrap(err, "error met while pushing compiler tree")
 	}
 
@@ -242,7 +249,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 		return errors.Wrap(err, "failed adding Metadata file to repository")
 	}
 
-	if err := d.pushImageFromArtifact(a, r); err != nil {
+	if err := d.pushImageFromArtifact(a, d.b); err != nil {
 		return errors.Wrap(err, "error met while pushing docker image from artifact")
 	}
 

pkg/package/package.go

@@ -113,6 +113,10 @@ type Package interface {
 	GetBuildTimestamp() string
 
 	Clone() Package
+
+	GetMetadataFilePath() string
+	SetTreeDir(s string)
+	GetTreeDir() string
 }
 
 type Tree interface {
@@ -210,6 +214,11 @@ func (t *DefaultPackage) JSON() ([]byte, error) {
 	return buffer.Bytes(), err
 }
 
+// GetMetadataFilePath returns the canonical name of an artifact metadata file
+func (d *DefaultPackage) GetMetadataFilePath() string {
+	return d.GetFingerPrint() + ".metadata.yaml"
+}
+
 // DefaultPackage represent a standard package definition
 type DefaultPackage struct {
 	ID               int               `storm:"id,increment" json:"id"` // primary key with auto increment
@@ -235,6 +244,8 @@ type DefaultPackage struct {
 	BuildTimestamp string   `json:"buildtimestamp,omitempty"`
 
 	Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
+
+	treeDir string
 }
 
 // State represent the package state
@@ -251,6 +262,12 @@ func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*D
 	}
 }
 
+func (p *DefaultPackage) SetTreeDir(s string) {
+	p.treeDir = s
+}
+func (p *DefaultPackage) GetTreeDir() string {
+	return p.treeDir
+}
 func (p *DefaultPackage) String() string {
 	b, err := p.JSON()
 	if err != nil {

pkg/tree/compiler_recipe.go

@@ -98,6 +98,7 @@ func (r *CompilerRecipe) Load(path string) error {
 			}
 			// Path is set only internally when tree is loaded from disk
 			pack.SetPath(filepath.Dir(currentpath))
+			pack.SetTreeDir(path)
 
 			// Instead of rdeps, have a different tree for build deps.
 			compileDefPath := pack.Rel(CompilerDefinitionFile)
@@ -126,18 +127,25 @@ func (r *CompilerRecipe) Load(path string) error {
 			}
 
 		case CollectionFile:
+
 			dat, err := ioutil.ReadFile(currentpath)
 			if err != nil {
 				return errors.Wrap(err, "Error reading file "+currentpath)
 			}
+
 			packs, err := pkg.DefaultPackagesFromYaml(dat)
 			if err != nil {
 				return errors.Wrap(err, "Error reading yaml "+currentpath)
 			}
+
 			packsRaw, err := pkg.GetRawPackages(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading raw packages from "+currentpath)
+			}
 
 			for _, pack := range packs {
 				pack.SetPath(filepath.Dir(currentpath))
+				pack.SetTreeDir(path)
 
 				// Instead of rdeps, have a different tree for build deps.
 				compileDefPath := pack.Rel(CompilerDefinitionFile)
@@ -170,9 +178,7 @@ func (r *CompilerRecipe) Load(path string) error {
 					return errors.Wrap(err, "Error creating package "+pack.GetName())
 				}
 			}
-
 		}
-
 		return nil
 	}