Tag 0.14.4

- Update vendor and run go mod tidy

cmd/build.go

@@ -16,14 +16,18 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	"github.com/mudler/luet/pkg/installer"
+
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -40,17 +44,17 @@ var buildCmd = &cobra.Command{
 	Long: `Builds one or more packages from a tree (current directory is implied):
 
 	$ luet build utils/busybox utils/yq ...
-	
+
 Builds all packages
-	
+
 	$ luet build --all
-	
+
 Builds only the leaf packages:
-	
+
 	$ luet build --full
 
 Build package revdeps:
-	
+
 	$ luet build --revdeps utils/yq
 
 Build package without dependencies (needs the images already in the host, or either need to be available online):
@@ -65,7 +69,6 @@ Build packages specifying multiple definition trees:
 		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
 		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
 		viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged"))
-		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
 		viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
 		viper.BindPFlag("all", cmd.Flags().Lookup("all"))
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
@@ -97,10 +100,9 @@ Build packages specifying multiple definition trees:
 		privileged := viper.GetBool("privileged")
 		revdeps := viper.GetBool("revdeps")
 		all := viper.GetBool("all")
-		databaseType := viper.GetString("database")
 		compressionType := viper.GetString("compression")
 		imageRepository := viper.GetString("image-repository")
-		values := viper.GetString("values")
+		values := viper.GetStringSlice("values")
 		wait := viper.GetBool("wait")
 		push := viper.GetBool("push")
 		pull := viper.GetBool("pull")
@@ -109,6 +111,8 @@ Build packages specifying multiple definition trees:
 		onlydeps := viper.GetBool("onlydeps")
 		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
 		full, _ := cmd.Flags().GetBool("full")
+		rebuild, _ := cmd.Flags().GetBool("rebuild")
+
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		var results Results
 		backendArgs := viper.GetStringSlice("backend-args")
@@ -118,34 +122,28 @@ Build packages specifying multiple definition trees:
 			LuetCfg.GetLogging().SetLogLevel("error")
 		}
 		pretend, _ := cmd.Flags().GetBool("pretend")
-		compilerSpecs := compiler.NewLuetCompilationspecs()
+		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
+
+		compilerSpecs := compilerspec.NewLuetCompilationspecs()
 		var db pkg.PackageDatabase
 
-		compilerBackend := backend.NewBackend(backendType)
+		compilerBackend, err := compiler.NewBackend(backendType)
+		helpers.CheckErr(err)
 
-		switch databaseType {
-		case "memory":
-			db = pkg.NewInMemoryDatabase(false)
-
-		case "boltdb":
-			tmpdir, err := ioutil.TempDir("", "package")
-			if err != nil {
-				Fatal(err)
-			}
-			db = pkg.NewBoltDatabase(tmpdir)
-
-		}
+		db = pkg.NewInMemoryDatabase(false)
 		defer db.Clean()
 
 		generalRecipe := tree.NewCompilerRecipe(db)
 
+		if fromRepo {
+			if err := installer.LoadBuildTree(generalRecipe, db, LuetCfg); err != nil {
+				Warning("errors while loading trees from repositories", err.Error())
+			}
+		}
+
 		for _, src := range treePaths {
 			Info("Loading tree", src)
-
-			err := generalRecipe.Load(src)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
+			helpers.CheckErr(generalRecipe.Load(src))
 		}
 
 		Info("Building in", dst)
@@ -154,38 +152,44 @@ Build packages specifying multiple definition trees:
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
 		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
-
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
 		LuetCfg.GetGeneral().ShowBuildOutput = LuetCfg.Viper.GetBool("general.show_build_output")
 
-		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
-
-		opts := compiler.NewDefaultCompilerOptions()
-		opts.SolverOptions = *LuetCfg.GetSolverOptions()
-		opts.ImageRepository = imageRepository
-		opts.PullFirst = pull
-		opts.KeepImg = keepImages
-		opts.Push = push
-		opts.OnlyDeps = onlydeps
-		opts.NoDeps = nodeps
-		opts.Wait = wait
-		opts.PackageTargetOnly = onlyTarget
-		opts.BuildValuesFile = values
-		var solverOpts solver.Options
-		if concurrent {
-			solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
-		} else {
-			solverOpts = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+		opts := &LuetSolverOptions{
+			Type:        stype,
+			LearnRate:   float32(rate),
+			Discount:    float32(discount),
+			MaxAttempts: attempts,
 		}
 
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts, solverOpts)
-		luetCompiler.SetBackendArgs(backendArgs)
-		luetCompiler.SetConcurrency(concurrency)
-		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
+		Debug("Solver", opts.CompactString())
+
+		if concurrent {
+			opts.Options = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
+		} else {
+			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+		}
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
+			options.NoDeps(nodeps),
+			options.WithBackendType(backendType),
+			options.PushImages(push),
+			options.WithBuildValues(values),
+			options.WithPullRepositories(pullRepo),
+			options.WithPushRepository(imageRepository),
+			options.Rebuild(rebuild),
+			options.WithSolverOptions(*opts),
+			options.Wait(wait),
+			options.OnlyTarget(onlyTarget),
+			options.PullFirst(pull),
+			options.KeepImg(keepImages),
+			options.OnlyDeps(onlydeps),
+			options.BackendArgs(backendArgs),
+			options.Concurrency(concurrency),
+			options.WithCompressionType(compression.Implementation(compressionType)),
+		)
+
 		if full {
 			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
 			if err != nil {
@@ -198,7 +202,6 @@ Build packages specifying multiple definition trees:
 			}
 		} else if !all {
 			for _, a := range args {
-
 				pack, err := helpers.ParsePackageStr(a)
 				if err != nil {
 					Fatal("Invalid package string ", a, ": ", err.Error())
@@ -226,13 +229,13 @@ Build packages specifying multiple definition trees:
 			}
 		}
 
-		var artifact []compiler.Artifact
+		var artifact []*artifact.PackageArtifact
 		var errs []error
 		if revdeps {
 			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
 		} else if pretend {
-			toCalculate := []compiler.CompilationSpec{}
+			toCalculate := []*compilerspec.LuetCompilationSpec{}
 			if full {
 				var err error
 				toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...)
@@ -282,6 +285,7 @@ Build packages specifying multiple definition trees:
 				}
 			}
 		} else {
+
 			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
 		}
 		if len(errs) != 0 {
@@ -291,7 +295,7 @@ Build packages specifying multiple definition trees:
 			Fatal("Bailing out")
 		}
 		for _, a := range artifact {
-			Info("Artifact generated:", a.GetPath())
+			Info("Artifact generated:", a.Path)
 		}
 	},
 }
@@ -304,12 +308,11 @@ func init() {
 
 	buildCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the tree to use.")
 	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
-	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
-	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
+	buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
-	buildCmd.Flags().String("values", "", "Build values file to interpolate with each package")
+	buildCmd.Flags().StringSlice("values", []string{}, "Build values file to interpolate with each package")
 	buildCmd.Flags().StringSliceP("backend-args", "a", []string{}, "Backend args")
 
 	buildCmd.Flags().String("destination", filepath.Join(path, "build"), "Destination folder")
@@ -328,8 +331,10 @@ func init() {
 	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	buildCmd.Flags().Bool("live-output", LuetCfg.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
-
+	buildCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
+	buildCmd.Flags().Bool("rebuild", false, "To combine with --pull. Allows to rebuild the target package even if an image is available, against a local values file")
 	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
+	buildCmd.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")
 
 	buildCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 

cmd/create-repo.go

@@ -18,11 +18,13 @@ import (
 	"os"
 	"path/filepath"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
-	. "github.com/mudler/luet/pkg/logger"
+
+	//	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
@@ -74,7 +76,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var err error
-		var repo installer.Repository
+		var repo *installer.LuetSystemRepository
 
 		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("output")
@@ -90,19 +92,19 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		metaName := viper.GetString("meta-filename")
 		source_repo := viper.GetString("repo")
 		backendType := viper.GetString("backend")
+		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
 
 		treeFile := installer.NewDefaultTreeRepositoryFile()
 		metaFile := installer.NewDefaultMetaRepositoryFile()
-		compilerBackend := backend.NewBackend(backendType)
+		compilerBackend, err := compiler.NewBackend(backendType)
+		helpers.CheckErr(err)
 		force := viper.GetBool("force-push")
 		imagePush := viper.GetBool("push-images")
 
 		if source_repo != "" {
 			// Search for system repository
 			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
+			helpers.CheckErr(err)
 
 			if len(treePaths) <= 0 {
 				treePaths = []string{lrepo.TreePath}
@@ -118,19 +120,23 @@ Create a repository from the metadata description defined in the luet.yaml confi
 				lrepo.Priority,
 				packages,
 				treePaths,
-				pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force)
+				pkg.NewInMemoryDatabase(false),
+				compilerBackend,
+				dst,
+				imagePush,
+				force,
+				fromRepo,
+				LuetCfg)
+			helpers.CheckErr(err)
 
 		} else {
 			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
-				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force)
-		}
-
-		if err != nil {
-			Fatal("Error: " + err.Error())
+				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force, fromRepo, LuetCfg)
+			helpers.CheckErr(err)
 		}
 
 		if treetype != "" {
-			treeFile.SetCompressionType(compiler.CompressionImplementation(treetype))
+			treeFile.SetCompressionType(compression.Implementation(treetype))
 		}
 
 		if treeName != "" {
@@ -138,7 +144,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		}
 
 		if metatype != "" {
-			metaFile.SetCompressionType(compiler.CompressionImplementation(metatype))
+			metaFile.SetCompressionType(compression.Implementation(metatype))
 		}
 
 		if metaName != "" {
@@ -149,17 +155,15 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
 		err = repo.Write(dst, reset, true)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
+		helpers.CheckErr(err)
+
 	},
 }
 
 func init() {
 	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
+	helpers.CheckErr(err)
+
 	createrepoCmd.Flags().String("packages", filepath.Join(path, "build"), "Packages folder (output from build)")
 	createrepoCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the source trees to use.")
 	createrepoCmd.Flags().String("output", filepath.Join(path, "build"), "Destination for generated archives. With 'docker' repository type, it should be an image reference (e.g 'foo/bar')")
@@ -178,6 +182,7 @@ func init() {
 	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
 	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
+	createrepoCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

cmd/database/create.go

@@ -18,7 +18,8 @@ package cmd_database
 import (
 	"io/ioutil"
 
-	"github.com/mudler/luet/pkg/compiler"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -66,21 +67,21 @@ For reference, inspect a "metadata.yaml" file generated while running "luet buil
 				if err != nil {
 					Fatal("Failed reading ", a, ": ", err.Error())
 				}
-				art, err := compiler.NewPackageArtifactFromYaml(dat)
+				art, err := artifact.NewPackageArtifactFromYaml(dat)
 				if err != nil {
 					Fatal("Failed reading yaml ", a, ": ", err.Error())
 				}
 
-				files := art.GetFiles()
+				files := art.Files
 
-				if _, err := systemDB.CreatePackage(art.GetCompileSpec().GetPackage()); err != nil {
+				if _, err := systemDB.CreatePackage(art.CompileSpec.GetPackage()); err != nil {
 					Fatal("Failed to create ", a, ": ", err.Error())
 				}
-				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.GetCompileSpec().GetPackage().GetFingerPrint(), Files: files}); err != nil {
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.CompileSpec.GetPackage().GetFingerPrint(), Files: files}); err != nil {
 					Fatal("Failed setting package files for ", a, ": ", err.Error())
 				}
 
-				Info(art.GetCompileSpec().GetPackage().HumanReadableString(), " created")
+				Info(art.CompileSpec.GetPackage().HumanReadableString(), " created")
 			}
 
 		},

cmd/helpers/cli.go

@@ -22,6 +22,8 @@ import (
 	"regexp"
 	"strings"
 
+	. "github.com/mudler/luet/pkg/logger"
+
 	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	pkg "github.com/mudler/luet/pkg/package"
 	version "github.com/mudler/luet/pkg/versioner"
@@ -56,7 +58,8 @@ func packageData(p string) (string, string) {
 }
 func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
 
-	if !strings.HasPrefix(p, "=") {
+	if !(strings.HasPrefix(p, "=") || strings.HasPrefix(p, ">") ||
+		strings.HasPrefix(p, "<")) {
 		ver := ">=0"
 		cat := ""
 		name := ""
@@ -111,3 +114,9 @@ func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
 
 	return pack, nil
 }
+
+func CheckErr(err error) {
+	if err != nil {
+		Fatal(err)
+	}
+}

cmd/helpers/cli_test.go

@@ -66,5 +66,37 @@ var _ = Describe("CLI Helpers", func() {
 			Expect(pack.GetCategory()).To(Equal("cat"))
 			Expect(pack.GetVersion()).To(Equal("1.2"))
 		})
+
+		It("accept gentoo regex parsing with with condition", func() {
+			pack, err := ParsePackageStr(">=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition2", func() {
+			pack, err := ParsePackageStr("<cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("<1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition3", func() {
+			pack, err := ParsePackageStr(">cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition4", func() {
+			pack, err := ParsePackageStr("<=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("<=1.2"))
+		})
 	})
 })

cmd/install.go

@@ -70,16 +70,6 @@ To force install a package:
 			toInstall = append(toInstall, pack)
 		}
 
-		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
-		repos := installer.Repositories{}
-		for _, repo := range LuetCfg.SystemRepositories {
-			if !repo.Enable {
-				continue
-			}
-			r := installer.NewSystemRepository(repo)
-			repos = append(repos, r)
-		}
-
 		stype := LuetCfg.Viper.GetString("solver.type")
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
@@ -111,6 +101,7 @@ To force install a package:
 		}
 
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		repos := installer.SystemRepositories(LuetCfg)
 
 		// Load config protect configs
 		installer.LoadConfigProtectConfs(LuetCfg)

cmd/pack.go

@@ -20,7 +20,9 @@ import (
 	"time"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
-	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -64,21 +66,21 @@ Afterwards, you can use the content generated and associate it with a tree and a
 			Fatal("Invalid package string ", packageName, ": ", err.Error())
 		}
 
-		spec := &compiler.LuetCompilationSpec{Package: p}
-		artifact := compiler.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
-		artifact.SetCompressionType(compiler.CompressionImplementation(compressionType))
-		err = artifact.Compress(sourcePath, concurrency)
+		spec := &compilerspec.LuetCompilationSpec{Package: p}
+		a := artifact.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
+		a.CompressionType = compression.Implementation(compressionType)
+		err = a.Compress(sourcePath, concurrency)
 		if err != nil {
 			Fatal("failed compressing ", packageName, ": ", err.Error())
 		}
-		artifact.SetCompileSpec(spec)
-		filelist, err := artifact.FileList()
+		a.CompileSpec = spec
+		filelist, err := a.FileList()
 		if err != nil {
 			Fatal("failed generating file list for ", packageName, ": ", err.Error())
 		}
-		artifact.SetFiles(filelist)
-		artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
-		err = artifact.WriteYaml(dst)
+		a.Files = filelist
+		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
+		err = a.WriteYaml(dst)
 		if err != nil {
 			Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
 		}

cmd/repo/list.go

@@ -72,8 +72,8 @@ func NewRepoListCommand() *cobra.Command {
 					if repo.Cached {
 
 						r := installer.NewSystemRepository(repo)
-						localRepo, _ := r.(*installer.LuetSystemRepository).ReadSpecFile(filepath.Join(repobasedir,
-							installer.REPOSITORY_SPECFILE), false)
+						localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir,
+							installer.REPOSITORY_SPECFILE))
 						if localRepo != nil {
 							tsec, _ := strconv.ParseInt(localRepo.GetLastUpdate(), 10, 64)
 							repoRevision = Bold(Red(localRepo.GetRevision())).String() +

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.11.7"
+	LuetCLIVersion = "0.14.4"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/search.go

@@ -172,7 +172,7 @@ func searchOnline(term string, l list.Writer, t table.Writer, label, labelMatch,
 					Hidden:     m.Package.IsHidden(),
 				}
 				if m.Artifact != nil {
-					r.Files = m.Artifact.GetFiles()
+					r.Files = m.Artifact.Files
 				}
 				results.Packages = append(results.Packages, *r)
 			}
@@ -190,7 +190,7 @@ func searchOnline(term string, l list.Writer, t table.Writer, label, labelMatch,
 						Hidden:     revdep.IsHidden(),
 					}
 					if m.Artifact != nil {
-						r.Files = m.Artifact.GetFiles()
+						r.Files = m.Artifact.Files
 					}
 					results.Packages = append(results.Packages, *r)
 				}
@@ -262,7 +262,7 @@ func searchFiles(term string, l list.Writer, t table.Writer) Results {
 				Category:   m.Package.GetCategory(),
 				Repository: m.Repo.GetName(),
 				Hidden:     m.Package.IsHidden(),
-				Files:      m.Artifact.GetFiles(),
+				Files:      m.Artifact.Files,
 			})
 	}
 	return results

cmd/tree/images.go

@@ -25,6 +25,7 @@ import (
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/options"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -57,6 +58,7 @@ func NewTreeImageCommand() *cobra.Command {
 
 			treePath, _ := cmd.Flags().GetStringArray("tree")
 			imageRepository := viper.GetString("image-repository")
+			pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
 			out, _ := cmd.Flags().GetString("output")
 			if out != "terminal" {
@@ -73,12 +75,15 @@ func NewTreeImageCommand() *cobra.Command {
 			}
 			compilerBackend := backend.NewSimpleDockerBackend()
 
-			opts := compiler.NewDefaultCompilerOptions()
-			opts.SolverOptions = *LuetCfg.GetSolverOptions()
-			opts.ImageRepository = imageRepository
-
-			solverOpts := solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
-			luetCompiler := compiler.NewLuetCompiler(compilerBackend, reciper.GetDatabase(), opts, solverOpts)
+			opts := *LuetCfg.GetSolverOptions()
+			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
+			luetCompiler := compiler.NewLuetCompiler(
+				compilerBackend,
+				reciper.GetDatabase(),
+				options.WithPushRepository(imageRepository),
+				options.WithPullRepositories(pullRepo),
+				options.WithSolverOptions(opts),
+			)
 
 			a := args[0]
 
@@ -135,6 +140,7 @@ func NewTreeImageCommand() *cobra.Command {
 	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
 	ans.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+	ans.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")
 
 	return ans
 }

go.mod

@@ -4,7 +4,7 @@ go 1.14
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/Sabayon/pkgs-checker v0.7.2
+	github.com/Sabayon/pkgs-checker v0.8.1
 	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/briandowns/spinner v1.12.1-0.20201220203425-e201aaea0a31
@@ -16,12 +16,13 @@ require (
 	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
-	github.com/fsouza/go-dockerclient v1.6.4
 	github.com/genuinetools/img v0.5.11
 	github.com/ghodss/yaml v1.0.0
 	github.com/google/go-containerregistry v0.2.1
+	github.com/google/renameio v1.0.0
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/hashicorp/go-version v1.2.0
+	github.com/hashicorp/go-version v1.2.1
+	github.com/imdario/mergo v0.3.8
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/jedib0t/go-pretty/v6 v6.0.5
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
@@ -35,7 +36,7 @@ require (
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+	github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3
@@ -45,13 +46,13 @@ require (
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
 	github.com/pkg/errors v0.9.1
 	github.com/schollz/progressbar/v3 v3.7.1
-	github.com/sirupsen/logrus v1.6.0
-	github.com/spf13/cobra v1.0.0
-	github.com/spf13/viper v1.7.0
+	github.com/sirupsen/logrus v1.7.0
+	github.com/spf13/cobra v1.1.1
+	github.com/spf13/viper v1.7.1
 	github.com/theupdateframework/notary v0.7.0
 	go.etcd.io/bbolt v1.3.5
 	go.uber.org/atomic v1.5.1 // indirect
-	go.uber.org/multierr v1.4.0 // indirect
+	go.uber.org/multierr v1.4.0
 	go.uber.org/zap v1.13.0
 	google.golang.org/grpc v1.29.1
 	gopkg.in/yaml.v2 v2.3.0

go.sum

@@ -81,6 +81,8 @@ github.com/Microsoft/go-winio v0.4.15-0.20200113171025-3fe6c5262873/go.mod h1:tT
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg=
 github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
+github.com/MottainaiCI/simplestreams-builder v0.1.0 h1:A8KJN22Xkx7NUKC9/zWmd1UhIqRn3bdHo0wv/HsAHx8=
+github.com/MottainaiCI/simplestreams-builder v0.1.0/go.mod h1:+Gbv6dg6TPHWq4oDjZY1vn978PLCEZ2hOu8kvn+S7t4=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -89,11 +91,12 @@ github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/Sabayon/pkgs-checker v0.7.2 h1:mh53u5D7FTCeBJevYQA9cCxAWGTSuKqw7m/x7GsQVb0=
-github.com/Sabayon/pkgs-checker v0.7.2/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
+github.com/Sabayon/pkgs-checker v0.8.1 h1:pVen975z9WIecq7luntUn+0XzGdiyz2CsDay8w+ZmOw=
+github.com/Sabayon/pkgs-checker v0.8.1/go.mod h1:GC9PBUzcq0QVEBGRA1IIMXf6wHxo34KH5BeqoyJsLpo=
 github.com/Sereal/Sereal v0.0.0-20181211220259-509a78ddbda3 h1:Xu7z47ZiE/J+sKXHZMGxEor/oY2q6dq51fkO0JqdSwY=
 github.com/Sereal/Sereal v0.0.0-20181211220259-509a78ddbda3/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM=
 github.com/Shopify/logrus-bugsnag v0.0.0-20170309145241-6dbc35f2c30d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
+github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
@@ -142,18 +145,24 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=
 github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
+github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y=
 github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/briandowns/spinner v1.12.1-0.20201220203425-e201aaea0a31 h1:yInAg9pE5qGec5eQ7XdfOTTaGwGxD3bKFVjmD6VKkwc=
 github.com/briandowns/spinner v1.12.1-0.20201220203425-e201aaea0a31/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
 github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
+github.com/bugsnag/bugsnag-go v1.0.5-0.20150529004307-13fd6b8acda0 h1:s7+5BfS4WFJoVF9pnB8kBk03S7pZXRdKamnV0FOl5Sc=
 github.com/bugsnag/bugsnag-go v1.0.5-0.20150529004307-13fd6b8acda0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
+github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
+github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec h1:4XvMn0XuV7qxCH22gbnR79r+xTUaLOSA0GW/egpO3SQ=
@@ -173,13 +182,12 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
 github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 h1:lkAMpLVBDaj17e85keuznYcH5rqI438v41pKcBl4ZxQ=
 github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d h1:UKAt78F1OvM4ceTn1VvXuYuatXohsFU1eSI2IBtTw9g=
@@ -251,6 +259,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/deislabs/oras v0.8.1/go.mod h1:Mx0rMSbBNaNfY9hjpccEnxkOqJL6KGjtxNHPLC4G4As=
 github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73 h1:OGNva6WhsKst5OZf7eZOklDztV3hwtTHovdrLHV+MsA=
 github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@@ -311,6 +320,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
@@ -396,9 +406,12 @@ github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85n
 github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
+github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
 github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
 github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w=
 github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs=
@@ -426,6 +439,7 @@ github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5
 github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -464,6 +478,7 @@ github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkY
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/certificate-transparency-go v1.0.10-0.20180222191210-5ab67e519c93 h1:jc2UWq7CbdszqeH6qu1ougXMIUBfSy8Pbh/anURYbGI=
 github.com/google/certificate-transparency-go v1.0.10-0.20180222191210-5ab67e519c93/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -487,6 +502,8 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/renameio v1.0.0 h1:xhp2CnJmgQmpJU4RY8chagahUq5mbPPAbiSQstKpVMA=
+github.com/google/renameio v1.0.0/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk=
 github.com/google/shlex v0.0.0-20150127133951-6f45313302b9 h1:JM174NTeGNJ2m/oLH3UOWOvWQQKd+BoL3hcSCUWFLt0=
 github.com/google/shlex v0.0.0-20150127133951-6f45313302b9/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -524,6 +541,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
+github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
@@ -543,6 +561,8 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
+github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
@@ -581,8 +601,11 @@ github.com/jedib0t/go-pretty/v6 v6.0.5/go.mod h1:MTr6FgcfNdnN5wPVBzJ6mhJeDyiF0yB
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 h1:sHsPfNMAG70QAvKbddQ0uScZCHQoZsT5NykGRCeeeIs=
 github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
+github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8 h1:CZkYfurY6KGhVtlalI4QwQ6T0Cu6iuY3e0x5RLu96WE=
 github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
+github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d h1:jRQLvyVGL+iVtDElaEIDdKwpPqUIZJfzkNLV34htpEc=
 github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
 github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -603,6 +626,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI=
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
@@ -639,6 +663,7 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm
 github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY=
 github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
@@ -681,6 +706,8 @@ github.com/mattn/go-sqlite3 v1.6.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOq
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do=
 github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI=
+github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
@@ -731,8 +758,8 @@ github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d h1:fKh+rvw
 github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d/go.mod h1:puRUWSwyecW2V355tKncwPVPRAjQBduPsFjG0mrV/Nw=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87 h1:mGz7T8KvmHH0gLWPI5tQne8xl2cO3T8wrrb6Aa16Jxo=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87/go.mod h1:1w4zI1LYXDeiUXqedPcrT5eQJnmKR6dbg5iJMgSIP/Y=
-github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82 h1:Hkefw2tzoKATVUTFsCtDlUnY180+OE851qGbq45ATxk=
-github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82/go.mod h1:4P/ULate+2QxoAQtojaRjyO5VGMhV0KLnSdAS8nuBbo=
+github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af h1:jixIxEgLSqu24eMiyzfCI+roa5IaOUhF546ePSFyHeY=
+github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290 h1:426hFyXMpXeqIeGJn2cGAW9ogvM2Jf+Jv23gtVPvBLM=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290/go.mod h1:uP5BBgFxq2wNWo7n1vnY5SSbgL0WDshVJrOO12tZ/lA=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
@@ -926,6 +953,8 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
@@ -953,6 +982,8 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
@@ -971,6 +1002,8 @@ github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs=
 github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw=
 github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
+github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
@@ -1432,6 +1465,7 @@ google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEG
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII=
 gopkg.in/cenkalti/backoff.v2 v2.2.1/go.mod h1:S0QdOvT2AlerfSBkp0O+dk+bbIMaNbEmVk876gPCthU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1454,6 +1488,7 @@ gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 h1:d4KQkxAaAiRY2h5Zqis161Pv91A37uZyJOx73duwUwM=
 gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1/go.mod h1:WbjuEoo1oadwzQ4apSDU+JTvmllEHtsNHS6y7vFc7iw=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=

pkg/compiler/backend.go

@@ -1,33 +1,52 @@
-// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package backend
+package compiler
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
-	. "github.com/mudler/luet/pkg/logger"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 
-	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/pkg/errors"
+
+	. "github.com/mudler/luet/pkg/logger"
 )
 
+func NewBackend(s string) (CompilerBackend, error) {
+	var compilerBackend CompilerBackend
+
+	switch s {
+	case backend.ImgBackend:
+		compilerBackend = backend.NewSimpleImgBackend()
+	case backend.DockerBackend:
+		compilerBackend = backend.NewSimpleDockerBackend()
+	default:
+		return nil, errors.New("invalid backend. Unsupported")
+	}
+
+	return compilerBackend, nil
+}
+
+type CompilerBackend interface {
+	BuildImage(backend.Options) error
+	ExportImage(backend.Options) error
+	RemoveImage(backend.Options) error
+	ImageDefinitionToTar(backend.Options) error
+	ExtractRootfs(opts backend.Options, keepPerms bool) error
+
+	CopyImage(string, string) error
+	DownloadImage(opts backend.Options) error
+
+	Push(opts backend.Options) error
+	ImageAvailable(string) bool
+
+	ImageExists(string) bool
+}
+
 // GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
 // example of json return: [
 //   {
@@ -54,46 +73,46 @@ import (
 //     }
 //   }
 // ]
-func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
+func GenerateChanges(b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) {
 
-	res := compiler.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
+	res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
 
 	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
 	defer os.RemoveAll(tmpdiffs) // clean up
 
 	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
 	defer os.RemoveAll(srcRootFS) // clean up
 
 	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
 	defer os.RemoveAll(dstRootFS) // clean up
 
-	srcImageExtract := compiler.CompilerBackendOptions{
+	srcImageExtract := backend.Options{
 		ImageName:   fromImage.ImageName,
 		Destination: srcRootFS,
 	}
 	Debug("Extracting source image", fromImage.ImageName)
 	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
 	}
 
-	dstImageExtract := compiler.CompilerBackendOptions{
+	dstImageExtract := backend.Options{
 		ImageName:   toImage.ImageName,
 		Destination: dstRootFS,
 	}
 	Debug("Extracting destination image", toImage.ImageName)
 	err = b.ExtractRootfs(dstImageExtract, false)
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
 	}
 
 	// Get Additions/Changes. dst -> src
@@ -114,7 +133,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
 
 			if sizeA != sizeB {
 				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
-				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
+				res.Diffs.Changes = append(res.Diffs.Changes, artifact.ArtifactNode{
 					Name: filepath.Join("/", realpath),
 					Size: int(sizeB),
 				})
@@ -127,7 +146,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
 			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
 				sizeB = s.Size()
 			}
-			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
+			res.Diffs.Additions = append(res.Diffs.Additions, artifact.ArtifactNode{
 				Name: filepath.Join("/", realpath),
 				Size: int(sizeB),
 			})
@@ -138,7 +157,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
 		return nil
 	})
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
 	}
 
 	// Get deletions. src -> dst
@@ -150,7 +169,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
 		realpath := strings.Replace(path, srcRootFS, "", -1)
 		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
 			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
-			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
+			res.Diffs.Deletions = append(res.Diffs.Deletions, artifact.ArtifactNode{
 				Name: filepath.Join("/", realpath),
 			})
 		}
@@ -158,8 +177,71 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
 		return nil
 	})
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
 	}
 
-	return []compiler.ArtifactLayer{res}, nil
+	diffs := []artifact.ArtifactLayer{res}
+
+	if config.LuetCfg.GetGeneral().Debug {
+		summary := ComputeArtifactLayerSummary(diffs)
+		for _, l := range summary.Layers {
+			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
+				l.FromImage, l.ToImage,
+				l.AddFiles, l.AddSizes,
+				l.DelFiles, l.DelSizes,
+				l.ChangeFiles, l.ChangeSizes))
+		}
+	}
+
+	return diffs, nil
+}
+
+type ArtifactLayerSummary struct {
+	FromImage   string `json:"image1"`
+	ToImage     string `json:"image2"`
+	AddFiles    int    `json:"add_files"`
+	AddSizes    int64  `json:"add_sizes"`
+	DelFiles    int    `json:"del_files"`
+	DelSizes    int64  `json:"del_sizes"`
+	ChangeFiles int    `json:"change_files"`
+	ChangeSizes int64  `json:"change_sizes"`
+}
+
+type ArtifactLayersSummary struct {
+	Layers []ArtifactLayerSummary `json:"summary"`
+}
+
+func ComputeArtifactLayerSummary(diffs []artifact.ArtifactLayer) ArtifactLayersSummary {
+
+	ans := ArtifactLayersSummary{
+		Layers: make([]ArtifactLayerSummary, 0),
+	}
+
+	for _, layer := range diffs {
+		sum := ArtifactLayerSummary{
+			FromImage:   layer.FromImage,
+			ToImage:     layer.ToImage,
+			AddFiles:    0,
+			AddSizes:    0,
+			DelFiles:    0,
+			DelSizes:    0,
+			ChangeFiles: 0,
+			ChangeSizes: 0,
+		}
+		for _, a := range layer.Diffs.Additions {
+			sum.AddFiles++
+			sum.AddSizes += int64(a.Size)
+		}
+		for _, d := range layer.Diffs.Deletions {
+			sum.DelFiles++
+			sum.DelSizes += int64(d.Size)
+		}
+		for _, c := range layer.Diffs.Changes {
+			sum.ChangeFiles++
+			sum.ChangeSizes += int64(c.Size)
+		}
+		ans.Layers = append(ans.Layers, sum)
+	}
+
+	return ans
 }

pkg/compiler/backend/common.go

@@ -18,7 +18,6 @@ package backend
 import (
 	"os/exec"
 
-	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -36,16 +35,13 @@ func imageAvailable(image string) bool {
 	return err == nil
 }
 
-func NewBackend(s string) compiler.CompilerBackend {
-	var compilerBackend compiler.CompilerBackend
-
-	switch s {
-	case ImgBackend:
-		compilerBackend = NewSimpleImgBackend()
-	case DockerBackend:
-		compilerBackend = NewSimpleDockerBackend()
-	}
-	return compilerBackend
+type Options struct {
+	ImageName      string
+	SourcePath     string
+	DockerFileName string
+	Destination    string
+	Context        string
+	BackendArgs    []string
 }
 
 func runCommand(cmd *exec.Cmd) error {
@@ -75,7 +71,7 @@ func runCommand(cmd *exec.Cmd) error {
 	return nil
 }
 
-func genBuildCommand(opts compiler.CompilerBackendOptions) []string {
+func genBuildCommand(opts Options) []string {
 	context := opts.Context
 
 	if context == "" {

pkg/compiler/backend/simpledocker.go

@@ -17,7 +17,6 @@ package backend
 
 import (
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -26,11 +25,8 @@ import (
 
 	bus "github.com/mudler/luet/pkg/bus"
 
-	docker "github.com/fsouza/go-dockerclient"
 	capi "github.com/mudler/docker-companion/api"
 
-	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -39,12 +35,12 @@ import (
 
 type SimpleDocker struct{}
 
-func NewSimpleDockerBackend() compiler.CompilerBackend {
+func NewSimpleDockerBackend() *SimpleDocker {
 	return &SimpleDocker{}
 }
 
 // TODO: Missing still: labels, and build args expansion
-func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) BuildImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePreBuild, opts)
 
@@ -59,24 +55,6 @@ func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
 
 	Info(":whale: Building image " + name + " done")
 
-	if os.Getenv("DOCKER_SQUASH") == "true" {
-		Info(":whale: Squashing image " + name)
-		var client *docker.Client
-
-		Spinner(22)
-		defer SpinnerStop()
-		client, err = docker.NewClientFromEnv()
-		if err != nil {
-			return errors.Wrap(err, "could not connect to the Docker daemon")
-		}
-		err = capi.Squash(client, name, name)
-		if err != nil {
-			return errors.Wrap(err, "Failed squashing image")
-		}
-
-		Info(":whale: Squashing image " + name + " done")
-	}
-
 	bus.Manager.Publish(bus.EventImagePostBuild, opts)
 
 	return nil
@@ -93,7 +71,7 @@ func (*SimpleDocker) CopyImage(src, dst string) error {
 	return nil
 }
 
-func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) DownloadImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePull, opts)
 
@@ -132,7 +110,7 @@ func (*SimpleDocker) ImageAvailable(imagename string) bool {
 	return imageAvailable(imagename)
 }
 
-func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) RemoveImage(opts Options) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
 	out, err := exec.Command("docker", buildarg...).CombinedOutput()
@@ -144,7 +122,7 @@ func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
-func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) Push(opts Options) error {
 	name := opts.ImageName
 	pusharg := []string{"push", name}
 	bus.Manager.Publish(bus.EventImagePrePush, opts)
@@ -163,7 +141,7 @@ func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
-func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
 	}
@@ -176,7 +154,7 @@ func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions
 	return nil
 }
 
-func (*SimpleDocker) ExportImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) ExportImage(opts Options) error {
 	name := opts.ImageName
 	path := opts.Destination
 
@@ -199,7 +177,7 @@ type ManifestEntry struct {
 	Layers []string `json:"Layers"`
 }
 
-func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
+func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
 	name := opts.ImageName
 	dst := opts.Destination
 
@@ -214,7 +192,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
 	Spinner(22)
 	defer SpinnerStop()
 
-	if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: name, Destination: imageExport}); err != nil {
+	if err := b.ExportImage(Options{ImageName: name, Destination: imageExport}); err != nil {
 		return errors.Wrap(err, "failed while extracting rootfs for "+name)
 	}
 
@@ -229,7 +207,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
 		}
 		defer os.RemoveAll(tempUnpack) // clean up
 		imageExport := filepath.Join(tempUnpack, "image.tar")
-		if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
+		if err := b.ExportImage(Options{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
 			return errors.Wrap(err, "while exporting image before extraction")
 		}
 		src = imageExport
@@ -281,7 +259,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
 		return err
 	}
 
-	err = export.UnPackLayers(layers_sha, dst, "")
+	err = export.UnPackLayers(layers_sha, dst, "containerd")
 	if err != nil {
 		return err
 	}
@@ -293,21 +271,3 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
 
 	return nil
 }
-
-// Changes retrieves changes between image layers
-func (d *SimpleDocker) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
-	diffs, err := GenerateChanges(d, fromImage, toImage)
-
-	if config.LuetCfg.GetGeneral().Debug {
-		summary := compiler.ComputeArtifactLayerSummary(diffs)
-		for _, l := range summary.Layers {
-			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
-				l.FromImage, l.ToImage,
-				l.AddFiles, l.AddSizes,
-				l.DelFiles, l.DelSizes,
-				l.ChangeFiles, l.ChangeSizes))
-		}
-	}
-
-	return diffs, err
-}

pkg/compiler/backend/simpledocker_test.go

@@ -16,9 +16,11 @@
 package backend_test
 
 import (
+	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
-	"github.com/mudler/luet/pkg/solver"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 
 	"io/ioutil"
 	"os"
@@ -41,13 +43,10 @@ var _ = Describe("Docker backend", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
 			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 			Expect(lspec.Image).To(Equal("luet/base"))
 			Expect(lspec.Seed).To(Equal("alpine"))
@@ -71,7 +70,7 @@ ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin`))
 			b := NewSimpleDockerBackend()
-			opts := CompilerBackendOptions{
+			opts := backend.Options{
 				ImageName:      "luet/base",
 				SourcePath:     tmpdir,
 				DockerFileName: "Dockerfile",
@@ -95,7 +94,7 @@ ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
-			opts2 := CompilerBackendOptions{
+			opts2 := backend.Options{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
@@ -106,26 +105,26 @@ RUN echo bar > /test2`))
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
-			artifacts := []ArtifactNode{{
+			artifacts := []artifact.ArtifactNode{{
 				Name: "/luetbuild/LuetDockerfile",
 				Size: 175,
 			}}
 			if os.Getenv("DOCKER_BUILDKIT") == "1" {
-				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+				artifacts = append(artifacts, artifact.ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
 			}
-			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
-			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4})
 
-			Expect(b.Changes(opts, opts2)).To(Equal(
-				[]ArtifactLayer{{
+			Expect(compiler.GenerateChanges(b, opts, opts2)).To(Equal(
+				[]artifact.ArtifactLayer{{
 					FromImage: "luet/base",
 					ToImage:   "test",
-					Diffs: ArtifactDiffs{
+					Diffs: artifact.ArtifactDiffs{
 						Additions: artifacts,
 					},
 				}}))
 
-			opts2 = CompilerBackendOptions{
+			opts2 = backend.Options{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",

pkg/compiler/backend/simpleimg.go

@@ -22,7 +22,6 @@ import (
 
 	bus "github.com/mudler/luet/pkg/bus"
 
-	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/pkg/errors"
@@ -30,12 +29,12 @@ import (
 
 type SimpleImg struct{}
 
-func NewSimpleImgBackend() compiler.CompilerBackend {
+func NewSimpleImgBackend() *SimpleImg {
 	return &SimpleImg{}
 }
 
 // TODO: Missing still: labels, and build args expansion
-func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) BuildImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePreBuild, opts)
 
@@ -56,7 +55,7 @@ func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
-func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) RemoveImage(opts Options) error {
 	name := opts.ImageName
 	buildarg := []string{"rm", name}
 	Spinner(22)
@@ -70,7 +69,7 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
-func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) DownloadImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePull, opts)
 
@@ -123,7 +122,7 @@ func (*SimpleImg) ImageExists(imagename string) bool {
 	return false
 }
 
-func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleImg) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
 	}
@@ -136,7 +135,7 @@ func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) e
 	return nil
 }
 
-func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) ExportImage(opts Options) error {
 	name := opts.ImageName
 	path := opts.Destination
 	buildarg := []string{"save", "-o", path, name}
@@ -154,7 +153,7 @@ func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
 }
 
 // ExtractRootfs extracts the docker image content inside the destination
-func (s *SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
+func (s *SimpleImg) ExtractRootfs(opts Options, keepPerms bool) error {
 	name := opts.ImageName
 	path := opts.Destination
 
@@ -180,13 +179,7 @@ func (s *SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerm
 	return nil
 }
 
-// TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (i *SimpleImg) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
-	return GenerateChanges(i, fromImage, toImage)
-}
-
-func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) Push(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePush, opts)
 

pkg/compiler/backend_test.go

@@ -13,10 +13,10 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package backend_test
+package compiler_test
 
 import (
-	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
 
 	. "github.com/onsi/ginkgo"
@@ -24,7 +24,7 @@ import (
 )
 
 var _ = Describe("Docker image diffs", func() {
-	var b compiler.CompilerBackend
+	var b CompilerBackend
 
 	BeforeEach(func() {
 		b = NewSimpleDockerBackend()
@@ -32,7 +32,7 @@ var _ = Describe("Docker image diffs", func() {
 
 	Context("Generate diffs from docker images", func() {
 		It("Detect no changes", func() {
-			opts := compiler.CompilerBackendOptions{
+			opts := Options{
 				ImageName: "alpine:latest",
 			}
 			err := b.DownloadImage(opts)
@@ -47,18 +47,18 @@ var _ = Describe("Docker image diffs", func() {
 		})
 
 		It("Detects additions and changed files", func() {
-			err := b.DownloadImage(compiler.CompilerBackendOptions{
+			err := b.DownloadImage(Options{
 				ImageName: "quay.io/mocaccino/micro",
 			})
 			Expect(err).ToNot(HaveOccurred())
-			err = b.DownloadImage(compiler.CompilerBackendOptions{
+			err = b.DownloadImage(Options{
 				ImageName: "quay.io/mocaccino/extra",
 			})
 			Expect(err).ToNot(HaveOccurred())
 
-			layers, err := GenerateChanges(b, compiler.CompilerBackendOptions{
+			layers, err := GenerateChanges(b, Options{
 				ImageName: "quay.io/mocaccino/micro",
-			}, compiler.CompilerBackendOptions{
+			}, Options{
 				ImageName: "quay.io/mocaccino/extra",
 			})
 			Expect(err).ToNot(HaveOccurred())

pkg/compiler/compiler_test.go

@@ -21,9 +21,11 @@ import (
 
 	. "github.com/mudler/luet/pkg/compiler"
 	sd "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -39,7 +41,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -54,12 +56,11 @@ var _ = Describe("Compiler", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -83,7 +84,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -98,12 +99,11 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 		})
@@ -118,7 +118,7 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/templates")
 			Expect(err).ToNot(HaveOccurred())
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 			pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -142,7 +142,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -159,15 +159,14 @@ var _ = Describe("Compiler", func() {
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2, spec3))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(3))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
@@ -199,7 +198,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -207,23 +206,22 @@ var _ = Describe("Compiler", func() {
 			Expect(err).ToNot(HaveOccurred())
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
-			artifacts2, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec2))
+			artifacts2, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts2)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			for _, artifact := range artifacts2 {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			Expect(helpers.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
@@ -241,7 +239,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -250,15 +248,14 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -276,7 +273,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -285,15 +282,14 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -312,7 +308,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -321,15 +317,14 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -348,7 +343,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -357,14 +352,13 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
@@ -382,7 +376,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -391,14 +385,13 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
@@ -416,7 +409,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -425,14 +418,13 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -454,7 +446,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -463,15 +455,14 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
@@ -495,7 +486,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -504,16 +495,15 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
@@ -539,7 +529,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -548,16 +538,15 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
@@ -581,7 +570,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -591,13 +580,13 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(2))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
@@ -619,7 +608,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
 			Expect(err).ToNot(HaveOccurred())
@@ -629,13 +618,13 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(6))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(6))
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
@@ -658,19 +647,19 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(4))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			// A deps on B, so A artifacts are here:
@@ -710,7 +699,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -722,17 +711,16 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-				for _, d := range artifact.GetDependencies() {
-					Expect(helpers.Exists(d.GetPath())).To(BeTrue())
-					Expect(helpers.Untar(d.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				for _, d := range artifact.Dependencies {
+					Expect(helpers.Exists(d.Path)).To(BeTrue())
+					Expect(helpers.Untar(d.Path, tmpdir, false)).ToNot(HaveOccurred())
 				}
 			}
 
@@ -753,7 +741,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -765,12 +753,11 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 			Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
@@ -786,7 +773,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{
 				Name:     "dironly",
@@ -814,12 +801,10 @@ var _ = Describe("Compiler", func() {
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir2)
 
-			compiler.SetConcurrency(1)
-
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(2))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(0))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(0))
 
 			Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
@@ -841,11 +826,11 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
-			compiler.SetCompressionType(GZip)
+			compiler.Options.CompressionType = compression.GZip
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
 
 			tmpdir, err := ioutil.TempDir("", "tree")
@@ -853,12 +838,11 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
 			Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred())
@@ -877,7 +861,7 @@ var _ = Describe("Compiler", func() {
 			err := generalRecipe.Load("../../tests/fixtures/includeimage")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
 			Expect(err).ToNot(HaveOccurred())
@@ -896,11 +880,11 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
-			compiler.SetCompressionType(GZip)
+			compiler.Options.CompressionType = compression.GZip
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
 
 			tmpdir, err := ioutil.TempDir("", "tree")
@@ -908,20 +892,19 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
-			Expect(artifacts[0].GetFiles()).To(ContainElement("bin/busybox"))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
+			Expect(artifacts[0].Files).To(ContainElement("bin/busybox"))
 
 			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
 
 			art, err := LoadArtifactFromYaml(spec)
 			Expect(err).ToNot(HaveOccurred())
 
-			files := art.GetFiles()
+			files := art.Files
 			Expect(files).To(ContainElement("bin/busybox"))
 		})
 	})

pkg/compiler/interface.go

@@ -1,201 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler
-
-import (
-	"runtime"
-
-	"github.com/mudler/luet/pkg/config"
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
-)
-
-type Compiler interface {
-	Compile(bool, CompilationSpec) (Artifact, error)
-	CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
-	ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error)
-	SetConcurrency(i int)
-	FromPackage(pkg.Package) (CompilationSpec, error)
-	FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error)
-	SetBackend(CompilerBackend)
-	GetBackend() CompilerBackend
-
-	SetBackendArgs([]string)
-	SetCompressionType(t CompressionImplementation)
-}
-
-type CompilerBackendOptions struct {
-	ImageName      string
-	SourcePath     string
-	DockerFileName string
-	Destination    string
-	Context        string
-	BackendArgs    []string
-}
-
-type CompilerOptions struct {
-	ImageRepository          string
-	PullFirst, KeepImg, Push bool
-	Concurrency              int
-	CompressionType          CompressionImplementation
-
-	Wait            bool
-	OnlyDeps        bool
-	NoDeps          bool
-	SolverOptions   config.LuetSolverOptions
-	BuildValuesFile string
-
-	PackageTargetOnly bool
-}
-
-func NewDefaultCompilerOptions() *CompilerOptions {
-	return &CompilerOptions{
-		ImageRepository: "luet/cache",
-		PullFirst:       false,
-		Push:            false,
-		CompressionType: None,
-		KeepImg:         true,
-		Concurrency:     runtime.NumCPU(),
-		OnlyDeps:        false,
-		NoDeps:          false,
-	}
-}
-
-type CompilerBackend interface {
-	BuildImage(CompilerBackendOptions) error
-	ExportImage(CompilerBackendOptions) error
-	RemoveImage(CompilerBackendOptions) error
-	Changes(fromImage, toImage CompilerBackendOptions) ([]ArtifactLayer, error)
-	ImageDefinitionToTar(CompilerBackendOptions) error
-	ExtractRootfs(opts CompilerBackendOptions, keepPerms bool) error
-
-	CopyImage(string, string) error
-	DownloadImage(opts CompilerBackendOptions) error
-
-	Push(opts CompilerBackendOptions) error
-	ImageAvailable(string) bool
-
-	ImageExists(string) bool
-}
-
-type Artifact interface {
-	GetPath() string
-	SetPath(string)
-	GetDependencies() []Artifact
-	SetDependencies(d []Artifact)
-	GetSourceAssertion() solver.PackagesAssertions
-	SetSourceAssertion(as solver.PackagesAssertions)
-
-	SetCompileSpec(as CompilationSpec)
-	GetCompileSpec() CompilationSpec
-	WriteYaml(dst string) error
-	Unpack(dst string, keepPerms bool) error
-	Compress(src string, concurrency int) error
-	SetCompressionType(t CompressionImplementation)
-	FileList() ([]string, error)
-	Hash() error
-	Verify() error
-
-	SetFiles(f []string)
-	GetFiles() []string
-	GetFileName() string
-
-	GetChecksums() Checksums
-	SetChecksums(c Checksums)
-
-	GenerateFinalImage(string, CompilerBackend, bool) (CompilerBackendOptions, error)
-	GetUncompressedName() string
-}
-
-type ArtifactNode struct {
-	Name string `json:"Name"`
-	Size int    `json:"Size"`
-}
-type ArtifactDiffs struct {
-	Additions []ArtifactNode `json:"Adds"`
-	Deletions []ArtifactNode `json:"Dels"`
-	Changes   []ArtifactNode `json:"Mods"`
-}
-type ArtifactLayer struct {
-	FromImage string        `json:"Image1"`
-	ToImage   string        `json:"Image2"`
-	Diffs     ArtifactDiffs `json:"Diff"`
-}
-type ArtifactLayerSummary struct {
-	FromImage   string `json:"image1"`
-	ToImage     string `json:"image2"`
-	AddFiles    int    `json:"add_files"`
-	AddSizes    int64  `json:"add_sizes"`
-	DelFiles    int    `json:"del_files"`
-	DelSizes    int64  `json:"del_sizes"`
-	ChangeFiles int    `json:"change_files"`
-	ChangeSizes int64  `json:"change_sizes"`
-}
-type ArtifactLayersSummary struct {
-	Layers []ArtifactLayerSummary `json:"summary"`
-}
-
-// CompilationSpec represent a compilation specification derived from a package
-type CompilationSpec interface {
-	ImageUnpack() bool // tells if the definition is just an image
-	GetIncludes() []string
-	GetExcludes() []string
-
-	RenderBuildImage() (string, error)
-	WriteBuildImageDefinition(string) error
-
-	RenderStepImage(image string) (string, error)
-	WriteStepImageDefinition(fromimage, path string) error
-
-	GetPackage() pkg.Package
-	BuildSteps() []string
-
-	GetSeedImage() string
-	SetSeedImage(string)
-
-	GetImage() string
-	SetImage(string)
-
-	SetOutputPath(string)
-	GetOutputPath() string
-	Rel(string) string
-
-	GetPreBuildSteps() []string
-
-	GetSourceAssertion() solver.PackagesAssertions
-	SetSourceAssertion(as solver.PackagesAssertions)
-
-	GetRetrieve() []string
-	CopyRetrieves(dest string) error
-
-	SetPackageDir(string)
-	GetPackageDir() string
-
-	EmptyPackage() bool
-	UnpackedPackage() bool
-	HasImageSource() bool
-	IsVirtual() bool
-}
-
-type CompilationSpecs interface {
-	Unique() CompilationSpecs
-	Len() int
-	All() []CompilationSpec
-	Add(CompilationSpec)
-	Remove(s CompilationSpecs) CompilationSpecs
-}

pkg/compiler/types/artifact/artifact.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler
+package artifact
 
 import (
 	"archive/tar"
@@ -27,7 +27,6 @@ import (
 	"path/filepath"
 	"regexp"
 
-	system "github.com/docker/docker/pkg/system"
 	zstd "github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 
@@ -36,6 +35,9 @@ import (
 	"sync"
 
 	bus "github.com/mudler/luet/pkg/bus"
+	backend "github.com/mudler/luet/pkg/compiler/backend"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -45,54 +47,30 @@ import (
 	yaml "gopkg.in/yaml.v2"
 )
 
-type CompressionImplementation string
-
-const (
-	None      CompressionImplementation = "none" // e.g. tar for standard packages
-	GZip      CompressionImplementation = "gzip"
-	Zstandard CompressionImplementation = "zstd"
-)
-
-type ArtifactIndex []Artifact
-
-func (i ArtifactIndex) CleanPath() ArtifactIndex {
-	newIndex := ArtifactIndex{}
-	for _, n := range i {
-		art := n.(*PackageArtifact)
-		// FIXME: This is a dup and makes difficult to add attributes to artifacts
-		newIndex = append(newIndex, &PackageArtifact{
-			Path:            path.Base(n.GetPath()),
-			SourceAssertion: art.SourceAssertion,
-			CompileSpec:     art.CompileSpec,
-			Dependencies:    art.Dependencies,
-			CompressionType: art.CompressionType,
-			Checksums:       art.Checksums,
-			Files:           art.Files,
-		})
-	}
-	return newIndex
-	//Update if exists, otherwise just create
-}
-
 //  When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository
 // which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact.
 // In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages.
 type PackageArtifact struct {
 	Path string `json:"path"`
 
-	Dependencies    []*PackageArtifact        `json:"dependencies"`
-	CompileSpec     *LuetCompilationSpec      `json:"compilationspec"`
-	Checksums       Checksums                 `json:"checksums"`
-	SourceAssertion solver.PackagesAssertions `json:"-"`
-	CompressionType CompressionImplementation `json:"compressiontype"`
-	Files           []string                  `json:"files"`
+	Dependencies    []*PackageArtifact                `json:"dependencies"`
+	CompileSpec     *compilerspec.LuetCompilationSpec `json:"compilationspec"`
+	Checksums       Checksums                         `json:"checksums"`
+	SourceAssertion solver.PackagesAssertions         `json:"-"`
+	CompressionType compression.Implementation        `json:"compressiontype"`
+	Files           []string                          `json:"files"`
 }
 
-func NewPackageArtifact(path string) Artifact {
-	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: None}
+func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
+	copy := *p
+	return &copy
 }
 
-func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
+func NewPackageArtifact(path string) *PackageArtifact {
+	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: compression.None}
+}
+
+func NewPackageArtifactFromYaml(data []byte) (*PackageArtifact, error) {
 	p := &PackageArtifact{Checksums: Checksums{}}
 	err := yaml.Unmarshal(data, &p)
 	if err != nil {
@@ -102,42 +80,6 @@ func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
 	return p, err
 }
 
-func LoadArtifactFromYaml(spec CompilationSpec) (Artifact, error) {
-
-	metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml"
-	dat, err := ioutil.ReadFile(spec.Rel(metaFile))
-	if err != nil {
-		return nil, errors.Wrap(err, "Error reading file "+metaFile)
-	}
-	art, err := NewPackageArtifactFromYaml(dat)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error writing file "+metaFile)
-	}
-	// It is relative, set it back to abs
-	art.SetPath(spec.Rel(art.GetPath()))
-	return art, nil
-}
-
-func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) {
-	a.CompressionType = t
-}
-
-func (a *PackageArtifact) GetChecksums() Checksums {
-	return a.Checksums
-}
-
-func (a *PackageArtifact) SetChecksums(c Checksums) {
-	a.Checksums = c
-}
-
-func (a *PackageArtifact) SetFiles(f []string) {
-	a.Files = f
-}
-
-func (a *PackageArtifact) GetFiles() []string {
-	return a.Files
-}
-
 func (a *PackageArtifact) Hash() error {
 	return a.Checksums.Generate(a)
 }
@@ -181,8 +123,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 	}
 	//p := a.CompileSpec.GetPackage().GetPath()
 
-	mangle.GetCompileSpec().GetPackage().SetPath("")
-	for _, ass := range mangle.GetCompileSpec().GetSourceAssertion() {
+	mangle.CompileSpec.GetPackage().SetPath("")
+	for _, ass := range mangle.CompileSpec.GetSourceAssertion() {
 		ass.Package.SetPath("")
 	}
 
@@ -191,7 +133,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
 	}
 
-	err = ioutil.WriteFile(filepath.Join(dst, a.GetCompileSpec().GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm)
+	err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetMetadataFilePath()), data, os.ModePerm)
 	if err != nil {
 		return errors.Wrap(err, "While writing PackageArtifact YAML")
 	}
@@ -201,48 +143,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 	return nil
 }
 
-func (a *PackageArtifact) GetSourceAssertion() solver.PackagesAssertions {
-	return a.SourceAssertion
-}
-
-func (a *PackageArtifact) SetCompileSpec(as CompilationSpec) {
-	a.CompileSpec = as.(*LuetCompilationSpec)
-}
-
-func (a *PackageArtifact) GetCompileSpec() CompilationSpec {
-	return a.CompileSpec
-}
-
-func (a *PackageArtifact) SetSourceAssertion(as solver.PackagesAssertions) {
-	a.SourceAssertion = as
-}
-
-func (a *PackageArtifact) GetDependencies() []Artifact {
-	ret := []Artifact{}
-	for _, d := range a.Dependencies {
-		ret = append(ret, d)
-	}
-	return ret
-}
-
-func (a *PackageArtifact) SetDependencies(d []Artifact) {
-	ret := []*PackageArtifact{}
-	for _, dd := range d {
-		ret = append(ret, dd.(*PackageArtifact))
-	}
-	a.Dependencies = ret
-}
-
-func (a *PackageArtifact) GetPath() string {
-	return a.Path
-}
-
 func (a *PackageArtifact) GetFileName() string {
-	return path.Base(a.GetPath())
-}
-
-func (a *PackageArtifact) SetPath(p string) {
-	a.Path = p
+	return path.Base(a.Path)
 }
 
 func (a *PackageArtifact) genDockerfile() string {
@@ -253,14 +155,20 @@ COPY . /`
 
 // CreateArtifactForFile creates a new artifact from the given file
 func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
-
+	if _, err := os.Stat(s); os.IsNotExist(err) {
+		return nil, errors.Wrap(err, "artifact path doesn't exist")
+	}
 	fileName := path.Base(s)
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
 		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
 	}
 	defer os.RemoveAll(archive) // clean up
-	helpers.CopyFile(s, filepath.Join(archive, fileName))
+	dst := filepath.Join(archive, fileName)
+	if err := helpers.CopyFile(s, dst); err != nil {
+		return nil, errors.Wrapf(err, "error while copying %s to %s", s, dst)
+	}
+
 	artifact, err := LuetCfg.GetSystem().TempDir("artifact")
 	if err != nil {
 		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
@@ -274,9 +182,13 @@ func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageAr
 	return a, a.Compress(archive, 1)
 }
 
+type ImageBuilder interface {
+	BuildImage(backend.Options) error
+}
+
 // GenerateFinalImage takes an artifact and builds a Docker image with its content
-func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend, keepPerms bool) (CompilerBackendOptions, error) {
-	builderOpts := CompilerBackendOptions{}
+func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) {
+	builderOpts := backend.Options{}
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
 		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
@@ -311,7 +223,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend
 		return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path)
 	}
 
-	builderOpts = CompilerBackendOptions{
+	builderOpts = backend.Options{
 		ImageName:      imageName,
 		SourcePath:     archive,
 		DockerFileName: dockerFile,
@@ -326,7 +238,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend
 func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	switch a.CompressionType {
 
-	case Zstandard:
+	case compression.Zstandard:
 		err := helpers.Tar(src, a.Path)
 		if err != nil {
 			return err
@@ -364,7 +276,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 
 		a.Path = zstdFile
 		return nil
-	case GZip:
+	case compression.GZip:
 		err := helpers.Tar(src, a.Path)
 		if err != nil {
 			return err
@@ -409,10 +321,10 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 
 func (a *PackageArtifact) getCompressedName() string {
 	switch a.CompressionType {
-	case Zstandard:
+	case compression.Zstandard:
 		return a.Path + ".zst"
 
-	case GZip:
+	case compression.GZip:
 		return a.Path + ".gz"
 	}
 	return a.Path
@@ -421,7 +333,7 @@ func (a *PackageArtifact) getCompressedName() string {
 // GetUncompressedName returns the artifact path without the extension suffix
 func (a *PackageArtifact) GetUncompressedName() string {
 	switch a.CompressionType {
-	case Zstandard, GZip:
+	case compression.Zstandard, compression.GZip:
 		return strings.TrimSuffix(a.Path, filepath.Ext(a.Path))
 	}
 	return a.Path
@@ -509,13 +421,13 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
 
 	switch a.CompressionType {
-	case Zstandard:
+	case compression.Zstandard:
 		// Create the uncompressed archive
-		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		archive, err := os.Create(a.Path + ".uncompressed")
 		if err != nil {
 			return err
 		}
-		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer os.RemoveAll(a.Path + ".uncompressed")
 		defer archive.Close()
 
 		original, err := os.Open(a.Path)
@@ -534,22 +446,22 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 
 		_, err = io.Copy(archive, d)
 		if err != nil {
-			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
+			return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
 		}
 
-		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
 			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
 		}
 		return nil
-	case GZip:
+	case compression.GZip:
 		// Create the uncompressed archive
-		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		archive, err := os.Create(a.Path + ".uncompressed")
 		if err != nil {
 			return err
 		}
-		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer os.RemoveAll(a.Path + ".uncompressed")
 		defer archive.Close()
 
 		original, err := os.Open(a.Path)
@@ -567,10 +479,10 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 
 		_, err = io.Copy(archive, r)
 		if err != nil {
-			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
+			return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
 		}
 
-		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
 			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
@@ -578,7 +490,7 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 		return nil
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.UntarProtect(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner,
+		return helpers.UntarProtect(a.Path, dst, LuetCfg.GetGeneral().SameOwner,
 			protectedFiles, tarModifier)
 	}
 	return errors.New("Compression type must be supplied")
@@ -588,12 +500,12 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 func (a *PackageArtifact) FileList() ([]string, error) {
 	var tr *tar.Reader
 	switch a.CompressionType {
-	case Zstandard:
-		archive, err := os.Create(a.GetPath() + ".uncompressed")
+	case compression.Zstandard:
+		archive, err := os.Create(a.Path + ".uncompressed")
 		if err != nil {
 			return []string{}, err
 		}
-		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer os.RemoveAll(a.Path + ".uncompressed")
 		defer archive.Close()
 
 		original, err := os.Open(a.Path)
@@ -609,13 +521,13 @@ func (a *PackageArtifact) FileList() ([]string, error) {
 		}
 		defer r.Close()
 		tr = tar.NewReader(r)
-	case GZip:
+	case compression.GZip:
 		// Create the uncompressed archive
-		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		archive, err := os.Create(a.Path + ".uncompressed")
 		if err != nil {
 			return []string{}, err
 		}
-		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer os.RemoveAll(a.Path + ".uncompressed")
 		defer archive.Close()
 
 		original, err := os.Open(a.Path)
@@ -634,7 +546,7 @@ func (a *PackageArtifact) FileList() ([]string, error) {
 
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		tarFile, err := os.Open(a.GetPath())
+		tarFile, err := os.Open(a.Path)
 		if err != nil {
 			return []string{}, errors.Wrap(err, "Could not open package archive")
 		}
@@ -671,47 +583,16 @@ type CopyJob struct {
 	Artifact string
 }
 
-func copyXattr(srcPath, dstPath, attr string) error {
-	data, err := system.Lgetxattr(srcPath, attr)
-	if err != nil {
-		return err
-	}
-	if data != nil {
-		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func doCopyXattrs(srcPath, dstPath string) error {
-	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
-		return err
-	}
-
-	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
-}
-
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
 	for job := range s {
-		//Info("#"+strconv.Itoa(i), "copying", job.Src, "to", job.Dst)
-		// if dir, err := helpers.IsDirectory(job.Src); err == nil && dir {
-		// 	err = helpers.CopyDir(job.Src, job.Dst)
-		// 	if err != nil {
-		// 		Warning("Error copying dir", job, err)
-		// 	}
-		// 	continue
-		// }
-
 		_, err := os.Lstat(job.Dst)
 		if err != nil {
 			Debug("Copying ", job.Src)
-			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
+			if err := helpers.DeepCopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
-			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
@@ -729,8 +610,24 @@ func compileRegexes(regexes []string) []*regexp.Regexp {
 	return result
 }
 
+type ArtifactNode struct {
+	Name string `json:"Name"`
+	Size int    `json:"Size"`
+}
+type ArtifactDiffs struct {
+	Additions []ArtifactNode `json:"Adds"`
+	Deletions []ArtifactNode `json:"Dels"`
+	Changes   []ArtifactNode `json:"Mods"`
+}
+
+type ArtifactLayer struct {
+	FromImage string        `json:"Image1"`
+	ToImage   string        `json:"Image2"`
+	Diffs     ArtifactDiffs `json:"Diff"`
+}
+
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t CompressionImplementation) (Artifact, error) {
+func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) {
 
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
@@ -852,45 +749,10 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 	wg.Wait()
 
 	a := NewPackageArtifact(dst)
-	a.SetCompressionType(t)
+	a.CompressionType = t
 	err = a.Compress(archive, concurrency)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}
 	return a, nil
 }
-
-func ComputeArtifactLayerSummary(diffs []ArtifactLayer) ArtifactLayersSummary {
-
-	ans := ArtifactLayersSummary{
-		Layers: make([]ArtifactLayerSummary, 0),
-	}
-
-	for _, layer := range diffs {
-		sum := ArtifactLayerSummary{
-			FromImage:   layer.FromImage,
-			ToImage:     layer.ToImage,
-			AddFiles:    0,
-			AddSizes:    0,
-			DelFiles:    0,
-			DelSizes:    0,
-			ChangeFiles: 0,
-			ChangeSizes: 0,
-		}
-		for _, a := range layer.Diffs.Additions {
-			sum.AddFiles++
-			sum.AddSizes += int64(a.Size)
-		}
-		for _, d := range layer.Diffs.Deletions {
-			sum.DelFiles++
-			sum.DelSizes += int64(d.Size)
-		}
-		for _, c := range layer.Diffs.Changes {
-			sum.ChangeFiles++
-			sum.ChangeSizes += int64(c.Size)
-		}
-		ans.Layers = append(ans.Layers, sum)
-	}
-
-	return ans
-}

pkg/compiler/types/artifact/artifact_test.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler_test
+package artifact_test
 
 import (
 	"io/ioutil"
@@ -22,7 +22,10 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
-	"github.com/mudler/luet/pkg/solver"
+	backend "github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/compiler/types/artifact"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
@@ -43,13 +46,10 @@ var _ = Describe("Artifact", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
 			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 			Expect(lspec.Image).To(Equal("luet/base"))
 			Expect(lspec.Seed).To(Equal("alpine"))
@@ -81,7 +81,7 @@ ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin`))
 			b := NewSimpleDockerBackend()
-			opts := CompilerBackendOptions{
+			opts := backend.Options{
 				ImageName:      "luet/base",
 				SourcePath:     tmpdir,
 				DockerFileName: "Dockerfile",
@@ -105,7 +105,7 @@ ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
-			opts2 := CompilerBackendOptions{
+			opts2 := backend.Options{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
@@ -114,7 +114,7 @@ RUN echo bar > /test2`))
 			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
-			diffs, err := b.Changes(opts, opts2)
+			diffs, err := compiler.GenerateChanges(b, opts, opts2)
 			Expect(err).ToNot(HaveOccurred())
 
 			artifacts := []ArtifactNode{{
@@ -135,13 +135,13 @@ RUN echo bar > /test2`))
 						Additions: artifacts,
 					},
 				}}))
-			err = b.ExtractRootfs(CompilerBackendOptions{ImageName: "test", Destination: rootfs}, false)
+			err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, None)
+			a, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
-			err = helpers.Untar(artifact.GetPath(), unpacked, false)
+			err = helpers.Untar(a.Path, unpacked, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
 			Expect(helpers.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
@@ -152,13 +152,13 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content2).To(Equal("bar\n"))
 
-			err = artifact.Hash()
+			err = a.Hash()
 			Expect(err).ToNot(HaveOccurred())
-			err = artifact.Verify()
+			err = a.Verify()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
 
-			err = artifact.Verify()
+			err = a.Verify()
 			Expect(err).To(HaveOccurred())
 		})
 
@@ -183,13 +183,13 @@ RUN echo bar > /test2`))
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "foo", "bar", "test"), testString, 0644)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
-			artifact.SetCompileSpec(&LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}})
+			a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}
 
-			err = artifact.Compress(tmpdir, 1)
+			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := artifact.GenerateFinalImage(resultingImage, b, false)
+			opts, err := a.GenerateFinalImage(resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(opts.ImageName).To(Equal(resultingImage))
 
@@ -199,7 +199,7 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(result) // clean up
 
-			err = b.ExtractRootfs(CompilerBackendOptions{ImageName: resultingImage, Destination: result}, false)
+			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			content, err := ioutil.ReadFile(filepath.Join(result, "test"))
@@ -225,13 +225,13 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpWork) // clean up
 
-			artifact := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
-			artifact.SetCompileSpec(&LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}})
+			a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}
 
-			err = artifact.Compress(tmpdir, 1)
+			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := artifact.GenerateFinalImage(resultingImage, b, false)
+			opts, err := a.GenerateFinalImage(resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(opts.ImageName).To(Equal(resultingImage))
 
@@ -241,7 +241,7 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(result) // clean up
 
-			err = b.ExtractRootfs(CompilerBackendOptions{ImageName: resultingImage, Destination: result}, false)
+			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.DirectoryIsEmpty(result)).To(BeFalse())
@@ -253,15 +253,15 @@ RUN echo bar > /test2`))
 
 		It("Retrieves uncompressed name", func() {
 			a := NewPackageArtifact("foo.tar.gz")
-			a.SetCompressionType(compiler.GZip)
+			a.CompressionType = (compression.GZip)
 			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
 
 			a = NewPackageArtifact("foo.tar.zst")
-			a.SetCompressionType(compiler.Zstandard)
+			a.CompressionType = compression.Zstandard
 			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
 
 			a = NewPackageArtifact("foo.tar")
-			a.SetCompressionType(compiler.None)
+			a.CompressionType = compression.None
 			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
 		})
 	})

pkg/compiler/types/artifact/checksum.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler
+package artifact
 
 import (
 
@@ -43,7 +43,7 @@ type HashOptions struct {
 }
 
 // Generate generates all Checksums supported for the artifact
-func (c *Checksums) Generate(a Artifact) error {
+func (c *Checksums) Generate(a *PackageArtifact) error {
 	return c.generateSHA256(a)
 }
 
@@ -56,13 +56,13 @@ func (c Checksums) Compare(d Checksums) error {
 	return nil
 }
 
-func (c *Checksums) generateSHA256(a Artifact) error {
+func (c *Checksums) generateSHA256(a *PackageArtifact) error {
 	return c.generateSum(a, HashOptions{Hasher: sha256.New(), Type: SHA256})
 }
 
-func (c *Checksums) generateSum(a Artifact, opts HashOptions) error {
+func (c *Checksums) generateSum(a *PackageArtifact, opts HashOptions) error {
 
-	f, err := os.Open(a.GetPath())
+	f, err := os.Open(a.Path)
 	if err != nil {
 		return err
 	}

pkg/compiler/types/artifact/checksum_test.go

@@ -13,13 +13,13 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler_test
+package artifact_test
 
 import (
 	"io/ioutil"
 	"os"
 
-	. "github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/types/artifact"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

pkg/compiler/types/compression/compression.go

@@ -0,0 +1,9 @@
+package compression
+
+type Implementation string
+
+const (
+	None      Implementation = "none" // e.g. tar for standard packages
+	GZip      Implementation = "gzip"
+	Zstandard Implementation = "zstd"
+)

pkg/compiler/types/options/compiler_options.go

@@ -0,0 +1,199 @@
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package options
+
+import (
+	"runtime"
+
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/solver"
+)
+
+type Compiler struct {
+	PushImageRepository      string
+	PullImageRepository      []string
+	PullFirst, KeepImg, Push bool
+	Concurrency              int
+	CompressionType          compression.Implementation
+
+	Wait            bool
+	OnlyDeps        bool
+	NoDeps          bool
+	SolverOptions   config.LuetSolverOptions
+	BuildValuesFile []string
+	BuildValues     []map[string]interface{}
+
+	PackageTargetOnly bool
+	Rebuild           bool
+
+	BackendArgs []string
+
+	BackendType string
+}
+
+func NewDefaultCompiler() *Compiler {
+	return &Compiler{
+		PushImageRepository: "luet/cache",
+		PullFirst:           false,
+		Push:                false,
+		CompressionType:     compression.None,
+		KeepImg:             true,
+		Concurrency:         runtime.NumCPU(),
+		OnlyDeps:            false,
+		NoDeps:              false,
+		SolverOptions:       config.LuetSolverOptions{Options: solver.Options{Concurrency: 1, Type: solver.SingleCoreSimple}},
+	}
+}
+
+type Option func(cfg *Compiler) error
+
+func (cfg *Compiler) Apply(opts ...Option) error {
+	for _, opt := range opts {
+		if opt == nil {
+			continue
+		}
+		if err := opt(cfg); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func WithOptions(opt *Compiler) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg = opt
+		return nil
+	}
+}
+
+func WithBackendType(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BackendType = r
+		return nil
+	}
+}
+
+func WithBuildValues(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BuildValuesFile = r
+		return nil
+	}
+}
+
+func WithPullRepositories(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PullImageRepository = r
+		return nil
+	}
+}
+
+func WithPushRepository(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		if len(cfg.PullImageRepository) == 0 {
+			cfg.PullImageRepository = []string{cfg.PushImageRepository}
+		}
+		cfg.PushImageRepository = r
+		return nil
+	}
+}
+
+func BackendArgs(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BackendArgs = r
+		return nil
+	}
+}
+
+func PullFirst(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PullFirst = b
+		return nil
+	}
+}
+
+func KeepImg(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.KeepImg = b
+		return nil
+	}
+}
+
+func Rebuild(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Rebuild = b
+		return nil
+	}
+}
+
+func PushImages(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Push = b
+		return nil
+	}
+}
+
+func Wait(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Wait = b
+		return nil
+	}
+}
+
+func OnlyDeps(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.OnlyDeps = b
+		return nil
+	}
+}
+
+func OnlyTarget(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PackageTargetOnly = b
+		return nil
+	}
+}
+
+func NoDeps(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.NoDeps = b
+		return nil
+	}
+}
+
+func Concurrency(i int) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		if i == 0 {
+			i = runtime.NumCPU()
+		}
+		cfg.Concurrency = i
+		return nil
+	}
+}
+
+func WithCompressionType(t compression.Implementation) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.CompressionType = t
+		return nil
+	}
+}
+
+func WithSolverOptions(c config.LuetSolverOptions) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.SolverOptions = c
+		return nil
+	}
+}

pkg/compiler/types/spec/spec.go

@@ -13,12 +13,14 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler
+package compilerspec
 
 import (
 	"io/ioutil"
 	"path/filepath"
 
+	options "github.com/mudler/luet/pkg/compiler/types/options"
+
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/otiai10/copy"
@@ -27,7 +29,7 @@ import (
 
 type LuetCompilationspecs []LuetCompilationSpec
 
-func NewLuetCompilationspecs(s ...CompilationSpec) CompilationSpecs {
+func NewLuetCompilationspecs(s ...*LuetCompilationSpec) *LuetCompilationspecs {
 	all := LuetCompilationspecs{}
 
 	for _, spec := range s {
@@ -40,7 +42,7 @@ func (specs LuetCompilationspecs) Len() int {
 	return len(specs)
 }
 
-func (specs *LuetCompilationspecs) Remove(s CompilationSpecs) CompilationSpecs {
+func (specs *LuetCompilationspecs) Remove(s *LuetCompilationspecs) *LuetCompilationspecs {
 	newSpecs := LuetCompilationspecs{}
 SPECS:
 	for _, spec := range specs.All() {
@@ -54,16 +56,12 @@ SPECS:
 	return &newSpecs
 }
 
-func (specs *LuetCompilationspecs) Add(s CompilationSpec) {
-	c, ok := s.(*LuetCompilationSpec)
-	if !ok {
-		panic("LuetCompilationspecs supports only []LuetCompilationSpec")
-	}
-	*specs = append(*specs, *c)
+func (specs *LuetCompilationspecs) Add(s *LuetCompilationSpec) {
+	*specs = append(*specs, *s)
 }
 
-func (specs *LuetCompilationspecs) All() []CompilationSpec {
-	var cspecs []CompilationSpec
+func (specs *LuetCompilationspecs) All() []*LuetCompilationSpec {
+	var cspecs []*LuetCompilationSpec
 	for i, _ := range *specs {
 		f := (*specs)[i]
 		cspecs = append(cspecs, &f)
@@ -72,7 +70,7 @@ func (specs *LuetCompilationspecs) All() []CompilationSpec {
 	return cspecs
 }
 
-func (specs *LuetCompilationspecs) Unique() CompilationSpecs {
+func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
 	newSpecs := LuetCompilationspecs{}
 	seen := map[string]bool{}
 
@@ -103,9 +101,11 @@ type LuetCompilationSpec struct {
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
 	Excludes   []string `json:"excludes"`
+
+	BuildOptions *options.Compiler `json:"build_options"`
 }
 
-func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
+func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
 	var spec LuetCompilationSpec
 	err := yaml.Unmarshal(b, &spec)
 	if err != nil {
@@ -118,6 +118,10 @@ func (cs *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
 	return cs.SourceAssertion
 }
 
+func (cs *LuetCompilationSpec) SetBuildOptions(b options.Compiler) {
+	cs.BuildOptions = &b
+}
+
 func (cs *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) {
 	cs.SourceAssertion = as
 }

pkg/compiler/types/spec/spec_test.go

@@ -13,17 +13,18 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler_test
+package compilerspec_test
 
 import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -32,41 +33,41 @@ import (
 var _ = Describe("Spec", func() {
 	Context("Luet specs", func() {
 		It("Allows normal operations", func() {
-			testSpec := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
-			testSpec2 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}}
-			testSpec3 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}}
-			testSpec4 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
+			testSpec := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
+			testSpec2 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}}
+			testSpec3 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}}
+			testSpec4 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
 
-			specs := NewLuetCompilationspecs(testSpec, testSpec2)
+			specs := compilerspec.NewLuetCompilationspecs(testSpec, testSpec2)
 			Expect(specs.Len()).To(Equal(2))
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2}))
+			Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2}))
 			specs.Add(testSpec3)
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
+			Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
 			specs.Add(testSpec4)
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
+			Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
 			newSpec := specs.Unique()
-			Expect(newSpec.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
+			Expect(newSpec.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
 
-			newSpec2 := specs.Remove(NewLuetCompilationspecs(testSpec, testSpec2))
-			Expect(newSpec2.All()).To(Equal([]CompilationSpec{testSpec3}))
+			newSpec2 := specs.Remove(compilerspec.NewLuetCompilationspecs(testSpec, testSpec2))
+			Expect(newSpec2.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec3}))
 
 		})
 		Context("virtuals", func() {
 			When("is empty", func() {
 				It("is virtual", func() {
-					spec := &LuetCompilationSpec{}
+					spec := &compilerspec.LuetCompilationSpec{}
 					Expect(spec.IsVirtual()).To(BeTrue())
 				})
 			})
 			When("has defined steps", func() {
 				It("is not a virtual", func() {
-					spec := &LuetCompilationSpec{Steps: []string{"foo"}}
+					spec := &compilerspec.LuetCompilationSpec{Steps: []string{"foo"}}
 					Expect(spec.IsVirtual()).To(BeFalse())
 				})
 			})
 			When("has defined image", func() {
 				It("is not a virtual", func() {
-					spec := &LuetCompilationSpec{Image: "foo"}
+					spec := &compilerspec.LuetCompilationSpec{Image: "foo"}
 					Expect(spec.IsVirtual()).To(BeFalse())
 				})
 			})
@@ -82,13 +83,10 @@ var _ = Describe("Spec", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
 			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 			Expect(lspec.Image).To(Equal("luet/base"))
 			Expect(lspec.Seed).To(Equal("alpine"))
@@ -137,13 +135,10 @@ RUN echo bar > /test2`))
 
 		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-		spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+		lspec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
 		Expect(err).ToNot(HaveOccurred())
 
-		lspec, ok := spec.(*LuetCompilationSpec)
-		Expect(ok).To(BeTrue())
-
 		Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 		Expect(lspec.Image).To(Equal("luet/base"))
 		Expect(lspec.Seed).To(Equal("alpine"))

pkg/config/config.go

@@ -65,6 +65,7 @@ type LuetGeneralConfig struct {
 }
 
 type LuetSolverOptions struct {
+	solver.Options
 	Type           string            `mapstructure:"type"`
 	LearnRate      float32           `mapstructure:"rate"`
 	Discount       float32           `mapstructure:"discount"`
@@ -159,8 +160,8 @@ type LuetRepository struct {
 	Enable         bool              `json:"enable" yaml:"enable" mapstructure:"enable"`
 	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
 	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
-	TreePath       string            `json:"tree_path,omitempty" yaml:"tree_path,omitempty" mapstructure:"tree_path"`
-	MetaPath       string            `json:"meta_path,omitempty" yaml:"meta_path,omitempty" mapstructure:"meta_path"`
+	TreePath       string            `json:"treepath,omitempty" yaml:"treepath,omitempty" mapstructure:"treepath"`
+	MetaPath       string            `json:"metapath,omitempty" yaml:"metapath,omitempty" mapstructure:"metapath"`
 	Verify         bool              `json:"verify,omitempty" yaml:"verify,omitempty" mapstructure:"verify"`
 
 	// Serialized options not used in repository configuration

pkg/helpers/archive.go

@@ -95,9 +95,7 @@ func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modi
 		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
 
 		opts := &archive.TarOptions{
-			// NOTE: NoLchown boolean is used for chmod of the symlink
-			// Probably it's needed set this always to true.
-			NoLchown:        true,
+			NoLchown:        false,
 			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
 			ContinueOnError: true,
 		}
@@ -201,12 +199,8 @@ func Untar(src, dest string, sameOwner bool) error {
 	defer in.Close()
 
 	if sameOwner {
-		// PRE: i have root privileged.
-
 		opts := &archive.TarOptions{
-			// NOTE: NoLchown boolean is used for chmod of the symlink
-			// Probably it's needed set this always to true.
-			NoLchown:        true,
+			NoLchown:        false,
 			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
 			ContinueOnError: true,
 		}

pkg/helpers/docker.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"encoding/hex"
 	"os"
+	"strings"
 
 	"github.com/docker/cli/cli/trust"
 	"github.com/docker/distribution/reference"
@@ -122,3 +123,7 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
 	err = c.Unpack(image, dest)
 	return listedImage, err
 }
+
+func StripInvalidStringsFromImage(s string) string {
+	return strings.ReplaceAll(s, "+", "-")
+}

pkg/helpers/docker_test.go

@@ -0,0 +1,30 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("StripInvalidStringsFromImage", func() {
+	Context("Image names", func() {
+		It("strips invalid chars", func() {
+			Expect(StripInvalidStringsFromImage("foo+bar")).To(Equal("foo-bar"))
+		})
+	})
+})

pkg/helpers/file.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"sort"
@@ -26,10 +27,42 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/docker/docker/pkg/system"
+	"github.com/google/renameio"
 	copy "github.com/otiai10/copy"
 	"github.com/pkg/errors"
 )
 
+var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+func RandStringRunes(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letterRunes[rand.Intn(len(letterRunes))]
+	}
+	return string(b)
+}
+
+func Move(src, dst string) error {
+	f, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	t, err := renameio.TempFile("", dst)
+	if err != nil {
+		return err
+	}
+	defer t.Cleanup()
+
+	_, err = io.Copy(t, f)
+	if err != nil {
+		return err
+	}
+	return t.CloseAtomicallyReplace()
+}
+
 func OrderFiles(target string, files []string) ([]string, []string) {
 
 	var newFiles []string
@@ -135,9 +168,27 @@ func Read(file string) (string, error) {
 	return string(dat), nil
 }
 
+func EnsureDirPerm(src, dst string) {
+	if info, err := os.Lstat(filepath.Dir(src)); err == nil {
+		if _, err := os.Lstat(filepath.Dir(dst)); os.IsNotExist(err) {
+			err := os.MkdirAll(filepath.Dir(dst), info.Mode().Perm())
+			if err != nil {
+				fmt.Println("warning: failed creating", filepath.Dir(dst), err.Error())
+			}
+			if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+				if err := os.Lchown(filepath.Dir(dst), int(stat.Uid), int(stat.Gid)); err != nil {
+					fmt.Println("warning: failed chowning", filepath.Dir(dst), err.Error())
+				}
+			}
+		}
+	} else {
+		EnsureDir(dst)
+	}
+}
+
 func EnsureDir(fileName string) error {
 	dirName := filepath.Dir(fileName)
-	if _, serr := os.Stat(dirName); serr != nil {
+	if _, serr := os.Stat(dirName); os.IsNotExist(serr) {
 		merr := os.MkdirAll(dirName, os.ModePerm) // FIXME: It should preserve permissions from src to dst instead
 		if merr != nil {
 			return merr
@@ -146,12 +197,39 @@ func EnsureDir(fileName string) error {
 	return nil
 }
 
-// CopyFile copies the contents of the file named src to the file named
+func CopyFile(src, dst string) (err error) {
+	return copy.Copy(src, dst, copy.Options{
+		Sync:      true,
+		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
+}
+
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
+// DeepCopyFile copies the contents of the file named src to the file named
 // by dst. The file will be created if it does not already exist. If the
 // destination file exists, all it's contents will be replaced by the contents
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
-func CopyFile(src, dst string) (err error) {
+func DeepCopyFile(src, dst string) (err error) {
 	// Workaround for https://github.com/otiai10/copy/issues/47
 	fi, err := os.Lstat(src)
 	if err != nil {
@@ -161,7 +239,7 @@ func CopyFile(src, dst string) (err error) {
 	fm := fi.Mode()
 	switch {
 	case fm&os.ModeNamedPipe != 0:
-		EnsureDir(dst)
+		EnsureDirPerm(src, dst)
 		if err := syscall.Mkfifo(dst, uint32(fi.Mode())); err != nil {
 			return errors.Wrap(err, "failed creating pipe")
 		}
@@ -173,6 +251,9 @@ func CopyFile(src, dst string) (err error) {
 		return nil
 	}
 
+	//filepath.Dir(src)
+	EnsureDirPerm(src, dst)
+
 	err = copy.Copy(src, dst, copy.Options{
 		Sync:      true,
 		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
@@ -180,11 +261,12 @@ func CopyFile(src, dst string) (err error) {
 		return err
 	}
 	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
-		if err := os.Chown(dst, int(stat.Uid), int(stat.Gid)); err != nil {
-			fmt.Println("failed chowning", dst, err.Error())
+		if err := os.Lchown(dst, int(stat.Uid), int(stat.Gid)); err != nil {
+			fmt.Println("warning: failed chowning", dst, err.Error())
 		}
 	}
-	return err
+
+	return doCopyXattrs(src, dst)
 }
 
 func IsDirectory(path string) (bool, error) {

pkg/helpers/helm.go

@@ -3,6 +3,7 @@ package helpers
 import (
 	"io/ioutil"
 
+	"github.com/imdario/mergo"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 	"helm.sh/helm/v3/pkg/chart"
@@ -37,7 +38,38 @@ func RenderHelm(template string, values, d map[string]interface{}) (string, erro
 
 type templatedata map[string]interface{}
 
-func RenderFiles(toTemplate, valuesFile string, defaultFile string) (string, error) {
+// UnMarshalValues unmarshal values files and joins them into a unique templatedata
+// the join happens from right to left, so any rightmost value file overwrites the content of the ones before it.
+func UnMarshalValues(values []string) (templatedata, error) {
+	dst := templatedata{}
+	if len(values) > 0 {
+		for _, bv := range reverse(values) {
+			current := templatedata{}
+
+			defBuild, err := ioutil.ReadFile(bv)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+bv)
+			}
+			err = yaml.Unmarshal(defBuild, &current)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+bv)
+			}
+			if err := mergo.Merge(&dst, current); err != nil {
+				return nil, errors.Wrap(err, "merging values file "+bv)
+			}
+		}
+	}
+	return dst, nil
+}
+
+func reverse(s []string) []string {
+	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+		s[i], s[j] = s[j], s[i]
+	}
+	return s
+}
+
+func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string, error) {
 	raw, err := ioutil.ReadFile(toTemplate)
 	if err != nil {
 		return "", errors.Wrap(err, "reading file "+toTemplate)
@@ -52,20 +84,14 @@ func RenderFiles(toTemplate, valuesFile string, defaultFile string) (string, err
 	}
 
 	var values templatedata
-	d := templatedata{}
-	if len(defaultFile) > 0 {
-		def, err := ioutil.ReadFile(defaultFile)
-		if err != nil {
-			return "", errors.Wrap(err, "reading file "+valuesFile)
-		}
-		if err = yaml.Unmarshal(def, &d); err != nil {
-			return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
-		}
-	}
-
 	if err = yaml.Unmarshal(val, &values); err != nil {
 		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
 	}
 
-	return RenderHelm(string(raw), values, d)
+	dst, err := UnMarshalValues(defaultFile)
+	if err != nil {
+		return "", errors.Wrap(err, "unmarshalling values")
+	}
+
+	return RenderHelm(string(raw), values, dst)
 }

pkg/helpers/helm_test.go

@@ -114,11 +114,46 @@ foo: "bar"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, "")
+			res, err := RenderFiles(toTemplate, values)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bar"))
 		})
 
+		It("Render files merging defaults", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+			d2 := filepath.Join(testDir, "default2.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}{{.Values.bar}}{{.Values.b}}`)
+			writeFile(values, `
+foo: "bar"
+b: "f"
+`)
+			writeFile(d, `
+foo: "baz"
+`)
+
+			writeFile(d2, `
+foo: "do"
+bar: "nei"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d2, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bazneif"))
+
+			res, err = RenderFiles(toTemplate, values, d, d2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("doneif"))
+		})
+
 		It("doesn't interpolate if no one provides the values", func() {
 			testDir, err := ioutil.TempDir(os.TempDir(), "test")
 			Expect(err).ToNot(HaveOccurred())

pkg/helpers/imgworker/unpack.go

@@ -71,7 +71,7 @@ func (c *Client) Unpack(image, dest string) error {
 		// Unpack the tarfile to the rootfs path.
 		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
 		if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{
-			NoLchown:        true,
+			NoLchown:        false,
 			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
 		}); err != nil {
 			return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err)

pkg/helpers/match.go

@@ -17,9 +17,18 @@
 package helpers
 
 import (
+	"reflect"
 	"regexp"
 )
 
+func ReverseAny(s interface{}) {
+	n := reflect.ValueOf(s).Len()
+	swap := reflect.Swapper(s)
+	for i, j := 0, n-1; i < j; i, j = i+1, j-1 {
+		swap(i, j)
+	}
+}
+
 func MapMatchRegex(m *map[string]string, r *regexp.Regexp) bool {
 	ans := false
 

pkg/installer/client/docker.go

@@ -26,9 +26,10 @@ import (
 	"github.com/docker/go-units"
 	"github.com/pkg/errors"
 
-	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/helpers/imgworker"
 	. "github.com/mudler/luet/pkg/logger"
 )
 
@@ -51,7 +52,7 @@ func NewDockerClient(r RepoData) *DockerClient {
 	return &DockerClient{RepoData: r, auth: auth}
 }
 
-func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	//var u *url.URL = nil
 	var err error
 	var temp string
@@ -59,8 +60,8 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
 	Spinner(22)
 	defer SpinnerStop()
 
-	var resultingArtifact compiler.Artifact
-	artifactName := path.Base(artifact.GetPath())
+	var resultingArtifact *artifact.PackageArtifact
+	artifactName := path.Base(a.Path)
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 	Debug("Cache file", cacheFile)
 	if err := helpers.EnsureDir(cacheFile); err != nil {
@@ -78,9 +79,9 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
 	// Check if file is already in cache
 	if helpers.Exists(cacheFile) {
 		Debug("Cache hit for artifact", artifactName)
-		resultingArtifact = artifact
-		resultingArtifact.SetPath(cacheFile)
-		resultingArtifact.SetChecksums(compiler.Checksums{})
+		resultingArtifact = a
+		resultingArtifact.Path = cacheFile
+		resultingArtifact.Checksums = artifact.Checksums{}
 	} else {
 
 		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
@@ -91,7 +92,7 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
 
 		for _, uri := range c.RepoData.Urls {
 
-			imageName := fmt.Sprintf("%s:%s", uri, artifact.GetCompileSpec().GetPackage().ImageID())
+			imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID())
 			Info("Downloading image", imageName)
 
 			contentstore, err := config.LuetCfg.GetSystem().TempDir("contentstore")
@@ -111,11 +112,11 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
 			Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
 			Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile)
 
-			newart := artifact
+			newart := a
 			// We discard checksum, that are checked while during pull and unpack
-			newart.SetChecksums(compiler.Checksums{})
-			newart.SetPath(cacheFile)                    // First set to cache file
-			newart.SetPath(newart.GetUncompressedName()) // Calculate the real path from cacheFile
+			newart.Checksums = artifact.Checksums{}
+			newart.Path = cacheFile                    // First set to cache file
+			newart.Path = newart.GetUncompressedName() // Calculate the real path from cacheFile
 			err = newart.Compress(temp, 1)
 			if err != nil {
 				Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))
@@ -138,8 +139,8 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
 func (c *DockerClient) DownloadFile(name string) (string, error) {
 	var file *os.File = nil
 	var err error
-	var temp string
-
+	var temp, contentstore string
+	var info *imgworker.ListedImage
 	// Files should be in URI/repository:<file>
 	ok := false
 
@@ -149,22 +150,21 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 	}
 
 	for _, uri := range c.RepoData.Urls {
-
 		file, err = config.LuetCfg.GetSystem().TempFile("DockerClient")
 		if err != nil {
 			continue
 		}
 
-		contentstore, err := config.LuetCfg.GetSystem().TempDir("contentstore")
+		contentstore, err = config.LuetCfg.GetSystem().TempDir("contentstore")
 		if err != nil {
 			Warning("Cannot create contentstore", err.Error())
 			continue
 		}
 
-		imageName := fmt.Sprintf("%s:%s", uri, name)
+		imageName := fmt.Sprintf("%s:%s", uri, helpers.StripInvalidStringsFromImage(name))
 		Info("Downloading", imageName)
 
-		info, err := helpers.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
+		info, err = helpers.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
 		if err != nil {
 			Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
 			continue
@@ -175,7 +175,6 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 
 		Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name())
 		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
-
 		if err != nil {
 			continue
 		}

pkg/installer/client/docker_test.go

@@ -20,7 +20,9 @@ import (
 	"os"
 	"path/filepath"
 
-	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -54,9 +56,9 @@ var _ = Describe("Docker client", func() {
 		})
 
 		It("Downloads artifacts", func() {
-			f, err := c.DownloadArtifact(&compiler.PackageArtifact{
+			f, err := c.DownloadArtifact(&artifact.PackageArtifact{
 				Path: "test.tar",
-				CompileSpec: &compiler.LuetCompilationSpec{
+				CompileSpec: &compilerspec.LuetCompilationSpec{
 					Package: &pkg.DefaultPackage{
 						Name:     "c",
 						Category: "test",
@@ -71,7 +73,7 @@ var _ = Describe("Docker client", func() {
 			Expect(f.Unpack(tmpdir, false)).ToNot(HaveOccurred())
 			Expect(helpers.Read(filepath.Join(tmpdir, "c"))).To(Equal("c\n"))
 			Expect(helpers.Read(filepath.Join(tmpdir, "cd"))).To(Equal("c\n"))
-			os.RemoveAll(f.GetPath())
+			os.RemoveAll(f.Path)
 		})
 	})
 })

pkg/installer/client/http.go

@@ -24,9 +24,9 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	. "github.com/mudler/luet/pkg/logger"
 
-	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 
@@ -66,13 +66,13 @@ func Round(input float64) float64 {
 	return math.Floor(input + 0.5)
 }
 
-func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	var u *url.URL = nil
 	var err error
 	var req *grab.Request
 	var temp string
 
-	artifactName := path.Base(artifact.GetPath())
+	artifactName := path.Base(a.Path)
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 	ok := false
 
@@ -168,8 +168,8 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 		}
 	}
 
-	newart := artifact
-	newart.SetPath(cacheFile)
+	newart := a
+	newart.Path = cacheFile
 	return newart, nil
 }
 

pkg/installer/client/http_test.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"path/filepath"
 
-	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	helpers "github.com/mudler/luet/pkg/helpers"
 
 	. "github.com/mudler/luet/pkg/installer/client"
@@ -63,10 +63,10 @@ var _ = Describe("Http client", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
-			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
+			path, err := c.DownloadArtifact(&artifact.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path.GetPath())).To(Equal("test"))
-			os.RemoveAll(path.GetPath())
+			Expect(helpers.Read(path.Path)).To(Equal("test"))
+			os.RemoveAll(path.Path)
 		})
 
 	})

pkg/installer/client/local.go

@@ -20,10 +20,10 @@ import (
 	"path"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 
-	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/helpers"
 )
 
@@ -35,11 +35,11 @@ func NewLocalClient(r RepoData) *LocalClient {
 	return &LocalClient{RepoData: r}
 }
 
-func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+func (c *LocalClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	var err error
 
 	rootfs := ""
-	artifactName := path.Base(artifact.GetPath())
+	artifactName := path.Base(a.Path)
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 
 	if !config.LuetCfg.ConfigFromHost {
@@ -74,8 +74,8 @@ func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Art
 		}
 	}
 
-	newart := artifact
-	newart.SetPath(cacheFile)
+	newart := a
+	newart.Path = cacheFile
 	return newart, nil
 }
 

pkg/installer/client/local_test.go

@@ -20,7 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
-	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	helpers "github.com/mudler/luet/pkg/helpers"
 
 	. "github.com/mudler/luet/pkg/installer/client"
@@ -56,10 +56,10 @@ var _ = Describe("Local client", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
-			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
+			path, err := c.DownloadArtifact(&artifact.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path.GetPath())).To(Equal("test"))
-			os.RemoveAll(path.GetPath())
+			Expect(helpers.Read(path.Path)).To(Equal("test"))
+			os.RemoveAll(path.Path)
 		})
 
 	})

pkg/installer/finalizer.go

@@ -16,6 +16,7 @@
 package installer
 
 import (
+	"os"
 	"os/exec"
 
 	"github.com/ghodss/yaml"
@@ -48,7 +49,7 @@ func (f *LuetFinalizer) RunInstall(s *System) error {
 	for _, c := range f.Install {
 		toRun := append(args, c)
 		Info(":shell: Executing finalizer on ", s.Target, cmd, toRun)
-		if s.Target == "/" {
+		if s.Target == string(os.PathSeparator) {
 			cmd := exec.Command(cmd, toRun...)
 			stdoutStderr, err := cmd.CombinedOutput()
 			if err != nil {

pkg/installer/installer.go

@@ -24,9 +24,10 @@ import (
 	"strings"
 	"sync"
 
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+
 	. "github.com/logrusorgru/aurora"
 	"github.com/mudler/luet/pkg/bus"
-	compiler "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -58,11 +59,11 @@ type LuetInstaller struct {
 
 type ArtifactMatch struct {
 	Package    pkg.Package
-	Artifact   compiler.Artifact
-	Repository Repository
+	Artifact   *artifact.PackageArtifact
+	Repository *LuetSystemRepository
 }
 
-func NewLuetInstaller(opts LuetInstallerOptions) Installer {
+func NewLuetInstaller(opts LuetInstallerOptions) *LuetInstaller {
 	return &LuetInstaller{Options: opts}
 }
 
@@ -106,11 +107,11 @@ func (l *LuetInstaller) computeUpgrade(syncedRepos Repositories, s *System) (pkg
 				continue
 			}
 			for _, artefact := range matches[0].Repo.GetIndex() {
-				if artefact.GetCompileSpec().GetPackage() == nil {
+				if artefact.CompileSpec.GetPackage() == nil {
 					return uninstall, toInstall, errors.New("Package in compilespec empty")
 
 				}
-				if artefact.GetCompileSpec().GetPackage().Matches(p) && artefact.GetCompileSpec().GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
+				if artefact.CompileSpec.GetPackage().Matches(p) && artefact.CompileSpec.GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
 					toInstall = append(toInstall, matches[0].Package).Unique()
 					uninstall = append(uninstall, p).Unique()
 				}
@@ -127,6 +128,8 @@ func packsToList(p pkg.Packages) string {
 	for _, pp := range p {
 		packs = append(packs, pp.HumanReadableString())
 	}
+
+	sort.Strings(packs)
 	return strings.Join(packs, " ")
 }
 
@@ -136,6 +139,7 @@ func matchesToList(artefacts map[string]ArtifactMatch) string {
 	for fingerprint, match := range artefacts {
 		packs = append(packs, fmt.Sprintf("%s (%s)", fingerprint, match.Repository.GetName()))
 	}
+	sort.Strings(packs)
 	return strings.Join(packs, " ")
 }
 
@@ -152,39 +156,7 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 		Info(":memo: note: will consider new build revisions while upgrading")
 	}
 
-	Spinner(32)
-	uninstall, toInstall, err := l.computeUpgrade(syncedRepos, s)
-	if err != nil {
-		return errors.Wrap(err, "failed computing upgrade")
-	}
-	SpinnerStop()
-
-	if len(uninstall) > 0 {
-		Info(":recycle: Packages that are going to be removed from the system:\n ", Yellow(packsToList(uninstall)).BgBlack().String())
-	}
-
-	if len(toInstall) > 0 {
-		Info(":zap:Packages that are going to be installed in the system:\n ", Green(packsToList(toInstall)).BgBlack().String())
-	}
-
-	if len(toInstall) == 0 && len(uninstall) == 0 {
-		Info("Nothing to do")
-		return nil
-	}
-
-	if l.Options.Ask {
-		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
-		if Ask() {
-			l.Options.Ask = false // Don't prompt anymore
-			return l.swap(syncedRepos, uninstall, toInstall, s, true)
-		} else {
-			return errors.New("Aborted by user")
-		}
-	}
-
-	Spinner(32)
-	defer SpinnerStop()
-	return l.swap(syncedRepos, uninstall, toInstall, s, true)
+	return l.checkAndUpgrade(syncedRepos, s)
 }
 
 func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
@@ -225,11 +197,19 @@ func (l *LuetInstaller) Swap(toRemove pkg.Packages, toInstall pkg.Packages, s *S
 			toRemoveFinal = append(toRemoveFinal, pp)
 		}
 	}
+	o := Option{
+		FullUninstall:      false,
+		Force:              true,
+		CheckConflicts:     false,
+		FullCleanUninstall: false,
+		NoDeps:             l.Options.NoDeps,
+		OnlyDeps:           false,
+	}
 
-	return l.swap(syncedRepos, toRemoveFinal, toInstall, s, false)
+	return l.swap(o, syncedRepos, toRemoveFinal, toInstall, s)
 }
 
-func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
+func (l *LuetInstaller) computeSwap(o Option, syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
 
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
@@ -244,8 +224,8 @@ func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packa
 
 	systemAfterChanges := &System{Database: installedtmp}
 
-	packs, err := l.computeUninstall(systemAfterChanges, toRemove...)
-	if err != nil && !l.Options.Force {
+	packs, err := l.computeUninstall(o, systemAfterChanges, toRemove...)
+	if err != nil && !o.Force {
 		Error("Failed computing uninstall for ", packsToList(toRemove))
 		return nil, nil, nil, nil, errors.Wrap(err, "computing uninstall "+packsToList(toRemove))
 	}
@@ -256,30 +236,16 @@ func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packa
 		}
 	}
 
-	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, toInstall, systemAfterChanges)
+	match, packages, assertions, allRepos, err := l.computeInstall(o, syncedRepos, toInstall, systemAfterChanges)
 	for _, p := range toInstall {
 		assertions = append(assertions, solver.PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
 	}
 	return match, packages, assertions, allRepos, err
 }
 
-func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System, forceNodeps bool) error {
-	forced := l.Options.Force
-	nodeps := l.Options.NoDeps
+func (l *LuetInstaller) swap(o Option, syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) error {
 
-	// We don't want any conflict with the installed to raise during the upgrade.
-	// In this way we both force uninstalls and we avoid to check with conflicts
-	// against the current system state which is pending to deletion
-	// E.g. you can't check for conflicts for an upgrade of a new version of A
-	// if the old A results installed in the system. This is due to the fact that
-	// now the solver enforces the constraints and explictly denies two packages
-	// of the same version installed.
-	l.Options.Force = true
-	if forceNodeps {
-		l.Options.NoDeps = true
-	}
-
-	match, packages, assertions, allRepos, err := l.computeSwap(syncedRepos, toRemove, toInstall, s)
+	match, packages, assertions, allRepos, err := l.computeSwap(o, syncedRepos, toRemove, toInstall, s)
 	if err != nil {
 		return errors.Wrap(err, "failed computing package replacement")
 	}
@@ -309,15 +275,223 @@ func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, to
 		return nil
 	}
 
-	err = l.Uninstall(s, toRemove...)
-	if err != nil && !l.Options.Force {
-		Error("Failed uninstall for ", packsToList(toRemove))
-		return errors.Wrap(err, "uninstalling "+packsToList(toRemove))
+	ops := l.getOpsWithOptions(toRemove, match, Option{
+		Force:         o.Force,
+		NoDeps:        false,
+		OnlyDeps:      o.OnlyDeps,
+		RunFinalizers: false,
+	}, o, syncedRepos, packages, assertions, allRepos)
+
+	err = l.runOps(ops, s)
+	if err != nil {
+		return errors.Wrap(err, "failed running installer options")
 	}
 
-	l.Options.Force = forced
-	l.Options.NoDeps = nodeps
-	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+	toFinalize, err := l.getFinalizers(allRepos, assertions, match, o.NoDeps)
+	if err != nil {
+		return errors.Wrap(err, "failed getting package to finalize")
+	}
+
+	return s.ExecuteFinalizers(toFinalize)
+}
+
+type Option struct {
+	Force              bool
+	NoDeps             bool
+	CheckConflicts     bool
+	FullUninstall      bool
+	FullCleanUninstall bool
+	OnlyDeps           bool
+	RunFinalizers      bool
+}
+
+type operation struct {
+	Option  Option
+	Package pkg.Package
+}
+
+type installOperation struct {
+	operation
+	Reposiories Repositories
+	Packages    pkg.Packages
+	Assertions  solver.PackagesAssertions
+	Database    pkg.PackageDatabase
+	Matches     map[string]ArtifactMatch
+}
+
+// installerOp is the operation that is sent to the
+// upgradeWorker's channel (todo)
+type installerOp struct {
+	Uninstall operation
+	Install   installOperation
+}
+
+func (l *LuetInstaller) runOps(ops []installerOp, s *System) error {
+	all := make(chan installerOp)
+
+	wg := new(sync.WaitGroup)
+
+	// Do the real install
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.installerOpWorker(i, wg, all, s)
+	}
+
+	for _, c := range ops {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
+
+	return nil
+}
+
+// TODO: use installerOpWorker in place of all the other workers.
+// This one is general enough to read a list of operations and execute them.
+func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, c <-chan installerOp, s *System) error {
+	defer wg.Done()
+
+	for p := range c {
+		if p.Uninstall.Package != nil {
+			Debug("Replacing package inplace")
+			toUninstall, uninstall, err := l.generateUninstallFn(p.Uninstall.Option, s, p.Uninstall.Package)
+			if err != nil {
+				Error("Failed to generate Uninstall function for" + err.Error())
+				continue
+				//return errors.Wrap(err, "while computing uninstall")
+			}
+
+			err = uninstall()
+			if err != nil {
+				Error("Failed uninstall for ", packsToList(toUninstall))
+				continue
+				//return errors.Wrap(err, "uninstalling "+packsToList(toUninstall))
+			}
+		}
+		if p.Install.Package != nil {
+			artMatch := p.Install.Matches[p.Install.Package.GetFingerPrint()]
+			ass := p.Install.Assertions.Search(p.Install.Package.GetFingerPrint())
+			packageToInstall, _ := p.Install.Packages.Find(p.Install.Package.GetPackageName())
+
+			err := l.install(
+				p.Install.Option,
+				p.Install.Reposiories,
+				map[string]ArtifactMatch{p.Install.Package.GetFingerPrint(): artMatch},
+				pkg.Packages{packageToInstall},
+				solver.PackagesAssertions{*ass},
+				p.Install.Database,
+				s,
+			)
+			if err != nil {
+				Error(err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// checks wheter we can uninstall and install in place and compose installer worker ops
+func (l *LuetInstaller) getOpsWithOptions(
+	toUninstall pkg.Packages, installMatch map[string]ArtifactMatch, installOpt, uninstallOpt Option,
+	syncedRepos Repositories, toInstall pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase) []installerOp {
+	resOps := []installerOp{}
+	for _, match := range installMatch {
+		if pack, err := toUninstall.Find(match.Package.GetPackageName()); err == nil {
+			resOps = append(resOps, installerOp{
+				Uninstall: operation{Package: pack, Option: uninstallOpt},
+				Install: installOperation{
+					operation: operation{
+						Package: match.Package,
+						Option:  installOpt,
+					},
+					Matches:     installMatch,
+					Packages:    toInstall,
+					Reposiories: syncedRepos,
+					Assertions:  solution,
+					Database:    allRepos,
+				},
+			})
+		} else {
+			resOps = append(resOps, installerOp{
+				Install: installOperation{
+					operation:   operation{Package: match.Package, Option: installOpt},
+					Matches:     installMatch,
+					Reposiories: syncedRepos,
+					Packages:    toInstall,
+					Assertions:  solution,
+					Database:    allRepos,
+				},
+			})
+		}
+	}
+
+	for _, p := range toUninstall {
+		found := false
+
+		for _, match := range installMatch {
+			if match.Package.GetPackageName() == p.GetPackageName() {
+				found = true
+			}
+
+		}
+		if !found {
+			resOps = append(resOps, installerOp{
+				Uninstall: operation{Package: p, Option: uninstallOpt},
+			})
+		}
+	}
+	return resOps
+}
+
+func (l *LuetInstaller) checkAndUpgrade(r Repositories, s *System) error {
+	Spinner(32)
+	uninstall, toInstall, err := l.computeUpgrade(r, s)
+	if err != nil {
+		return errors.Wrap(err, "failed computing upgrade")
+	}
+	SpinnerStop()
+
+	if len(uninstall) > 0 {
+		Info(":recycle: Packages that are going to be removed from the system:\n ", Yellow(packsToList(uninstall)).BgBlack().String())
+	}
+
+	if len(toInstall) > 0 {
+		Info(":zap:Packages that are going to be installed in the system:\n ", Green(packsToList(toInstall)).BgBlack().String())
+	}
+
+	if len(toInstall) == 0 && len(uninstall) == 0 {
+		Info("Nothing to do")
+		return nil
+	}
+
+	// We don't want any conflict with the installed to raise during the upgrade.
+	// In this way we both force uninstalls and we avoid to check with conflicts
+	// against the current system state which is pending to deletion
+	// E.g. you can't check for conflicts for an upgrade of a new version of A
+	// if the old A results installed in the system. This is due to the fact that
+	// now the solver enforces the constraints and explictly denies two packages
+	// of the same version installed.
+	o := Option{
+		FullUninstall:      false,
+		Force:              true,
+		CheckConflicts:     false,
+		FullCleanUninstall: false,
+		NoDeps:             true,
+		OnlyDeps:           false,
+	}
+
+	if l.Options.Ask {
+		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
+		if Ask() {
+			l.Options.Ask = false // Don't prompt anymore
+			return l.swap(o, r, uninstall, toInstall, s)
+		} else {
+			return errors.New("Aborted by user")
+		}
+	}
+
+	return l.swap(o, r, uninstall, toInstall, s)
 }
 
 func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
@@ -326,7 +500,20 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 		return err
 	}
 
-	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, cp, s)
+	if len(s.Database.World()) > 0 {
+		Info(":thinking: Checking for available upgrades")
+		if err := l.checkAndUpgrade(syncedRepos, s); err != nil {
+			return errors.Wrap(err, "while checking upgrades before install")
+		}
+	}
+
+	o := Option{
+		NoDeps:        l.Options.NoDeps,
+		Force:         l.Options.Force,
+		OnlyDeps:      l.Options.OnlyDeps,
+		RunFinalizers: true,
+	}
+	match, packages, assertions, allRepos, err := l.computeInstall(o, syncedRepos, cp, s)
 	if err != nil {
 		return err
 	}
@@ -363,12 +550,12 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
 		if Ask() {
 			l.Options.Ask = false // Don't prompt anymore
-			return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+			return l.install(o, syncedRepos, match, packages, assertions, allRepos, s)
 		} else {
 			return errors.New("Aborted by user")
 		}
 	}
-	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+	return l.install(o, syncedRepos, match, packages, assertions, allRepos, s)
 }
 
 func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string]ArtifactMatch) error {
@@ -406,12 +593,12 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 	for _, repo := range syncedRepos {
 		for _, artefact := range repo.GetIndex() {
 			Debug("Checking if",
-				artefact.GetCompileSpec().GetPackage().HumanReadableString(),
+				artefact.CompileSpec.GetPackage().HumanReadableString(),
 				"from", repo.GetName(), "is installed")
 		FILES:
-			for _, f := range artefact.GetFiles() {
+			for _, f := range artefact.Files {
 				if helpers.Exists(filepath.Join(s.Target, f)) {
-					p, err := repo.GetTree().GetDatabase().FindPackage(artefact.GetCompileSpec().GetPackage())
+					p, err := repo.GetTree().GetDatabase().FindPackage(artefact.CompileSpec.GetPackage())
 					if err != nil {
 						return err
 					}
@@ -435,7 +622,7 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Failed creating package")
 		}
-		s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.GetFiles()})
+		s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.Files})
 		Info(":zap:Reclaimed package:", pack.HumanReadableString())
 	}
 	Info("Done!")
@@ -443,7 +630,7 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 	return nil
 }
 
-func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
+func (l *LuetInstaller) computeInstall(o Option, syncedRepos Repositories, cp pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
 	var p pkg.Packages
 	toInstall := map[string]ArtifactMatch{}
 	allRepos := pkg.NewInMemoryDatabase(false)
@@ -476,11 +663,11 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 	var packagesToInstall pkg.Packages
 	var err error
 
-	if !l.Options.NoDeps {
+	if !o.NoDeps {
 		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		solution, err = solv.Install(p)
 		/// TODO: PackageAssertions needs to be a map[fingerprint]pack so lookup is in O(1)
-		if err != nil && !l.Options.Force {
+		if err != nil && !o.Force {
 			return toInstall, p, solution, allRepos, errors.Wrap(err, "Failed solving solution for package")
 		}
 		// Gathers things to install
@@ -493,7 +680,7 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 				packagesToInstall = append(packagesToInstall, assertion.Package)
 			}
 		}
-	} else if !l.Options.OnlyDeps {
+	} else if !o.OnlyDeps {
 		for _, currentPack := range p {
 			if _, err := s.Database.FindPackage(currentPack); err == nil {
 				// skip matching if it is installed already
@@ -512,11 +699,11 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 		}
 	A:
 		for _, artefact := range matches[0].Repo.GetIndex() {
-			if artefact.GetCompileSpec().GetPackage() == nil {
+			if artefact.CompileSpec.GetPackage() == nil {
 				return toInstall, p, solution, allRepos, errors.New("Package in compilespec empty")
 			}
-			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
-				currentPack.SetBuildTimestamp(artefact.GetCompileSpec().GetPackage().GetBuildTimestamp())
+			if matches[0].Package.Matches(artefact.CompileSpec.GetPackage()) {
+				currentPack.SetBuildTimestamp(artefact.CompileSpec.GetPackage().GetBuildTimestamp())
 				// Filter out already installed
 				if _, err := s.Database.FindPackage(currentPack); err != nil {
 					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
@@ -528,7 +715,47 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 	return toInstall, p, solution, allRepos, nil
 }
 
-func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
+func (l *LuetInstaller) getFinalizers(allRepos pkg.PackageDatabase, solution solver.PackagesAssertions, toInstall map[string]ArtifactMatch, nodeps bool) ([]pkg.Package, error) {
+	var toFinalize []pkg.Package
+	if !nodeps {
+		// TODO: Lower those errors as warning
+		for _, w := range toInstall {
+			// Finalizers needs to run in order and in sequence.
+			ordered, err := solution.Order(allRepos, w.Package.GetFingerPrint())
+			if err != nil {
+				return toFinalize, errors.Wrap(err, "While order a solution for "+w.Package.HumanReadableString())
+			}
+		ORDER:
+			for _, ass := range ordered {
+				if ass.Value {
+					installed, ok := toInstall[ass.Package.GetFingerPrint()]
+					if !ok {
+						// It was a dep already installed in the system, so we can skip it safely
+						continue ORDER
+					}
+					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
+					if err != nil {
+						return toFinalize, errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
+					}
+
+					toFinalize = append(toFinalize, treePackage)
+				}
+			}
+
+		}
+	} else {
+		for _, c := range toInstall {
+			treePackage, err := c.Repository.GetTree().GetDatabase().FindPackage(c.Package)
+			if err != nil {
+				return toFinalize, errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
+			}
+			toFinalize = append(toFinalize, treePackage)
+		}
+	}
+	return toFinalize, nil
+}
+
+func (l *LuetInstaller) install(o Option, syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
 	// Install packages into rootfs in parallel.
 	if err := l.download(syncedRepos, toInstall); err != nil {
 		return errors.Wrap(err, "Downloading packages")
@@ -557,52 +784,25 @@ func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]A
 	for _, c := range toInstall {
 		// Annotate to the system that the package was installed
 		_, err := s.Database.CreatePackage(c.Package)
-		if err != nil && !l.Options.Force {
+		if err != nil && !o.Force {
 			return errors.Wrap(err, "Failed creating package")
 		}
 		bus.Manager.Publish(bus.EventPackageInstall, c)
 	}
-	var toFinalize []pkg.Package
-	if !l.Options.NoDeps {
-		// TODO: Lower those errors as warning
-		for _, w := range p {
-			// Finalizers needs to run in order and in sequence.
-			ordered, err := solution.Order(allRepos, w.GetFingerPrint())
-			if err != nil {
-				return errors.Wrap(err, "While order a solution for "+w.HumanReadableString())
-			}
-		ORDER:
-			for _, ass := range ordered {
-				if ass.Value {
-					installed, ok := toInstall[ass.Package.GetFingerPrint()]
-					if !ok {
-						// It was a dep already installed in the system, so we can skip it safely
-						continue ORDER
-					}
-					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
-					if err != nil {
-						return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
-					}
 
-					toFinalize = append(toFinalize, treePackage)
-				}
-			}
+	if !o.RunFinalizers {
+		return nil
+	}
 
-		}
-	} else {
-		for _, c := range toInstall {
-			treePackage, err := c.Repository.GetTree().GetDatabase().FindPackage(c.Package)
-			if err != nil {
-				return errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
-			}
-			toFinalize = append(toFinalize, treePackage)
-		}
+	toFinalize, err := l.getFinalizers(allRepos, solution, toInstall, o.NoDeps)
+	if err != nil {
+		return errors.Wrap(err, "failed getting package to finalize")
 	}
 
 	return s.ExecuteFinalizers(toFinalize)
 }
 
-func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) {
+func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (*artifact.PackageArtifact, error) {
 
 	artifact, err := a.Repository.Client().DownloadArtifact(a.Artifact)
 	if err != nil {
@@ -616,26 +816,26 @@ func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, err
 	return artifact, nil
 }
 
-func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
+func (l *LuetInstaller) installPackage(m ArtifactMatch, s *System) error {
 
-	artifact, err := l.downloadPackage(a)
+	a, err := l.downloadPackage(m)
 	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Failed downloading package")
 	}
 
-	files, err := artifact.FileList()
+	files, err := a.FileList()
 	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Could not open package archive")
 	}
 
-	err = artifact.Unpack(s.Target, true)
+	err = a.Unpack(s.Target, true)
 	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Error met while unpacking rootfs")
 	}
 
 	// First create client and download
 	// Then unpack to system
-	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.Package.GetFingerPrint(), Files: files})
+	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: m.Package.GetFingerPrint(), Files: files})
 }
 
 func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch) error {
@@ -676,6 +876,51 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 	return nil
 }
 
+func checkAndPrunePath(path string) {
+	// check if now the target path is empty
+	targetPath := filepath.Dir(path)
+
+	fi, err := os.Lstat(targetPath)
+	if err != nil {
+		//	Warning("Dir not found (it was before?) ", err.Error())
+		return
+	}
+
+	switch mode := fi.Mode(); {
+	case mode.IsDir():
+		files, err := ioutil.ReadDir(targetPath)
+		if err != nil {
+			Warning("Failed reading folder", targetPath, err.Error())
+		}
+		if len(files) != 0 {
+			Debug("Preserving not-empty folder", targetPath)
+			return
+		}
+	}
+	if err = os.Remove(targetPath); err != nil {
+		Warning("Failed removing file (maybe not present in the system target anymore ?)", targetPath, err.Error())
+	}
+}
+
+// We will try to cleanup every path from the file, if the folders left behind are empty
+func pruneEmptyFilePath(path string) {
+	checkAndPrunePath(path)
+
+	// A path is for e.g. /usr/bin/bar
+	// we want to create an array as "/usr", "/usr/bin", "/usr/bin/bar"
+	paths := strings.Split(path, string(os.PathSeparator))
+	currentPath := filepath.Join(string(os.PathSeparator), paths[0])
+	allPaths := []string{currentPath}
+	for _, p := range paths[1:] {
+		currentPath = filepath.Join(currentPath, p)
+		allPaths = append(allPaths, currentPath)
+	}
+	helpers.ReverseAny(allPaths)
+	for _, p := range allPaths {
+		checkAndPrunePath(p)
+	}
+}
+
 func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 	var cp *config.ConfigProtect
 	annotationDir := ""
@@ -737,6 +982,8 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		if err = os.Remove(target); err != nil {
 			Warning("Failed removing file (maybe not present in the system target anymore ?)", target, err.Error())
 		}
+
+		pruneEmptyFilePath(target)
 	}
 
 	for _, f := range notPresent {
@@ -750,6 +997,8 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		if err = os.Remove(target); err != nil {
 			Debug("Failed removing file (not present in the system target)", target, err.Error())
 		}
+
+		pruneEmptyFilePath(target)
 	}
 
 	err = s.Database.RemovePackageFiles(p)
@@ -767,17 +1016,17 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 	return nil
 }
 
-func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.Packages, error) {
+func (l *LuetInstaller) computeUninstall(o Option, s *System, packs ...pkg.Package) (pkg.Packages, error) {
 
 	var toUninstall pkg.Packages
 	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) TODO - mark the uninstallation in db
 	// Get installed definition
-	checkConflicts := l.Options.CheckConflicts
-	full := l.Options.FullUninstall
-	if l.Options.Force == true { // IF forced, we want to remove the package and all its requires
-		checkConflicts = false
-		full = false
-	}
+	checkConflicts := o.CheckConflicts
+	full := o.FullUninstall
+	// if o.Force == true { // IF forced, we want to remove the package and all its requires
+	// 	checkConflicts = false
+	// 	full = false
+	// }
 
 	// Create a temporary DB with the installed packages
 	// so the solver is much faster finding the deptree
@@ -787,11 +1036,11 @@ func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.P
 		return toUninstall, errors.Wrap(err, "Failed create temporary in-memory db")
 	}
 
-	if !l.Options.NoDeps {
+	if !o.NoDeps {
 		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		var solution pkg.Packages
 		var err error
-		if l.Options.FullCleanUninstall {
+		if o.FullCleanUninstall {
 			solution, err = solv.UninstallUniverse(packs)
 			if err != nil {
 				return toUninstall, errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
@@ -812,32 +1061,47 @@ func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.P
 
 	return toUninstall, nil
 }
-func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
 
+func (l *LuetInstaller) generateUninstallFn(o Option, s *System, packs ...pkg.Package) (pkg.Packages, func() error, error) {
 	for _, p := range packs {
 		if packs, _ := s.Database.FindPackages(p); len(packs) == 0 {
-			return errors.New("Package not found in the system")
+			return nil, nil, errors.New("Package not found in the system")
 		}
-
 	}
 
-	Spinner(32)
-	toUninstall, err := l.computeUninstall(s, packs...)
+	toUninstall, err := l.computeUninstall(o, s, packs...)
 	if err != nil {
-		return errors.Wrap(err, "while computing uninstall")
+		return nil, nil, errors.Wrap(err, "while computing uninstall")
 	}
-	SpinnerStop()
 
 	uninstall := func() error {
 		for _, p := range toUninstall {
 			err := l.uninstall(p, s)
-			if err != nil && !l.Options.Force {
+			if err != nil && !o.Force {
 				return errors.Wrap(err, "Uninstall failed")
 			}
 		}
 		return nil
 	}
 
+	return toUninstall, uninstall, nil
+}
+
+func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
+
+	Spinner(32)
+	o := Option{
+		FullUninstall:      l.Options.FullUninstall,
+		Force:              l.Options.Force,
+		CheckConflicts:     l.Options.CheckConflicts,
+		FullCleanUninstall: l.Options.FullCleanUninstall,
+	}
+	toUninstall, uninstall, err := l.generateUninstallFn(o, s, packs...)
+	if err != nil {
+		return errors.Wrap(err, "while computing uninstall")
+	}
+	SpinnerStop()
+
 	if len(toUninstall) == 0 {
 		Info("Nothing to do")
 		return nil
@@ -855,4 +1119,4 @@ func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
 	return uninstall()
 }
 
-func (l *LuetInstaller) Repositories(r []Repository) { l.PackageRepositories = r }
+func (l *LuetInstaller) Repositories(r []*LuetSystemRepository) { l.PackageRepositories = r }

pkg/installer/installer_test.go

@@ -23,9 +23,10 @@ import (
 	//	. "github.com/mudler/luet/pkg/installer"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/helpers"
-	"github.com/mudler/luet/pkg/installer"
-	solver "github.com/mudler/luet/pkg/solver"
 
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -34,7 +35,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func stubRepo(tmpdir, tree string) (installer.Repository, error) {
+func stubRepo(tmpdir, tree string) (*LuetSystemRepository, error) {
 	return GenerateRepository(
 		"test",
 		"description",
@@ -43,10 +44,11 @@ func stubRepo(tmpdir, tree string) (installer.Repository, error) {
 		1,
 		tmpdir,
 		[]string{tree},
-		pkg.NewInMemoryDatabase(false), nil, "", false, false)
+		pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 }
 
 var _ = Describe("Installer", func() {
+
 	Context("Writes a repository definition", func() {
 		It("Writes a repo and can install packages from it", func() {
 			//repo:=NewLuetSystemRepository()
@@ -62,7 +64,9 @@ var _ = Describe("Installer", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
+				generalRecipe.GetDatabase(),
+				options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -77,12 +81,11 @@ var _ = Describe("Installer", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
-			artifact, err := c.Compile(false, spec)
+			a, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(a.Path)).To(BeTrue())
+			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -178,7 +181,7 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
 			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
-				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+				generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -193,12 +196,11 @@ urls:
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -217,7 +219,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			treeFile := NewDefaultTreeRepositoryFile()
-			treeFile.SetCompressionType(compiler.None)
+			treeFile.SetCompressionType(compression.None)
 			repo.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
@@ -298,7 +300,8 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -313,12 +316,11 @@ urls:
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -340,7 +342,7 @@ urls:
 				[]string{tmpdir}, 1,
 				tmpdir,
 				[]string{"../../tests/fixtures/buildable"},
-				pkg.NewInMemoryDatabase(false), nil, "", false, false)
+				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
@@ -423,7 +425,8 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -438,12 +441,11 @@ urls:
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -466,7 +468,7 @@ urls:
 				1,
 				tmpdir,
 				[]string{"../../tests/fixtures/buildable"},
-				pkg.NewInMemoryDatabase(false), nil, "", false, false)
+				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
@@ -518,7 +520,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), options.Concurrency(2))
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -526,11 +528,10 @@ urls:
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
 
 			spec.SetOutputPath(tmpdir2)
-			c.SetConcurrency(2)
 
 			artifact, err = c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
 
 			repo, err = stubRepo(tmpdir2, "../../tests/fixtures/alpine")
 			Expect(err).ToNot(HaveOccurred())
@@ -577,7 +578,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -595,9 +596,8 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
@@ -695,8 +695,8 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 			Expect(len(generalRecipeNewRepo.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase())
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -718,13 +718,12 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdirnewrepo)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec3))
 
 			Expect(errs).To(BeEmpty())
 
-			_, errs = c2.CompileParallel(false, compiler.NewLuetCompilationspecs(spec2))
+			_, errs = c2.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeEmpty())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
@@ -827,7 +826,12 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(
+				backend.NewSimpleDockerBackend(),
+				generalRecipe.GetDatabase(),
+				options.Concurrency(2),
+				options.WithCompressionType(compression.GZip),
+			)
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -844,9 +848,8 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
-			c.SetCompressionType(compiler.GZip)
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
@@ -985,7 +988,9 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.Concurrency(2),
+				options.WithCompressionType(compression.GZip))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1002,9 +1007,7 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
-			c.SetCompressionType(compiler.GZip)
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
@@ -1090,7 +1093,8 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.WithCompressionType(compression.GZip))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1104,9 +1108,7 @@ urls:
 			defer os.RemoveAll(tmpdir) // clean up
 			spec.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(1)
-			c.SetCompressionType(compiler.GZip)
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec3))
 
 			Expect(errs).To(BeEmpty())
 
@@ -1182,7 +1184,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(3))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase())
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1192,7 +1194,7 @@ urls:
 			defer os.RemoveAll(tmpdir2) // clean up
 			spec.SetOutputPath(tmpdir2)
 
-			_, errs = c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec))
+			_, errs = c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 
 			Expect(errs).To(BeEmpty())
 

pkg/installer/interface.go

@@ -16,67 +16,13 @@
 package installer
 
 import (
-	compiler "github.com/mudler/luet/pkg/compiler"
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/tree"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	//"github.com/mudler/luet/pkg/solver"
 )
 
-type Installer interface {
-	Install(pkg.Packages, *System) error
-	Uninstall(*System, ...pkg.Package) error
-	Upgrade(s *System) error
-	Reclaim(s *System) error
-
-	Repositories([]Repository)
-	SyncRepositories(bool) (Repositories, error)
-	Swap(pkg.Packages, pkg.Packages, *System) error
-}
-
 type Client interface {
-	DownloadArtifact(compiler.Artifact) (compiler.Artifact, error)
+	DownloadArtifact(*artifact.PackageArtifact) (*artifact.PackageArtifact, error)
 	DownloadFile(string) (string, error)
 }
 
-type Repositories []Repository
-
-type Repository interface {
-	GetName() string
-	GetDescription() string
-	GetUrls() []string
-	SetUrls([]string)
-	AddUrl(string)
-	GetPriority() int
-	GetIndex() compiler.ArtifactIndex
-	SetIndex(i compiler.ArtifactIndex)
-	GetTree() tree.Builder
-	SetTree(tree.Builder)
-	Write(path string, resetRevision, force bool) error
-	Sync(bool) (Repository, error)
-	GetTreePath() string
-	SetTreePath(string)
-	GetMetaPath() string
-	SetMetaPath(string)
-	GetType() string
-	SetType(string)
-	SetAuthentication(map[string]string)
-	GetAuthentication() map[string]string
-	GetRevision() int
-	IncrementRevision()
-	GetLastUpdate() string
-	SetLastUpdate(string)
-	Client() Client
-
-	SetPriority(int)
-	GetRepositoryFile(string) (LuetRepositoryFile, error)
-	SetRepositoryFile(string, LuetRepositoryFile)
-	SetName(p string)
-	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
-	GetBackend() compiler.CompilerBackend
-	SetBackend(b compiler.CompilerBackend)
-	FileSearch(pattern string) (pkg.Packages, error)
-	SearchArtefact(p pkg.Package) (compiler.Artifact, error)
-
-	SetVerify(bool)
-	GetVerify() bool
-}
+type Repositories []*LuetSystemRepository

pkg/installer/repository_docker.go

@@ -0,0 +1,270 @@
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+
+	"github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/mudler/luet/pkg/bus"
+	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	"github.com/pkg/errors"
+)
+
+type dockerRepositoryGenerator struct {
+	b                compiler.CompilerBackend
+	imagePrefix      string
+	imagePush, force bool
+}
+
+func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
+
+	Info("Generating docker images for packages in", l.imagePrefix)
+	var art []*artifact.PackageArtifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			Debug("Skipping", info.Name(), err.Error())
+			return nil
+		}
+		if info.IsDir() {
+			Debug("Skipping directories")
+			return nil
+		}
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil
+		}
+
+		if err := l.pushImageFromArtifact(artifact.NewPackageArtifact(currentpath), l.b); err != nil {
+			return errors.Wrap(err, "while pushing metadata file associated to the artifact")
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		a, err := artifact.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+		// Set the path relative to the file.
+		// The metadata contains the full path where the file was located during buildtime.
+		a.Path = filepath.Join(filepath.Dir(currentpath), filepath.Base(a.Path))
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(a.CompileSpec.Package); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				a.CompileSpec.Package.HumanReadableString()))
+			return nil
+		}
+
+		packageImage := fmt.Sprintf("%s:%s", l.imagePrefix, a.CompileSpec.GetPackage().ImageID())
+
+		if l.imagePush && l.b.ImageAvailable(packageImage) && !l.force {
+			Info("Image", packageImage, "already present, skipping. use --force-push to override")
+		} else {
+			Info("Generating final image", packageImage,
+				"for package ", a.CompileSpec.GetPackage().HumanReadableString())
+			if opts, err := a.GenerateFinalImage(packageImage, l.b, true); err != nil {
+				return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+			}
+		}
+		if l.imagePush {
+			if err := pushImage(l.b, packageImage, l.force); err != nil {
+				return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage)
+			}
+		}
+
+		art = append(art, a)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
+func pushImage(b compiler.CompilerBackend, image string, force bool) error {
+	if b.ImageAvailable(image) && !force {
+		Debug("Image", image, "already present, skipping")
+		return nil
+	}
+	return b.Push(backend.Options{ImageName: image})
+}
+
+func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string) error {
+	Debug("Generating image", imageTree)
+	if opts, err := a.GenerateFinalImage(imageTree, d.b, false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
+	}
+	if d.imagePush {
+		if err := pushImage(d.b, imageTree, true); err != nil {
+			return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree)
+		}
+	}
+	return nil
+}
+
+func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec string, r *LuetSystemRepository) error {
+	// create temp dir for metafile
+	metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaDir) // clean up
+
+	tempRepoFile := filepath.Join(metaDir, REPOSITORY_SPECFILE+".tar")
+	if err := helpers.Tar(repospec, tempRepoFile); err != nil {
+		return errors.Wrap(err, "Error met while archiving repository file")
+	}
+
+	a := artifact.NewPackageArtifact(tempRepoFile)
+	imageRepo := fmt.Sprintf("%s:%s", d.imagePrefix, REPOSITORY_SPECFILE)
+
+	if err := d.pushFileFromArtifact(a, imageRepo); err != nil {
+		return errors.Wrap(err, "while pushing file from artifact")
+	}
+	return nil
+}
+
+func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, b compiler.CompilerBackend) error {
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	treeArchive, err := artifact.CreateArtifactForFile(a.Path)
+	if err != nil {
+		return errors.Wrap(err, "failed generating checksums for tree")
+	}
+	imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, helpers.StripInvalidStringsFromImage(a.GetFileName()))
+
+	return d.pushFileFromArtifact(treeArchive, imageTree)
+}
+
+// Generate creates a Docker luet repository
+func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefix string, resetRevision bool) error {
+	// - Iterate over meta, build final images, push them if necessary
+	//   - while pushing, check if image already exists, and if exist push them only if --force is supplied
+	// - Generate final images for metadata and push
+
+	imageRepository := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
+
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
+
+	repoTemp, err := config.LuetCfg.GetSystem().TempDir("repo")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for repository")
+	}
+	defer os.RemoveAll(repoTemp) // clean up
+
+	if r.GetBackend().ImageAvailable(imageRepository) {
+		if err := r.GetBackend().DownloadImage(backend.Options{ImageName: imageRepository}); err != nil {
+			return errors.Wrapf(err, "while downloading '%s'", imageRepository)
+		}
+
+		if err := r.GetBackend().ExtractRootfs(backend.Options{ImageName: imageRepository, Destination: repoTemp}, false); err != nil {
+			return errors.Wrapf(err, "while extracting '%s'", imageRepository)
+		}
+	}
+
+	repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE)
+
+	// Increment the internal revision version by reading the one which is already available (if any)
+	if err := r.BumpRevision(repospec, resetRevision); err != nil {
+		return err
+	}
+
+	Info(fmt.Sprintf(
+		"For repository %s creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imageRepository,
+	})
+
+	// Create tree and repository file
+	a, err := r.AddTree(r.GetTree(), repoTemp, REPOFILE_TREE_KEY, NewDefaultTreeRepositoryFile())
+	if err != nil {
+		return errors.Wrap(err, "error met while adding runtime tree to repository")
+	}
+
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+		return errors.Wrap(err, "error met while pushing runtime tree")
+	}
+
+	a, err = r.AddTree(r.BuildTree, repoTemp, REPOFILE_COMPILER_TREE_KEY, NewDefaultCompilerTreeRepositoryFile())
+	if err != nil {
+		return errors.Wrap(err, "error met while adding compiler tree to repository")
+	}
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+		return errors.Wrap(err, "error met while pushing compiler tree")
+	}
+
+	// create temp dir for metafile
+	metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaDir) // clean up
+
+	a, err = r.AddMetadata(repospec, metaDir)
+	if err != nil {
+		return errors.Wrap(err, "failed adding Metadata file to repository")
+	}
+
+	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+		return errors.Wrap(err, "error met while pushing docker image from artifact")
+	}
+
+	if err := d.pushRepoMetadata(repospec, r); err != nil {
+		return errors.Wrap(err, "while pushing repository metadata tree")
+	}
+
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imagePrefix,
+	})
+	return nil
+}

pkg/installer/repository_local.go

@@ -0,0 +1,132 @@
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/mudler/luet/pkg/bus"
+	"github.com/pkg/errors"
+)
+
+type localRepositoryGenerator struct{}
+
+func (l *localRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
+	return buildPackageIndex(path, db)
+}
+
+func buildPackageIndex(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
+
+	var art []*artifact.PackageArtifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			Debug("Failed walking", err.Error())
+			return err
+		}
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil // Skip with no errors
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		a, err := artifact.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(a.CompileSpec.GetPackage()); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				a.CompileSpec.GetPackage().HumanReadableString()))
+			return nil
+		}
+
+		art = append(art, a)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
+// Generate creates a Local luet repository
+func (*localRepositoryGenerator) Generate(r *LuetSystemRepository, dst string, resetRevision bool) error {
+	err := os.MkdirAll(dst, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
+
+	repospec := filepath.Join(dst, REPOSITORY_SPECFILE)
+	// Increment the internal revision version by reading the one which is already available (if any)
+	if err := r.BumpRevision(repospec, resetRevision); err != nil {
+		return err
+	}
+
+	Info(fmt.Sprintf(
+		"Repository %s: creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
+
+	if _, err := r.AddTree(r.GetTree(), dst, REPOFILE_TREE_KEY, NewDefaultTreeRepositoryFile()); err != nil {
+		return errors.Wrap(err, "error met while adding runtime tree to repository")
+	}
+
+	if _, err := r.AddTree(r.BuildTree, dst, REPOFILE_COMPILER_TREE_KEY, NewDefaultCompilerTreeRepositoryFile()); err != nil {
+		return errors.Wrap(err, "error met while adding compiler tree to repository")
+	}
+
+	if _, err := r.AddMetadata(repospec, dst); err != nil {
+		return errors.Wrap(err, "failed adding Metadata file to repository")
+	}
+
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
+	return nil
+}

pkg/installer/repository_test.go

@@ -1,4 +1,4 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+// Copyright © 2019 Ettore Di Giacinto <mudler@sabayon.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -26,18 +26,19 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	config "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
-	"github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
-func dockerStubRepo(tmpdir, tree, image string, push, force bool) (installer.Repository, error) {
+func dockerStubRepo(tmpdir, tree, image string, push, force bool) (*LuetSystemRepository, error) {
 	return GenerateRepository(
 		"test",
 		"description",
@@ -46,7 +47,7 @@ func dockerStubRepo(tmpdir, tree, image string, push, force bool) (installer.Rep
 		1,
 		tmpdir,
 		[]string{tree},
-		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force)
+		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force, false, nil)
 }
 
 var _ = Describe("Repository", func() {
@@ -64,7 +65,7 @@ var _ = Describe("Repository", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -79,12 +80,11 @@ var _ = Describe("Repository", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -132,11 +132,11 @@ var _ = Describe("Repository", func() {
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase())
 			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -153,18 +153,16 @@ var _ = Describe("Repository", func() {
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			compiler2.SetConcurrency(1)
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			artifact2, err := compiler2.Compile(false, spec2)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact2.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact2.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact2.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact2.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -254,7 +252,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -266,12 +264,11 @@ urls:
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			localcompiler.SetConcurrency(1)
 
-			artifact, err := localcompiler.Compile(false, spec)
+			a, err := localcompiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(a.Path)).To(BeTrue())
+			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -300,9 +297,9 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(f)).To(ContainSubstring("name: test"))
 
-			a, err := c.DownloadArtifact(&compiler.PackageArtifact{
+			a, err = c.DownloadArtifact(&artifact.PackageArtifact{
 				Path: "test.tar",
-				CompileSpec: &compiler.LuetCompilationSpec{
+				CompileSpec: &compilerspec.LuetCompilationSpec{
 					Package: &pkg.DefaultPackage{
 						Name:     "b",
 						Category: "test",
@@ -329,7 +326,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(5))
 
-			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.99"})
 			Expect(err).ToNot(HaveOccurred())
@@ -341,12 +338,11 @@ urls:
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			localcompiler.SetConcurrency(1)
 
-			artifact, err := localcompiler.Compile(false, spec)
+			a, err := localcompiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(a.Path)).To(BeTrue())
+			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/virtuals", repoImage, true, true)
 			Expect(err).ToNot(HaveOccurred())
@@ -372,9 +368,9 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(f)).To(ContainSubstring("name: test"))
 
-			a, err := c.DownloadArtifact(&compiler.PackageArtifact{
+			a, err = c.DownloadArtifact(&artifact.PackageArtifact{
 				Path: "test.tar",
-				CompileSpec: &compiler.LuetCompilationSpec{
+				CompileSpec: &compilerspec.LuetCompilationSpec{
 					Package: &pkg.DefaultPackage{
 						Name:     "a",
 						Category: "test",
@@ -397,14 +393,14 @@ urls:
 			repos := Repositories{
 				&LuetSystemRepository{
 					Index: compiler.ArtifactIndex{
-						&compiler.PackageArtifact{
-							CompileSpec: &compiler.LuetCompilationSpec{
+						&artifact.PackageArtifact{
+							CompileSpec: &compilerspec.LuetCompilationSpec{
 								Package: &pkg.DefaultPackage{},
 							},
 							Path:  "bar",
 							Files: []string{"boo"},
 						},
-						&compiler.PackageArtifact{
+						&artifact.PackageArtifact{
 							Path:  "d",
 							Files: []string{"baz"},
 						},
@@ -414,15 +410,15 @@ urls:
 
 			matches := repos.SearchPackages("bo", FileSearch)
 			Expect(len(matches)).To(Equal(1))
-			Expect(matches[0].Artifact.GetPath()).To(Equal("bar"))
+			Expect(matches[0].Artifact.Path).To(Equal("bar"))
 		})
 
 		It("Searches packages", func() {
 			repo := &LuetSystemRepository{
 				Index: compiler.ArtifactIndex{
-					&compiler.PackageArtifact{
+					&artifact.PackageArtifact{
 						Path: "foo",
-						CompileSpec: &compiler.LuetCompilationSpec{
+						CompileSpec: &compilerspec.LuetCompilationSpec{
 							Package: &pkg.DefaultPackage{
 								Name:     "foo",
 								Category: "bar",
@@ -430,9 +426,9 @@ urls:
 							},
 						},
 					},
-					&compiler.PackageArtifact{
+					&artifact.PackageArtifact{
 						Path: "baz",
-						CompileSpec: &compiler.LuetCompilationSpec{
+						CompileSpec: &compilerspec.LuetCompilationSpec{
 							Package: &pkg.DefaultPackage{
 								Name:     "foo",
 								Category: "baz",
@@ -449,7 +445,7 @@ urls:
 				Version:  "1.0",
 			})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(a.GetPath()).To(Equal("baz"))
+			Expect(a.Path).To(Equal("baz"))
 
 			a, err = repo.SearchArtefact(&pkg.DefaultPackage{
 				Name:     "foo",
@@ -457,7 +453,7 @@ urls:
 				Version:  "1.0",
 			})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(a.GetPath()).To(Equal("foo"))
+			Expect(a.Path).To(Equal("foo"))
 
 			// Doesn't exist. so must fail
 			_, err = repo.SearchArtefact(&pkg.DefaultPackage{

pkg/installer/system.go

@@ -24,7 +24,7 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
 	executedFinalizer := map[string]bool{}
 	for _, p := range packs {
 		if helpers.Exists(p.Rel(tree.FinalizerFile)) {
-			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile), "")
+			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile))
 			if err != nil {
 				Warning("Failed rendering finalizer for ", p.HumanReadableString(), err.Error())
 				errs = multierror.Append(errs, err)

pkg/package/package.go

@@ -113,6 +113,12 @@ type Package interface {
 	GetBuildTimestamp() string
 
 	Clone() Package
+
+	GetMetadataFilePath() string
+	SetTreeDir(s string)
+	GetTreeDir() string
+
+	JSON() ([]byte, error)
 }
 
 type Tree interface {
@@ -210,6 +216,11 @@ func (t *DefaultPackage) JSON() ([]byte, error) {
 	return buffer.Bytes(), err
 }
 
+// GetMetadataFilePath returns the canonical name of an artifact metadata file
+func (d *DefaultPackage) GetMetadataFilePath() string {
+	return d.GetFingerPrint() + ".metadata.yaml"
+}
+
 // DefaultPackage represent a standard package definition
 type DefaultPackage struct {
 	ID               int               `storm:"id,increment" json:"id"` // primary key with auto increment
@@ -235,6 +246,8 @@ type DefaultPackage struct {
 	BuildTimestamp string   `json:"buildtimestamp,omitempty"`
 
 	Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
+
+	TreeDir string `json:"treedir,omitempty"`
 }
 
 // State represent the package state
@@ -251,6 +264,12 @@ func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*D
 	}
 }
 
+func (p *DefaultPackage) SetTreeDir(s string) {
+	p.TreeDir = s
+}
+func (p *DefaultPackage) GetTreeDir() string {
+	return p.TreeDir
+}
 func (p *DefaultPackage) String() string {
 	b, err := p.JSON()
 	if err != nil {
@@ -290,7 +309,7 @@ func (p *DefaultPackage) GetPackageName() string {
 }
 
 func (p *DefaultPackage) ImageID() string {
-	return strings.ReplaceAll(p.GetFingerPrint(), "+", "-")
+	return helpers.StripInvalidStringsFromImage(p.GetFingerPrint())
 }
 
 // GetBuildTimestamp returns the package build timestamp
@@ -626,6 +645,16 @@ func (set Packages) Best(v version.Versioner) Package {
 	return versionsMap[sorted[len(sorted)-1]]
 }
 
+func (set Packages) Find(packageName string) (Package, error) {
+	for _, p := range set {
+		if p.GetPackageName() == packageName {
+			return p, nil
+		}
+	}
+
+	return &DefaultPackage{}, errors.New("package not found")
+}
+
 func (set Packages) Unique() Packages {
 	var result Packages
 	uniq := make(map[string]Package)

pkg/tree/compiler_recipe.go

@@ -57,6 +57,17 @@ type CompilerRecipe struct {
 	Recipe
 }
 
+// CompilerRecipes copies tree 1:1 as they contain the specs
+// and the build context required for reproducible builds
+func (r *CompilerRecipe) Save(path string) error {
+	for _, p := range r.SourcePath {
+		if err := helpers.CopyDir(p, filepath.Join(path, filepath.Base(p))); err != nil {
+			return errors.Wrap(err, "while copying source tree")
+		}
+	}
+	return nil
+}
+
 func (r *CompilerRecipe) Load(path string) error {
 
 	r.SourcePath = append(r.SourcePath, path)
@@ -87,12 +98,13 @@ func (r *CompilerRecipe) Load(path string) error {
 			}
 			// Path is set only internally when tree is loaded from disk
 			pack.SetPath(filepath.Dir(currentpath))
+			pack.SetTreeDir(path)
 
 			// Instead of rdeps, have a different tree for build deps.
 			compileDefPath := pack.Rel(CompilerDefinitionFile)
 			if helpers.Exists(compileDefPath) {
 
-				dat, err := helpers.RenderFiles(compileDefPath, currentpath, "")
+				dat, err := helpers.RenderFiles(compileDefPath, currentpath)
 				if err != nil {
 					return errors.Wrap(err,
 						"Error templating file "+CompilerDefinitionFile+" from "+
@@ -115,18 +127,25 @@ func (r *CompilerRecipe) Load(path string) error {
 			}
 
 		case CollectionFile:
+
 			dat, err := ioutil.ReadFile(currentpath)
 			if err != nil {
 				return errors.Wrap(err, "Error reading file "+currentpath)
 			}
+
 			packs, err := pkg.DefaultPackagesFromYaml(dat)
 			if err != nil {
 				return errors.Wrap(err, "Error reading yaml "+currentpath)
 			}
+
 			packsRaw, err := pkg.GetRawPackages(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading raw packages from "+currentpath)
+			}
 
 			for _, pack := range packs {
 				pack.SetPath(filepath.Dir(currentpath))
+				pack.SetTreeDir(path)
 
 				// Instead of rdeps, have a different tree for build deps.
 				compileDefPath := pack.Rel(CompilerDefinitionFile)
@@ -159,9 +178,7 @@ func (r *CompilerRecipe) Load(path string) error {
 					return errors.Wrap(err, "Error creating package "+pack.GetName())
 				}
 			}
-
 		}
-
 		return nil
 	}
 

tests/fixtures/docker_repo/c/build.yaml

@@ -0,0 +1,11 @@
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo c > /c
+  - echo c > /cd
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"
+

tests/fixtures/docker_repo/c/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.0"

tests/fixtures/docker_repo/cat/b/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/docker_repo/cat/b/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/fixtures/docker_repo/cat/b/generate.sh

@@ -0,0 +1 @@
+echo generated > /artifact42

tests/fixtures/docker_repo/cat/cat2/a/build.yaml

@@ -0,0 +1,10 @@
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/docker_repo/cat/cat2/a/definition.yaml

@@ -0,0 +1,8 @@
+category: "test"
+name: "a"
+version: "1.0"
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"
+

tests/fixtures/docker_repo/d/build.yaml

@@ -0,0 +1,11 @@
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo s > /d
+  - echo dd > /dd
+requires:
+- category: "test"
+  name: "c"
+  version: "1.0"
+

tests/fixtures/docker_repo/d/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "d"
+version: "1.0"

tests/fixtures/docker_repo/interpolated/build.yaml

@@ -0,0 +1,3 @@
+steps:
+  - echo s > /interpolated-{{.Values.foo}}-{{.Values.extra}}
+image: "alpine"

tests/fixtures/docker_repo/interpolated/definition.yaml

@@ -0,0 +1,4 @@
+category: "test"
+name: "interpolated"
+version: "1.0+2"
+foo: "bar"

tests/fixtures/docker_repo/z/build.yaml

@@ -0,0 +1,3 @@
+steps:
+  - echo s > /z
+image: "alpine"

tests/fixtures/docker_repo/z/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "z"
+version: "1.0+2"

tests/fixtures/owners/delta/build.yaml

@@ -0,0 +1,8 @@
+image: "busybox"
+
+steps:
+- adduser foo -D
+- addgroup bar
+- touch /foo
+- chown foo:bar /foo
+- chmod 500 /foo

tests/fixtures/owners/delta/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "delta"
+version: "1.0"

tests/fixtures/owners/unpack/build.yaml

@@ -0,0 +1,13 @@
+image: "busybox"
+
+steps:
+- adduser baz -D
+- addgroup ba
+- touch /bar
+- chown baz:ba /bar
+- chmod 600 /bar
+- ls -liah /bar
+unpack: true
+
+includes:
+- bar

tests/fixtures/owners/unpack/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "unpack"
+version: "1.0"

tests/fixtures/perms/pkgA/0.1/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+unpack: true
+includes:
+  - /foo
+  - /foo/bar
+  - /foo/bar/.keep
+steps:
+- mkdir -p /foo/bar
+- touch /foo/bar/.keep
+- chown 100:100 /foo/bar
+- chown 101:101 /foo/bar/.keep

tests/fixtures/perms/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "perms"
+version: "0.1"

tests/fixtures/perms/pkgB/0.1/build.yaml

@@ -0,0 +1,7 @@
+image: "alpine"
+
+steps:
+- mkdir -p /foo/baz
+- touch /foo/baz/.keep
+- chown 100:100 /foo/baz
+- chown 101:101 /foo/baz/.keep

tests/fixtures/perms/pkgB/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "perms2"
+version: "0.1"

tests/integration/01_simple_docker.sh

@@ -3,27 +3,41 @@
 export LUET_NOLOCK=true
 
 oneTimeSetUp() {
-export tmpdir="$(mktemp -d)"
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
 }
 
 oneTimeTearDown() {
     rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
 }
 
 testBuild() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+    cat <<EOF > $tmpdir/default.yaml
+extra: "bar"
+foo: "baz"
+EOF
     mkdir $tmpdir/testbuild
-    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression zstd test/c@1.0 > /dev/null
+    luet build --tree "$ROOT_DIR/tests/fixtures/docker_repo" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --image-repository "${TEST_DOCKER_IMAGE}-cache" --push \
+               --compression zstd --values $tmpdir/default.yaml \
+               test/c@1.0 test/z test/interpolated #> /dev/null
     buildst=$?
     assertEquals 'builds successfully' "$buildst" "0"
     assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.zst' ]"
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
+    assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
 }
 
 testRepo() {
     # Disable tests which require a DOCKER registry
     [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
 
-    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    createres=$(luet create-repo --tree "$ROOT_DIR/tests/fixtures/docker_repo" \
     --output "${TEST_DOCKER_IMAGE}" \
     --packages $tmpdir/testbuild \
     --name "test" \
@@ -33,10 +47,11 @@ testRepo() {
     --tree-filename foo.tar \
     --meta-filename repository.meta.tar \
     --meta-compression zstd \
-    --type docker --push-images --force-push
+    --type docker --push-images --force-push)
 
     createst=$?
     assertEquals 'create repo successfully' "$createst" "0"
+    assertContains 'contains image push' "$createres" 'Pushed image: quay.io/mocaccinoos/integration-test:z-test-1.0-2'
 }
 
 testConfig() {
@@ -65,10 +80,12 @@ testInstall() {
     # Disable tests which require a DOCKER registry
     [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
 
-    luet install -y --config $tmpdir/luet.yaml test/c@1.0
+    luet install -y --config $tmpdir/luet.yaml test/c@1.0 test/z test/interpolated
     installst=$?
     assertEquals 'install test successfully' "$installst" "0"
     assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package Z installed' "[ -e '$tmpdir/testrootfs/z' ]"
+    assertTrue 'package interpolated installed' "[ -e '$tmpdir/testrootfs/interpolated-baz-bar' ]"
 }
 
 testReInstall() {
@@ -85,10 +102,11 @@ testUnInstall() {
     # Disable tests which require a DOCKER registry
     [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
 
-    luet uninstall -y --config $tmpdir/luet.yaml test/c@1.0
+    luet uninstall -y --config $tmpdir/luet.yaml test/c@1.0 test/z
     installst=$?
     assertEquals 'uninstall test successfully' "$installst" "0"
     assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package Z uninstalled' "[ ! -e '$tmpdir/testrootfs/z' ]"
 }
 
 testInstallAgain() {
@@ -96,11 +114,13 @@ testInstallAgain() {
     [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
 
     assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
-    output=$(luet install -y --config $tmpdir/luet.yaml test/c@1.0)
+    output=$(luet install -y --config $tmpdir/luet.yaml test/c@1.0 test/z)
     installst=$?
     assertEquals 'install test successfully' "$installst" "0"
     assertNotContains 'contains warning' "$output" 'No packages to install'
     assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package Z installed' "[ -e '$tmpdir/testrootfs/z' ]"
+    assertTrue 'package Z in cache' "[ -e '$tmpdir/testrootfs/packages/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zst' ]"
 }
 

tests/integration/01_simple_http.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+TEST_PORT="${TEST_PORT:-9090}"
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    kill '%1' || true
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression zstd test/c@1.0 > /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
+}
+
+testRepo() {
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --tree-compression zstd \
+    --tree-filename foo.tar \
+    --meta-filename repository.meta.tar \
+    --meta-compression zstd \
+    --type disk
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+
+    luet serve-repo --dir $tmpdir/testbuild --port $TEST_PORT &
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "http"
+     enable: true
+     urls:
+       - "http://127.0.0.1:$TEST_PORT"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install -y --config $tmpdir/luet.yaml test/c@1.0
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testReInstall() {
+    output=$(luet install -y --config $tmpdir/luet.yaml  test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertContains 'contains warning' "$output" 'No packages to install'
+}
+
+testUnInstall() {
+    luet uninstall -y --config $tmpdir/luet.yaml test/c@1.0
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+testInstallAgain() {
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    output=$(luet install -y --config $tmpdir/luet.yaml test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'No packages to install'
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zst' ]"
+}
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'cleanup test successfully' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/05_upgrade.sh

@@ -95,13 +95,6 @@ testInstall() {
     installst=$?
     assertEquals 'install test successfully' "$installst" "0"
 
-    luet install -y --config $tmpdir/luet.yaml test/a@1.1
-    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
-    installst=$?
-    assertEquals 'install test successfully' "$installst" "0"
-    assertTrue 'package keeps old A' "[ -e '$tmpdir/testrootfs/testaa' ]"
-    assertTrue 'package new A was not installed' "[ ! -e '$tmpdir/testrootfs/testlatest' ]"
-
     luet install -y --config $tmpdir/luet.yaml test/c@1.0
     installst=$?
     assertEquals 'install test successfully' "$installst" "0"

tests/integration/06_search.sh

@@ -84,26 +84,11 @@ EOF
 }
 
 testInstall() {
-    luet install -y --config $tmpdir/luet.yaml test/b@1.0
+    luet install -y --config $tmpdir/luet.yaml test/b@1.0 test/a@1.0 test/c@1.0
     installst=$?
     assertEquals 'install test successfully' "$installst" "0"
     assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/test5' ]"
-
-    luet install -y --config $tmpdir/luet.yaml test/a@1.0
     assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
-    installst=$?
-    assertEquals 'install test successfully' "$installst" "0"
-
-    luet install -y --config $tmpdir/luet.yaml test/a@1.1
-    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
-    installst=$?
-    assertEquals 'install test successfully' "$installst" "0"
-    assertTrue 'package keeps old A' "[ -e '$tmpdir/testrootfs/testaa' ]"
-    assertTrue 'package new A was not installed' "[ ! -e '$tmpdir/testrootfs/testlatest' ]"
-
-    luet install -y --config $tmpdir/luet.yaml test/c@1.0
-    installst=$?
-    assertEquals 'install test successfully' "$installst" "0"
     assertTrue 'package installed C' "[ -e '$tmpdir/testrootfs/c' ]"
 }
 

tests/integration/11_upgrade_universe.sh

@@ -85,26 +85,11 @@ EOF
 }
 
 testInstall() {
-    luet install -y --config $tmpdir/luet.yaml test/b@1.0
+    luet install -y --config $tmpdir/luet.yaml test/b@1.0 test/a@1.0 test/c@1.0
     installst=$?
     assertEquals 'install test successfully' "$installst" "0"
     assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/test5' ]"
-
-    luet install -y --config $tmpdir/luet.yaml test/a@1.0
     assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
-    installst=$?
-    assertEquals 'install test successfully' "$installst" "0"
-
-    luet install -y --config $tmpdir/luet.yaml test/a@1.1
-    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
-    installst=$?
-    assertEquals 'install test successfully' "$installst" "0"
-    assertTrue 'package keeps old A' "[ -e '$tmpdir/testrootfs/testaa' ]"
-    assertTrue 'package new A was not installed' "[ ! -e '$tmpdir/testrootfs/testlatest' ]"
-
-    luet install -y --config $tmpdir/luet.yaml test/c@1.0
-    installst=$?
-    assertEquals 'install test successfully' "$installst" "0"
     assertTrue 'package installed C' "[ -e '$tmpdir/testrootfs/c' ]"
 }
 

tests/integration/16_perms.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testbuild
+    luet build -d --tree "$ROOT_DIR/tests/fixtures/perms" --same-owner=true --destination $tmpdir/testbuild --compression gzip --full
+    buildst=$?
+    assertTrue 'create package perms 0.1' "[ -e '$tmpdir/testbuild/perms-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/perms" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    $ROOT_DIR/tests/integration/bin/luet install -y --config $tmpdir/luet.yaml test/perms@0.1 test/perms2@0.1
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+   
+    assertTrue 'package installed perms baz' "[ -d '$tmpdir/testrootfs/foo/baz' ]"
+    assertTrue 'package installed perms bar' "[ -d '$tmpdir/testrootfs/foo/bar' ]"
+
+    assertContains 'perms1' "$(stat -c %u:%g $tmpdir/testrootfs/foo/baz)" "100:100"
+    assertContains 'perms2' "$(stat -c %u:%g $tmpdir/testrootfs/foo/bar)" "100:100"
+    assertContains 'perms11' "$(stat -c %u:%g $tmpdir/testrootfs/foo/baz/.keep)" "101:101"
+    assertContains 'perms22' "$(stat -c %u:%g $tmpdir/testrootfs/foo/bar/.keep)" "101:101"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/26_owners.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+  rm -rf "$tmpdir"
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/owners" --destination $tmpdir/testbuild --compression gzip test/unpack test/delta
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package unpack' "[ -e '$tmpdir/testbuild/unpack-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package delta' "[ -e '$tmpdir/testbuild/delta-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/owners" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    luet install -y --config $tmpdir/luet.yaml test/unpack test/delta
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    fileUID=$(stat -c "%u" $tmpdir/testrootfs/foo)
+    fileGID=$(stat -c "%g" $tmpdir/testrootfs/foo)
+    filePerms=$(stat -c "%a" $tmpdir/testrootfs/foo)
+    assertEquals 'UID on /foo matches' "1000" "$fileUID"
+    assertEquals 'GID on /foo matches' "1001" "$fileGID"
+    assertEquals 'bits on /foo matches' "500" "$filePerms"
+
+    fileUID=$(stat -c "%u" $tmpdir/testrootfs/bar)
+    fileGID=$(stat -c "%g" $tmpdir/testrootfs/bar)
+    filePerms=$(stat -c "%a" $tmpdir/testrootfs/bar)
+    assertEquals 'UID on /bar matches' "1000" "$fileUID"
+    assertEquals 'GID on /bar matches' "1001" "$fileGID"
+    assertEquals 'bits on /bar matches' "600" "$filePerms"
+}
+
+testCleanup() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'cleanup test successfully' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2

tests/integration/27_buildwithnotree.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_engine: "memory"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testBuildWithNoTree() {
+    mkdir $tmpdir/testbuild2
+    mkdir $tmpdir/emptytree
+    luet build --from-repositories --tree $tmpdir/emptytree --config $tmpdir/luet.yaml test/c --destination $tmpdir/testbuild2
+    buildst=$?
+    assertEquals 'build test successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild2/c-test-1.0.package.tar' ]"
+}
+
+testRepo2() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild2/repository.yaml' ]"
+    luet create-repo --config $tmpdir/luet.yaml --from-repositories --tree $tmpdir/emptytree \
+    --output $tmpdir/testbuild2 \
+    --packages $tmpdir/testbuild2 \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild2/repository.yaml' ]"
+}
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package cleaned' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+testInstall2() {
+
+    cat <<EOF > $tmpdir/luet2.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_engine: "memory"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild2"
+EOF
+    luet install -y --config $tmpdir/luet2.yaml --system-target $tmpdir/foo test/c
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'db not created' "[ ! -e '$tmpdir/foo/var/cache/luet/luet.db' ]"
+    assertTrue 'package installed' "[ -e '$tmpdir/foo/c' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/27_buildwithnotree_http.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+TEST_PORT="${TEST_PORT:-9090}"
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    kill '%1' || true
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet serve-repo --dir $tmpdir/testbuild --port $TEST_PORT &
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_engine: "memory"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "http"
+     enable: true
+     urls:
+       - "http://127.0.0.1:$TEST_PORT"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testBuildWithNoTree() {
+    mkdir $tmpdir/testbuild2
+    mkdir $tmpdir/emptytree
+    luet build --from-repositories --tree $tmpdir/emptytree --config $tmpdir/luet.yaml test/c --destination $tmpdir/testbuild2
+    buildst=$?
+    assertEquals 'build test successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild2/c-test-1.0.package.tar' ]"
+}
+
+testRepo2() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild2/repository.yaml' ]"
+    luet create-repo --config $tmpdir/luet.yaml --from-repositories --tree $tmpdir/emptytree \
+    --output $tmpdir/testbuild2 \
+    --packages $tmpdir/testbuild2 \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild2/repository.yaml' ]"
+}
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package cleaned' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+testInstall2() {
+
+    cat <<EOF > $tmpdir/luet2.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_engine: "memory"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild2"
+EOF
+    luet install -y --config $tmpdir/luet2.yaml --system-target $tmpdir/foo test/c
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'db not created' "[ ! -e '$tmpdir/foo/var/cache/luet/luet.db' ]"
+    assertTrue 'package installed' "[ -e '$tmpdir/foo/c' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/27_buildwithnotree_values.sh

@@ -0,0 +1,302 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    cat <<EOF > $tmpdir/default.yaml
+bb: "ttt"
+EOF
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/build_values" --values $tmpdir/default.yaml --destination $tmpdir/testbuild --compression gzip  distro/a distro/b test/foo distro/c
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package B' "[ -e '$tmpdir/testbuild/b-distro-0.3.package.tar.gz' ]"
+    assertTrue 'create package A' "[ -e '$tmpdir/testbuild/a-distro-0.1.package.tar.gz' ]"
+    assertTrue 'create package C' "[ -e '$tmpdir/testbuild/c-distro-0.3.package.tar.gz' ]"
+    assertTrue 'create package foo' "[ -e '$tmpdir/testbuild/foo-test-1.1.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/build_values" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_engine: "memory"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testBuildWithNoTree() {
+    mkdir $tmpdir/testbuild2
+    mkdir $tmpdir/emptytree
+    luet build --from-repositories --tree $tmpdir/emptytree --config $tmpdir/luet.yaml distro/c --destination $tmpdir/testbuild2 --compression gzip distro/a distro/b test/foo distro/c
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package B' "[ -e '$tmpdir/testbuild2/b-distro-0.3.package.tar.gz' ]"
+    assertTrue 'create package A' "[ -e '$tmpdir/testbuild2/a-distro-0.1.package.tar.gz' ]"
+    assertTrue 'create package C' "[ -e '$tmpdir/testbuild2/c-distro-0.3.package.tar.gz' ]"
+    assertTrue 'create package foo' "[ -e '$tmpdir/testbuild2/foo-test-1.1.package.tar.gz' ]"
+}
+
+testRepo2() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild2/repository.yaml' ]"
+    luet create-repo --config $tmpdir/luet.yaml --from-repositories --tree $tmpdir/emptytree \
+    --output $tmpdir/testbuild2 \
+    --packages $tmpdir/testbuild2 \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild2/repository.yaml' ]"
+}
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package cleaned' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+testInstall2() {
+
+    cat <<EOF > $tmpdir/luet2.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild2"
+EOF
+    luet install -y --config $tmpdir/luet2.yaml distro/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/a' ]"
+    # Build time can interpolate on fields which aren't package properties.
+    assertTrue 'extra field on A' "[ -e '$tmpdir/testrootfs/build-extra-baz' ]"
+    assertTrue 'package installed A interpolated with values' "[ -e '$tmpdir/testrootfs/a-ttt' ]"
+    # Finalizers can interpolate only on package field. No extra fields are allowed at this time.
+    assertTrue 'finalizer executed on A' "[ -e '$tmpdir/testrootfs/finalize-a' ]"
+
+    installed=$(luet --config $tmpdir/luet2.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+
+    assertContains 'contains distro/a-0.1' "$installed" 'distro/a-0.1'
+
+    luet uninstall -y --config $tmpdir/luet2.yaml distro/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    # We do the same check for the others
+    luet install -y --config $tmpdir/luet2.yaml distro/b
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/b' ]"
+    assertTrue 'package installed B interpolated with values' "[ -e '$tmpdir/testrootfs/b-ttt' ]"
+    assertTrue 'extra field on B' "[ -e '$tmpdir/testrootfs/build-extra-f' ]"
+    assertTrue 'finalizer executed on B' "[ -e '$tmpdir/testrootfs/finalize-b' ]"
+
+    installed=$(luet --config $tmpdir/luet2.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+
+    assertContains 'contains distro/b-0.3' "$installed" 'distro/b-0.3'
+
+    luet uninstall -y --config $tmpdir/luet2.yaml distro/b
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    luet install -y --config $tmpdir/luet2.yaml distro/c
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed C' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'extra field on C' "[ -e '$tmpdir/testrootfs/build-extra-bar' ]"
+    assertTrue 'package installed C interpolated with values' "[ -e '$tmpdir/testrootfs/c-ttt' ]"
+    assertTrue 'finalizer executed on C' "[ -e '$tmpdir/testrootfs/finalize-c' ]"
+
+    installed=$(luet --config $tmpdir/luet2.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+
+    assertContains 'contains distro/c-0.3' "$installed" 'distro/c-0.3'
+
+    luet uninstall -y --config $tmpdir/luet2.yaml distro/c
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    luet install -y --config $tmpdir/luet2.yaml test/foo
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed foo' "[ -e '$tmpdir/testrootfs/foo' ]"
+    assertTrue 'package installed foo interpolated with values' "[ -e '$tmpdir/testrootfs/foo-ttt' ]"
+}
+
+
+testBuildWithNoTree3() {
+    cat <<EOF > $tmpdir/default.yaml
+bb: "newinterpolation"
+foo: "sq"
+EOF
+    mkdir $tmpdir/testbuild3
+    mkdir $tmpdir/emptytree
+    luet build --from-repositories --values $tmpdir/default.yaml --tree $tmpdir/emptytree --config $tmpdir/luet.yaml distro/c --destination $tmpdir/testbuild3 --compression gzip distro/a distro/b test/foo
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package B' "[ -e '$tmpdir/testbuild3/b-distro-0.3.package.tar.gz' ]"
+    assertTrue 'create package A' "[ -e '$tmpdir/testbuild3/a-distro-0.1.package.tar.gz' ]"
+    assertTrue 'create package C' "[ -e '$tmpdir/testbuild3/c-distro-0.3.package.tar.gz' ]"
+    assertTrue 'create package foo' "[ -e '$tmpdir/testbuild3/foo-test-1.1.package.tar.gz' ]"
+}
+
+testRepo3() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild3/repository.yaml' ]"
+    luet create-repo --config $tmpdir/luet.yaml --from-repositories --tree $tmpdir/emptytree \
+    --output $tmpdir/testbuild3 \
+    --packages $tmpdir/testbuild3 \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild3/repository.yaml' ]"
+}
+
+testInstall3() {
+    mkdir $tmpdir/testrootfs3
+
+    cat <<EOF > $tmpdir/luet2.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs3
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild3"
+EOF
+    luet install -y --config $tmpdir/luet2.yaml distro/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs3/a' ]"
+    # Build time can interpolate on fields which aren't package properties.
+    assertTrue 'extra field on A' "[ -e '$tmpdir/testrootfs3/build-extra-sq' ]"
+    assertTrue 'package installed A interpolated with values' "[ -e '$tmpdir/testrootfs3/a-newinterpolation' ]"
+    # Finalizers can interpolate only on package field. No extra fields are allowed at this time.
+    assertTrue 'finalizer executed on A' "[ -e '$tmpdir/testrootfs3/finalize-a' ]"
+    installed=$(luet --config $tmpdir/luet2.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+
+    assertContains 'contains distro/a-0.1' "$installed" 'distro/a-0.1'
+
+    luet uninstall -y --config $tmpdir/luet2.yaml distro/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    # We do the same check for the others
+    luet install -y --config $tmpdir/luet2.yaml distro/b
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs3/b' ]"
+    assertTrue 'package installed B interpolated with values' "[ -e '$tmpdir/testrootfs3/b-newinterpolation' ]"
+    assertTrue 'extra field on B' "[ -e '$tmpdir/testrootfs3/build-extra-sq' ]"
+    assertTrue 'finalizer executed on B' "[ -e '$tmpdir/testrootfs3/finalize-b' ]"
+    installed=$(luet --config $tmpdir/luet2.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+
+    assertContains 'contains distro/b-0.3' "$installed" 'distro/b-0.3'
+
+    luet uninstall -y --config $tmpdir/luet2.yaml distro/b
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    luet install -y --config $tmpdir/luet2.yaml distro/c
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed C' "[ -e '$tmpdir/testrootfs3/c' ]"
+    assertTrue 'extra field on C' "[ -e '$tmpdir/testrootfs3/build-extra-sq' ]"
+    assertTrue 'package installed C interpolated with values' "[ -e '$tmpdir/testrootfs3/c-newinterpolation' ]"
+    assertTrue 'finalizer executed on C' "[ -e '$tmpdir/testrootfs3/finalize-c' ]"
+
+    installed=$(luet --config $tmpdir/luet2.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+
+    assertContains 'contains distro/c-0.3' "$installed" 'distro/c-0.3'
+
+    luet uninstall -y --config $tmpdir/luet2.yaml distro/c
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    luet install -y --config $tmpdir/luet2.yaml test/foo
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed foo' "[ -e '$tmpdir/testrootfs3/foo' ]"
+    assertTrue 'package installed foo interpolated with values' "[ -e '$tmpdir/testrootfs3/foo-newinterpolation' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/28_nobuildtreedocker.sh

@@ -0,0 +1,158 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+  export tmpdir="$(mktemp -d)"
+  docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testConfig() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "${TEST_DOCKER_IMAGE}"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testBuild() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    mkdir $tmpdir/testbuild
+    mkdir $tmpdir/empty
+    build_output=$(luet build --pull --tree "$tmpdir/empty" \
+    --config $tmpdir/luet.yaml --concurrency 1 \
+    --from-repositories --destination $tmpdir/testbuild --compression zstd test/c@1.0 test/z test/interpolated)
+    buildst=$?
+    echo "$build_output"
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
+    assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
+    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:d620e573c81eab36a9dc5cc314e80fd7b6e04aeff26127de4225bf24fe1f8e71"
+    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:b0f34b0d2d271f0f2619324476b2857b3b39ca895bddc2474a741f3c8c1acbbc"
+    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:c1f11f48113cd71d8795a06c7b49e1558bd7211d2aa88f5d79a3334f0393c64d"
+}
+
+testRepo() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet create-repo \
+    --output "${TEST_DOCKER_IMAGE}-2" \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --tree-compression zstd \
+    --tree-filename foo.tar \
+    --tree "$tmpdir/empty" --config $tmpdir/luet.yaml --from-repositories \
+    --meta-filename repository.meta.tar \
+    --meta-compression zstd \
+    --type docker --push-images --force-push --debug
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+}
+
+testConfigClient() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    cat <<EOF > $tmpdir/luet-client.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "${TEST_DOCKER_IMAGE}-2"
+EOF
+    luet config --config $tmpdir/luet-client.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet install -y --config $tmpdir/luet-client.yaml test/c@1.0 test/z test/interpolated
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package Z installed' "[ -e '$tmpdir/testrootfs/z' ]"
+    assertTrue 'package interpolated installed' "[ -e '$tmpdir/testrootfs/interpolated-baz-bar' ]"
+}
+
+testReInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    output=$(luet install -y --config $tmpdir/luet-client.yaml  test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertContains 'contains warning' "$output" 'No packages to install'
+}
+
+testUnInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet uninstall -y --config $tmpdir/luet-client.yaml test/c@1.0
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+testInstallAgain() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    output=$(luet install -y --config $tmpdir/luet-client.yaml test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'No packages to install'
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zst' ]"
+}
+
+testCleanup() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet cleanup --config $tmpdir/luet-client.yaml
+    installst=$?
+    assertEquals 'cleanup test successfully' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/29_nobuildtreeinheritvalues.sh

@@ -0,0 +1,162 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testConfig() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "${TEST_DOCKER_IMAGE}"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testBuild() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+    cat <<EOF > $tmpdir/default.yaml
+extra: "an"
+EOF
+
+    mkdir $tmpdir/testbuild
+    mkdir $tmpdir/empty
+
+    # With --rebuild, the package gets ignored
+    build_output=$(luet build --pull --tree "$tmpdir/empty" \
+    --config $tmpdir/luet.yaml --values $tmpdir/default.yaml --concurrency 1 \
+    --from-repositories --destination $tmpdir/testbuild --compression zstd test/c@1.0 test/z test/interpolated)
+    buildst=$?
+    echo "$build_output"
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
+    assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
+    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Images available for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:c1f11f48113cd71d8795a06c7b49e1558bd7211d2aa88f5d79a3334f0393c64d"
+}
+
+testRepo() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet create-repo \
+    --output "${TEST_DOCKER_IMAGE}-2" \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --tree-compression zstd \
+    --tree-filename foo.tar \
+    --tree "$tmpdir/empty" --config $tmpdir/luet.yaml --from-repositories \
+    --meta-filename repository.meta.tar \
+    --meta-compression zstd \
+    --type docker --push-images --force-push --debug
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+}
+
+testConfigClient() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    cat <<EOF > $tmpdir/luet-client.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "${TEST_DOCKER_IMAGE}-2"
+EOF
+    luet config --config $tmpdir/luet-client.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet install -y --config $tmpdir/luet-client.yaml test/c@1.0 test/z test/interpolated
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package Z installed' "[ -e '$tmpdir/testrootfs/z' ]"
+    ls -liah $tmpdir/testrootfs/
+    assertTrue 'package interpolated installed' "[ -e '$tmpdir/testrootfs/interpolated-baz-bar' ]"
+}
+
+testReInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    output=$(luet install -y --config $tmpdir/luet-client.yaml  test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertContains 'contains warning' "$output" 'No packages to install'
+}
+
+testUnInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet uninstall -y --config $tmpdir/luet-client.yaml test/c@1.0
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+testInstallAgain() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    output=$(luet install -y --config $tmpdir/luet-client.yaml test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'No packages to install'
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zst' ]"
+}
+
+testCleanup() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet cleanup --config $tmpdir/luet-client.yaml
+    installst=$?
+    assertEquals 'cleanup test successfully' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/29_nobuildtreeinheritvalues_noignored.sh

@@ -0,0 +1,162 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testConfig() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "${TEST_DOCKER_IMAGE}"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testBuild() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+    cat <<EOF > $tmpdir/default.yaml
+extra: "an"
+EOF
+
+    mkdir $tmpdir/testbuild
+    mkdir $tmpdir/empty
+
+    # With --rebuild, the package gets rebuild with the values applied
+    build_output=$(luet build --pull --rebuild --tree "$tmpdir/empty" \
+    --config $tmpdir/luet.yaml --values $tmpdir/default.yaml --concurrency 1 \
+    --from-repositories --destination $tmpdir/testbuild --compression zstd test/c@1.0 test/z test/interpolated)
+    buildst=$?
+    echo "$build_output"
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
+    assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
+    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Downloading image quay.io/mocaccinoos/integration-test-cache:6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"
+}
+
+testRepo() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet create-repo \
+    --output "${TEST_DOCKER_IMAGE}-2" \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --tree-compression zstd \
+    --tree-filename foo.tar \
+    --tree "$tmpdir/empty" --config $tmpdir/luet.yaml --from-repositories \
+    --meta-filename repository.meta.tar \
+    --meta-compression zstd \
+    --type docker --push-images --force-push --debug
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+}
+
+testConfigClient() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    cat <<EOF > $tmpdir/luet-client.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "${TEST_DOCKER_IMAGE}-2"
+EOF
+    luet config --config $tmpdir/luet-client.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet install -y --config $tmpdir/luet-client.yaml test/c@1.0 test/z test/interpolated
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package Z installed' "[ -e '$tmpdir/testrootfs/z' ]"
+    ls -liah $tmpdir/testrootfs/
+    assertTrue 'package interpolated installed' "[ -e '$tmpdir/testrootfs/interpolated-baz-an' ]"
+}
+
+testReInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    output=$(luet install -y --config $tmpdir/luet-client.yaml  test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertContains 'contains warning' "$output" 'No packages to install'
+}
+
+testUnInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet uninstall -y --config $tmpdir/luet-client.yaml test/c@1.0
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+testInstallAgain() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    output=$(luet install -y --config $tmpdir/luet-client.yaml test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'No packages to install'
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zst' ]"
+}
+
+testCleanup() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet cleanup --config $tmpdir/luet-client.yaml
+    installst=$?
+    assertEquals 'cleanup test successfully' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/29_nobuildtreeinheritvalues_norebuild.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testConfig() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "${TEST_DOCKER_IMAGE}"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testBuild() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    # This will be ignored, as no --rebuild is supplied
+    cat <<EOF > $tmpdir/default.yaml
+extra: "an"
+EOF
+
+    mkdir $tmpdir/testbuild
+    mkdir $tmpdir/empty
+cp -rf  "$ROOT_DIR/tests/fixtures/docker_repo/interpolated" $tmpdir/empty/
+
+    cat <<EOF > $tmpdir/empty/interpolated/definition.yaml
+category: "test"
+name: "interpolated"
+version: "1.1"
+foo: "bar"
+EOF
+
+    build_output=$(luet build --pull --tree "$tmpdir/empty" \
+    --config $tmpdir/luet.yaml --values $tmpdir/default.yaml --concurrency 1 \
+    --from-repositories --destination $tmpdir/testbuild --compression zstd test/c@1.0 test/z test/interpolated)
+    buildst=$?
+    echo "$build_output"
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
+    assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.1.package.tar.zst' ]"
+}
+
+testRepo() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet create-repo \
+    --output "${TEST_DOCKER_IMAGE}-2" \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --tree-compression zstd \
+    --tree-filename foo.tar \
+    --tree "$tmpdir/empty" --config $tmpdir/luet.yaml --from-repositories \
+    --meta-filename repository.meta.tar \
+    --meta-compression zstd \
+    --type docker --push-images --force-push --debug
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+}
+
+testConfigClient() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    cat <<EOF > $tmpdir/luet-client.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "${TEST_DOCKER_IMAGE}-2"
+EOF
+    luet config --config $tmpdir/luet-client.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet install -y --config $tmpdir/luet-client.yaml test/c@1.0 test/z test/interpolated
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package Z installed' "[ -e '$tmpdir/testrootfs/z' ]"
+    ls -liah $tmpdir/testrootfs/
+    assertTrue 'package interpolated installed' "[ -e '$tmpdir/testrootfs/interpolated-bar-an' ]"
+}
+
+testReInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    output=$(luet install -y --config $tmpdir/luet-client.yaml  test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertContains 'contains warning' "$output" 'No packages to install'
+}
+
+testUnInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet uninstall -y --config $tmpdir/luet-client.yaml test/c@1.0
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+testInstallAgain() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    output=$(luet install -y --config $tmpdir/luet-client.yaml test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'No packages to install'
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zst' ]"
+}
+
+testCleanup() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet cleanup --config $tmpdir/luet-client.yaml
+    installst=$?
+    assertEquals 'cleanup test successfully' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

vendor/github.com/Sabayon/pkgs-checker/pkg/gentoo/pkg.go

@@ -1,6 +1,6 @@
 /*
 
-Copyright (C) 2017-2019  Daniele Rondina <geaaru@sabayonlinux.org>
+Copyright (C) 2017-2021  Daniele Rondina <geaaru@sabayonlinux.org>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+	"sort"
 	"strings"
 
 	version "github.com/hashicorp/go-version"
@@ -157,6 +158,15 @@ func (p *GentooPackage) GetPackageName() (ans string) {
 	return
 }
 
+func (p *GentooPackage) GetPackageNameWithSlot() (ans string) {
+	if p.Slot != "0" {
+		ans = fmt.Sprintf("%s:%s", p.GetPackageName(), p.Slot)
+	} else {
+		ans = p.GetPackageName()
+	}
+	return
+}
+
 func (p *GentooPackage) GetP() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.GetPV())
 }
@@ -169,6 +179,11 @@ func (p *GentooPackage) GetPV() string {
 	return fmt.Sprintf("%s", p.Version)
 }
 
+func (p *GentooPackage) GetPackageNameWithCond() (ans string) {
+	ans = fmt.Sprintf("%s%s", p.Condition.String(), p.GetPackageName())
+	return
+}
+
 func (p *GentooPackage) GetPVR() (ans string) {
 	if p.VersionSuffix != "" {
 		ans = fmt.Sprintf("%s%s", p.Version, p.VersionSuffix)
@@ -182,6 +197,201 @@ func (p *GentooPackage) GetPF() string {
 	return fmt.Sprintf("%s-%s", p.GetPN(), p.GetPVR())
 }
 
+func (p *GentooPackage) getVersions(i *GentooPackage) (*version.Version, *version.Version, error) {
+	var v1 *version.Version = nil
+	var v2 *version.Version = nil
+	var err error
+
+	if p.Category != i.Category {
+		return v1, v2, errors.New(
+			fmt.Sprintf("Wrong category for package %s", i.Name))
+	}
+
+	if p.Name != i.Name {
+		return v1, v2, errors.New(
+			fmt.Sprintf("Wrong name for package %s", i.Name))
+	}
+
+	if p.Version == "" {
+		return v1, v2, errors.New(
+			fmt.Sprintf("Package without version. I can't compare versions."))
+	}
+
+	if i.Version == "" {
+		return v1, v2, errors.New(
+			fmt.Sprintf("Package supply without version. I can't compare versions."))
+	}
+
+	v1s := p.Version
+	v2s := i.Version
+
+	if p.VersionBuild != "" {
+		v1s = p.Version + "+" + p.VersionBuild
+	}
+	v1, err = version.NewVersion(v1s)
+	if err != nil {
+		return nil, nil, err
+	}
+	if i.VersionBuild != "" {
+		v2s = i.Version + "+" + i.VersionBuild
+	}
+	v2, err = version.NewVersion(v2s)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return v1, v2, nil
+}
+
+func (p *GentooPackage) orderDifferentPkgs(i *GentooPackage, mode int) bool {
+	if p.Category != i.Category {
+		if mode == 0 {
+			return p.Category < i.Category
+		}
+		return p.Category > i.Category
+	}
+	if mode == 0 {
+		return p.Name < i.Name
+	}
+	return p.Name > i.Name
+}
+
+func (p *GentooPackage) GreaterThan(i *GentooPackage) (bool, error) {
+	var ans bool
+	if p.Category != i.Category || p.Name != i.Name {
+		return p.orderDifferentPkgs(i, 1), nil
+	}
+	v1, v2, err := p.getVersions(i)
+	if err != nil {
+		return false, err
+	}
+
+	if v1.Equal(v2) {
+		// Order suffix and VersionBuild
+		versionsSuffix := []string{
+			p.VersionSuffix + "+" + p.VersionBuild,
+			i.VersionSuffix + "+" + i.VersionBuild,
+		}
+
+		sort.Strings(versionsSuffix)
+		if versionsSuffix[1] == p.VersionSuffix+"+"+p.VersionBuild {
+			ans = true
+		} else {
+			ans = false
+		}
+
+	} else {
+		ans = v1.GreaterThan(v2)
+	}
+	return ans, nil
+}
+
+func (p *GentooPackage) LessThan(i *GentooPackage) (bool, error) {
+	var ans bool
+
+	if p.Category != i.Category || p.Name != i.Name {
+		return p.orderDifferentPkgs(i, 0), nil
+	}
+	v1, v2, err := p.getVersions(i)
+	if err != nil {
+		return false, err
+	}
+
+	if v1.Equal(v2) {
+		// Order suffix and VersionBuild
+		versionsSuffix := []string{
+			p.VersionSuffix + "+" + p.VersionBuild,
+			i.VersionSuffix + "+" + i.VersionBuild,
+		}
+
+		sort.Strings(versionsSuffix)
+		if versionsSuffix[0] == p.VersionSuffix+"+"+p.VersionBuild {
+			ans = true
+		} else {
+			ans = false
+		}
+
+	} else {
+		ans = v1.LessThan(v2)
+	}
+	return ans, nil
+}
+
+func (p *GentooPackage) LessThanOrEqual(i *GentooPackage) (bool, error) {
+	var ans bool
+	if p.Category != i.Category || p.Name != i.Name {
+		return p.orderDifferentPkgs(i, 0), nil
+	}
+	v1, v2, err := p.getVersions(i)
+	if err != nil {
+		return false, err
+	}
+
+	if v1.Equal(v2) {
+		// Order suffix and VersionBuild
+		versionsSuffix := []string{
+			p.VersionSuffix + "+" + p.VersionBuild,
+			i.VersionSuffix + "+" + i.VersionBuild,
+		}
+
+		sort.Strings(versionsSuffix)
+		if versionsSuffix[0] == p.VersionSuffix+"+"+p.VersionBuild {
+			ans = true
+		} else {
+			ans = false
+		}
+
+	} else {
+		ans = v1.LessThanOrEqual(v2)
+	}
+	return ans, nil
+}
+
+func (p *GentooPackage) GreaterThanOrEqual(i *GentooPackage) (bool, error) {
+	var ans bool
+
+	if p.Category != i.Category || p.Name != i.Name {
+		return p.orderDifferentPkgs(i, 1), nil
+	}
+	v1, v2, err := p.getVersions(i)
+	if err != nil {
+		return false, err
+	}
+
+	if v1.Equal(v2) {
+		// Order suffix and VersionBuild
+		versionsSuffix := []string{
+			p.VersionSuffix + "+" + p.VersionBuild,
+			i.VersionSuffix + "+" + i.VersionBuild,
+		}
+
+		sort.Strings(versionsSuffix)
+		if versionsSuffix[1] == p.VersionSuffix+"+"+p.VersionBuild {
+			ans = true
+		} else {
+			ans = false
+		}
+
+	} else {
+		ans = v1.LessThanOrEqual(v2)
+	}
+	return ans, nil
+}
+
+func (p *GentooPackage) Equal(i *GentooPackage) (bool, error) {
+	v1, v2, err := p.getVersions(i)
+	if err != nil {
+		return false, err
+	}
+	ans := v1.Equal(v2)
+
+	if ans && (p.VersionSuffix != i.VersionSuffix || p.VersionBuild != i.VersionBuild) {
+		ans = false
+	}
+
+	return ans, nil
+}
+
 func (p *GentooPackage) Admit(i *GentooPackage) (bool, error) {
 	var ans bool = false
 	var v1 *version.Version = nil
@@ -198,6 +408,11 @@ func (p *GentooPackage) Admit(i *GentooPackage) (bool, error) {
 			fmt.Sprintf("Wrong name for package %s", i.Name))
 	}
 
+	// Check Slot
+	if p.Slot != "" && i.Slot != "" && p.Slot != i.Slot {
+		return false, nil
+	}
+
 	v1s := p.Version
 	v2s := i.Version
 
@@ -358,12 +573,13 @@ func ParsePackageStr(pkg string) (*GentooPackage, error) {
 		"_beta[0-9-a-z]*",
 	)
 
-	words := strings.Split(pkg, "/")
-	if len(words) != 2 {
+	// The slash is used also in slot.
+	if strings.Index(pkg, "/") < 0 {
 		return nil, errors.New(fmt.Sprintf("Invalid package string %s", pkg))
 	}
-	ans.Category = words[0]
-	pkgname := words[1]
+
+	ans.Category = pkg[:strings.Index(pkg, "/")]
+	pkgname := pkg[strings.Index(pkg, "/")+1:]
 
 	// Validate category
 
@@ -412,16 +628,27 @@ func ParsePackageStr(pkg string) (*GentooPackage, error) {
 		}
 	}
 
+	// Check if there are use flags annotation
+	if strings.Index(pkgname, "[") > 0 {
+		useFlags := pkgname[strings.Index(pkgname, "[")+1 : strings.Index(pkgname, "]")]
+		ans.UseFlags = strings.Split(useFlags, ",")
+		p := pkgname[0:strings.Index(pkgname, "[")]
+		if strings.Index(pkgname, "]") < len(pkgname) {
+			p = p + pkgname[strings.Index(pkgname, "]")+1:len(pkgname)]
+		}
+		pkgname = p
+	}
+
 	// Check if has repository
 	if strings.Contains(pkgname, "::") {
-		words = strings.Split(pkgname, "::")
+		words := strings.Split(pkgname, "::")
 		ans.Repository = words[1]
 		pkgname = words[0]
 	}
 
 	// Check if has slot
 	if strings.Contains(pkgname, ":") {
-		words = strings.Split(pkgname, ":")
+		words := strings.Split(pkgname, ":")
 		ans.Slot = words[1]
 		pkgname = words[0]
 	}
@@ -466,3 +693,12 @@ func ParsePackageStr(pkg string) (*GentooPackage, error) {
 
 	return &ans, nil
 }
+
+type GentooPackageSorter []GentooPackage
+
+func (p GentooPackageSorter) Len() int      { return len(p) }
+func (p GentooPackageSorter) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+func (p GentooPackageSorter) Less(i, j int) bool {
+	ans, _ := p[i].LessThan(&p[j])
+	return ans
+}

vendor/github.com/codegangsta/inject/.gitignore

@@ -1,2 +0,0 @@
-inject
-inject.test

vendor/github.com/codegangsta/inject/LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2013 Jeremy Saenz
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/codegangsta/inject/README.md

@@ -1,92 +0,0 @@
-# inject
---
-    import "github.com/codegangsta/inject"
-
-Package inject provides utilities for mapping and injecting dependencies in
-various ways.
-
-Language Translations:
-* [简体中文](translations/README_zh_cn.md)
-
-## Usage
-
-#### func  InterfaceOf
-
-```go
-func InterfaceOf(value interface{}) reflect.Type
-```
-InterfaceOf dereferences a pointer to an Interface type. It panics if value is
-not an pointer to an interface.
-
-#### type Applicator
-
-```go
-type Applicator interface {
-	// Maps dependencies in the Type map to each field in the struct
-	// that is tagged with 'inject'. Returns an error if the injection
-	// fails.
-	Apply(interface{}) error
-}
-```
-
-Applicator represents an interface for mapping dependencies to a struct.
-
-#### type Injector
-
-```go
-type Injector interface {
-	Applicator
-	Invoker
-	TypeMapper
-	// SetParent sets the parent of the injector. If the injector cannot find a
-	// dependency in its Type map it will check its parent before returning an
-	// error.
-	SetParent(Injector)
-}
-```
-
-Injector represents an interface for mapping and injecting dependencies into
-structs and function arguments.
-
-#### func  New
-
-```go
-func New() Injector
-```
-New returns a new Injector.
-
-#### type Invoker
-
-```go
-type Invoker interface {
-	// Invoke attempts to call the interface{} provided as a function,
-	// providing dependencies for function arguments based on Type. Returns
-	// a slice of reflect.Value representing the returned values of the function.
-	// Returns an error if the injection fails.
-	Invoke(interface{}) ([]reflect.Value, error)
-}
-```
-
-Invoker represents an interface for calling functions via reflection.
-
-#### type TypeMapper
-
-```go
-type TypeMapper interface {
-	// Maps the interface{} value based on its immediate type from reflect.TypeOf.
-	Map(interface{}) TypeMapper
-	// Maps the interface{} value based on the pointer of an Interface provided.
-	// This is really only useful for mapping a value as an interface, as interfaces
-	// cannot at this time be referenced directly without a pointer.
-	MapTo(interface{}, interface{}) TypeMapper
-	// Provides a possibility to directly insert a mapping based on type and value.
-	// This makes it possible to directly map type arguments not possible to instantiate
-	// with reflect like unidirectional channels.
-	Set(reflect.Type, reflect.Value) TypeMapper
-	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
-	// the Type has not been mapped.
-	Get(reflect.Type) reflect.Value
-}
-```
-
-TypeMapper represents an interface for mapping interface{} values based on type.

vendor/github.com/codegangsta/inject/inject.go

@@ -1,187 +0,0 @@
-// Package inject provides utilities for mapping and injecting dependencies in various ways.
-package inject
-
-import (
-	"fmt"
-	"reflect"
-)
-
-// Injector represents an interface for mapping and injecting dependencies into structs
-// and function arguments.
-type Injector interface {
-	Applicator
-	Invoker
-	TypeMapper
-	// SetParent sets the parent of the injector. If the injector cannot find a
-	// dependency in its Type map it will check its parent before returning an
-	// error.
-	SetParent(Injector)
-}
-
-// Applicator represents an interface for mapping dependencies to a struct.
-type Applicator interface {
-	// Maps dependencies in the Type map to each field in the struct
-	// that is tagged with 'inject'. Returns an error if the injection
-	// fails.
-	Apply(interface{}) error
-}
-
-// Invoker represents an interface for calling functions via reflection.
-type Invoker interface {
-	// Invoke attempts to call the interface{} provided as a function,
-	// providing dependencies for function arguments based on Type. Returns
-	// a slice of reflect.Value representing the returned values of the function.
-	// Returns an error if the injection fails.
-	Invoke(interface{}) ([]reflect.Value, error)
-}
-
-// TypeMapper represents an interface for mapping interface{} values based on type.
-type TypeMapper interface {
-	// Maps the interface{} value based on its immediate type from reflect.TypeOf.
-	Map(interface{}) TypeMapper
-	// Maps the interface{} value based on the pointer of an Interface provided.
-	// This is really only useful for mapping a value as an interface, as interfaces
-	// cannot at this time be referenced directly without a pointer.
-	MapTo(interface{}, interface{}) TypeMapper
-	// Provides a possibility to directly insert a mapping based on type and value.
-	// This makes it possible to directly map type arguments not possible to instantiate
-	// with reflect like unidirectional channels.
-	Set(reflect.Type, reflect.Value) TypeMapper
-	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
-	// the Type has not been mapped.
-	Get(reflect.Type) reflect.Value
-}
-
-type injector struct {
-	values map[reflect.Type]reflect.Value
-	parent Injector
-}
-
-// InterfaceOf dereferences a pointer to an Interface type.
-// It panics if value is not an pointer to an interface.
-func InterfaceOf(value interface{}) reflect.Type {
-	t := reflect.TypeOf(value)
-
-	for t.Kind() == reflect.Ptr {
-		t = t.Elem()
-	}
-
-	if t.Kind() != reflect.Interface {
-		panic("Called inject.InterfaceOf with a value that is not a pointer to an interface. (*MyInterface)(nil)")
-	}
-
-	return t
-}
-
-// New returns a new Injector.
-func New() Injector {
-	return &injector{
-		values: make(map[reflect.Type]reflect.Value),
-	}
-}
-
-// Invoke attempts to call the interface{} provided as a function,
-// providing dependencies for function arguments based on Type.
-// Returns a slice of reflect.Value representing the returned values of the function.
-// Returns an error if the injection fails.
-// It panics if f is not a function
-func (inj *injector) Invoke(f interface{}) ([]reflect.Value, error) {
-	t := reflect.TypeOf(f)
-
-	var in = make([]reflect.Value, t.NumIn()) //Panic if t is not kind of Func
-	for i := 0; i < t.NumIn(); i++ {
-		argType := t.In(i)
-		val := inj.Get(argType)
-		if !val.IsValid() {
-			return nil, fmt.Errorf("Value not found for type %v", argType)
-		}
-
-		in[i] = val
-	}
-
-	return reflect.ValueOf(f).Call(in), nil
-}
-
-// Maps dependencies in the Type map to each field in the struct
-// that is tagged with 'inject'.
-// Returns an error if the injection fails.
-func (inj *injector) Apply(val interface{}) error {
-	v := reflect.ValueOf(val)
-
-	for v.Kind() == reflect.Ptr {
-		v = v.Elem()
-	}
-
-	if v.Kind() != reflect.Struct {
-		return nil // Should not panic here ?
-	}
-
-	t := v.Type()
-
-	for i := 0; i < v.NumField(); i++ {
-		f := v.Field(i)
-		structField := t.Field(i)
-		if f.CanSet() && (structField.Tag == "inject" || structField.Tag.Get("inject") != "") {
-			ft := f.Type()
-			v := inj.Get(ft)
-			if !v.IsValid() {
-				return fmt.Errorf("Value not found for type %v", ft)
-			}
-
-			f.Set(v)
-		}
-
-	}
-
-	return nil
-}
-
-// Maps the concrete value of val to its dynamic type using reflect.TypeOf,
-// It returns the TypeMapper registered in.
-func (i *injector) Map(val interface{}) TypeMapper {
-	i.values[reflect.TypeOf(val)] = reflect.ValueOf(val)
-	return i
-}
-
-func (i *injector) MapTo(val interface{}, ifacePtr interface{}) TypeMapper {
-	i.values[InterfaceOf(ifacePtr)] = reflect.ValueOf(val)
-	return i
-}
-
-// Maps the given reflect.Type to the given reflect.Value and returns
-// the Typemapper the mapping has been registered in.
-func (i *injector) Set(typ reflect.Type, val reflect.Value) TypeMapper {
-	i.values[typ] = val
-	return i
-}
-
-func (i *injector) Get(t reflect.Type) reflect.Value {
-	val := i.values[t]
-
-	if val.IsValid() {
-		return val
-	}
-
-	// no concrete types found, try to find implementors
-	// if t is an interface
-	if t.Kind() == reflect.Interface {
-		for k, v := range i.values {
-			if k.Implements(t) {
-				val = v
-				break
-			}
-		}
-	}
-
-	// Still no type found, try to look it up on the parent
-	if !val.IsValid() && i.parent != nil {
-		val = i.parent.Get(t)
-	}
-
-	return val
-
-}
-
-func (i *injector) SetParent(parent Injector) {
-	i.parent = parent
-}

vendor/github.com/codegangsta/inject/update_readme.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-go get github.com/robertkrimen/godocdown/godocdown
-godocdown > README.md

vendor/github.com/google/renameio/.golangci.yml

@@ -0,0 +1,5 @@
+linters:
+  disable:
+  - errcheck
+  enable:
+  - gofmt

vendor/github.com/google/renameio/CONTRIBUTING.md

@@ -0,0 +1,28 @@
+# How to Contribute
+
+We'd love to accept your patches and contributions to this project. There are
+just a few small guidelines you need to follow.
+
+## Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution;
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to <https://cla.developers.google.com/> to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
+## Code reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.
+
+## Community Guidelines
+
+This project follows [Google's Open Source Community
+Guidelines](https://opensource.google.com/conduct/).

vendor/github.com/google/renameio/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.