Prepare for 0.5.1 tag

Add resolvers to ehnance solver's heuristic

.gitignore

@@ -1,2 +1,4 @@
 *.swp
 luet
+tests/integration/shunit2
+tests/integration/bin

.travis.yml

@@ -9,7 +9,7 @@ before_install:
   - make deps
   - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && mkdir -p $HOME/bin && export PATH=$PATH:$HOME/bin && mv container-diff-linux-amd64 $HOME/bin/container-diff
 script:
-  - make multiarch-build test
+  - make multiarch-build test test-integration
 after_success:
   - make coverage
   - bash <(curl -s https://codecov.io/bash)

Makefile

@@ -21,6 +21,10 @@ test:
 	GO111MODULE=off go get github.com/onsi/gomega/...
 	ginkgo -race -r ./...
 
+.PHONY: test-integration
+test-integration:
+	tests/integration/run.sh
+
 .PHONY: coverage
 coverage:
 	go test ./... -race -coverprofile=coverage.txt -covermode=atomic
@@ -36,6 +40,8 @@ help:
 .PHONY: clean
 clean:
 	rm -rf release/
+	rm -rf tests/integration/shunit2
+	rm -rf tests/integration/bin
 
 .PHONY: deps
 deps:

README.md

@@ -4,18 +4,60 @@
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
 [![codecov](https://codecov.io/gh/mudler/luet/branch/master/graph/badge.svg)](https://codecov.io/gh/mudler/luet)
 
-Luet is a Package Manager based off from containers - it uses Docker (and other tech) to sandbox your builds and generate packages from them. It has no dependencies and it is well suitable for "from scratch" environments.
+Luet is a multi-platform Package Manager based off from containers - it uses Docker (and other tech) to sandbox your builds and generate packages from them. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
+
+It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/specfile/) in YAML notation to define both packages and rootfs. As it is based on containers, it can be used to build seed stages for Linux From Scratch installations and it can build and track updates for those systems.
+
+It is written entirely in Golang and where used as package manager, it can run in from scratch environment, with zero dependencies.
 
 ## In a glance
 
 - Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.
 - It builds, installs, uninstalls and perform upgrades on machines
-- Installer doesn't depend on anything
+- Installer doesn't depend on anything ( 0 dep installer !), statically built
 - Support for packages as "layers"
 - It uses SAT solving techniques to solve the deptree ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
 
+## Install
+
+To install luet, you can grab a release on the [Release page](https://github.com/mudler/luet/releases) or compile it in your machine (requires Golang installed):
+
+    $ git clone https://github.com/mudler/luet.git
+    $ cd luet
+    $ make build
+
 ## Status
 
 Luet is not feature-complete yet, it can build, install/uninstall/upgrade packages - but it doesn't support yet all the features you would normally expect from a Package Manager nowadays.
 
-Check out the [Wiki](https://github.com/mudler/luet/wiki) for more informations.
+# Dependency solving
+
+Luet uses SAT and Reinforcement learning engine for dependency solving.
+It encodes the package requirements into a SAT problem, using gophersat to solve the dependency tree and give a concrete model as result.
+
+## SAT encoding
+
+Each package and its constraints are encoded and built around [OPIUM](https://ranjitjhala.github.io/static/opium.pdf). Additionally, Luet treats
+also selectors seamlessly while building the model, adding *ALO* ( *At least one* ) and *AMO* ( *At most one* ) rules to guarantee coherence within the installed system.
+
+## Reinforcement learning
+
+Luet also implements a small and portable qlearning agent that will try to solve conflict on your behalf
+when they arises while trying to validate your queries against the system model.
+
+To leverage it, simply pass ```--solver-type qlearning``` to the subcommands that supports it ( you can check out by invoking ```--help``` ).
+
+## Documentation
+
+[Documentation](https://luet-lab.github.io/docs) is available, or
+run `luet --help`,  any subcommand is documented as well, try e.g.: `luet build --help`.
+
+## Authors
+
+Luet is here thanks to our amazing [contributors](https://github.com/mudler/luet/graphs/contributors)!.
+
+Luet was originally created by Ettore Di Giacinto, mudler@sabayon.org, mudler@gentoo.org.
+
+## License
+
+Luet is distributed under the terms of GPLv3, check out the LICENSE file.

cmd/build.go

@@ -15,17 +15,18 @@
 package cmd
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
-	"regexp"
-	"runtime"
 
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	tree "github.com/mudler/luet/pkg/tree"
 
+	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -35,25 +36,33 @@ var buildCmd = &cobra.Command{
 	Short: "build a package or a tree",
 	Long:  `build packages or trees from luet tree definitions. Packages are in [category]/[name]-[version] form`,
 	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("clean", cmd.Flags().Lookup("clean"))
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
 		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
 		viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged"))
 		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
 		viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
 		viper.BindPFlag("all", cmd.Flags().Lookup("all"))
+		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
+		clean := viper.GetBool("clean")
 		src := viper.GetString("tree")
 		dst := viper.GetString("destination")
-		concurrency := viper.GetInt("concurrency")
+		concurrency := LuetCfg.GetGeneral().Concurrency
 		backendType := viper.GetString("backend")
 		privileged := viper.GetBool("privileged")
 		revdeps := viper.GetBool("revdeps")
 		all := viper.GetBool("all")
 		databaseType := viper.GetString("database")
+		compressionType := viper.GetString("compression")
 
 		compilerSpecs := compiler.NewLuetCompilationspecs()
 		var compilerBackend compiler.CompilerBackend
@@ -88,20 +97,49 @@ var buildCmd = &cobra.Command{
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase())
 
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		opts := compiler.NewDefaultCompilerOptions()
+		opts.SolverOptions = *LuetCfg.GetSolverOptions()
+
+		opts.Clean = clean
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts)
+		luetCompiler.SetConcurrency(concurrency)
+		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
 		if !all {
 			for _, a := range args {
-				decodepackage, err := regexp.Compile(`^([<>]?\~?=?)((([^\/]+)\/)?(?U)(\S+))(-(\d+(\.\d+)*[a-z]?(_(alpha|beta|pre|rc|p)\d*)*(-r\d+)?))?$`)
+				gp, err := _gentoo.ParsePackageStr(a)
 				if err != nil {
-					Fatal("Error: " + err.Error())
+					Fatal("Invalid package string ", a, ": ", err.Error())
 				}
-				packageInfo := decodepackage.FindAllStringSubmatch(a, -1)
 
-				category := packageInfo[0][4]
-				name := packageInfo[0][5]
-				version := packageInfo[0][7]
-				spec, err := luetCompiler.FromPackage(&pkg.DefaultPackage{Name: name, Category: category, Version: version})
+				if gp.Version == "" {
+					gp.Version = "0"
+					gp.Condition = _gentoo.PkgCondGreaterEqual
+				}
+
+				pack := &pkg.DefaultPackage{
+					Name: gp.Name,
+					Version: fmt.Sprintf("%s%s%s",
+						pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+						gp.Version,
+						gp.VersionSuffix,
+					),
+					Category: gp.Category,
+					Uri:      make([]string, 0),
+				}
+				spec, err := luetCompiler.FromPackage(pack)
 				if err != nil {
 					Fatal("Error: " + err.Error())
 				}
@@ -126,10 +164,10 @@ var buildCmd = &cobra.Command{
 		var artifact []compiler.Artifact
 		var errs []error
 		if revdeps {
-			artifact, errs = luetCompiler.CompileWithReverseDeps(concurrency, privileged, compilerSpecs)
+			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
 		} else {
-			artifact, errs = luetCompiler.CompileParallel(concurrency, privileged, compilerSpecs)
+			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
 
 		}
 		if len(errs) != 0 {
@@ -149,14 +187,20 @@ func init() {
 	if err != nil {
 		Fatal(err)
 	}
+	buildCmd.Flags().Bool("clean", true, "Build all packages without considering the packages present in the build directory")
 	buildCmd.Flags().String("tree", path, "Source luet tree")
 	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
-	buildCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
 	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
 	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all packages in the tree")
 	buildCmd.Flags().String("destination", path, "Destination folder")
+	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
+
+	buildCmd.Flags().String("solver-type", "", "Solver strategy")
+	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 
 	RootCmd.AddCommand(buildCmd)
 }

cmd/cleanup.go

@@ -0,0 +1,70 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	config "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+var cleanupCmd = &cobra.Command{
+	Use:   "cleanup",
+	Short: "Clean packages cache.",
+	Long:  `remove downloaded packages tarballs and clean cache directory`,
+	Run: func(cmd *cobra.Command, args []string) {
+		var cleaned int = 0
+
+		// Check if cache dir exists
+		if helpers.Exists(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
+
+			files, err := ioutil.ReadDir(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
+			if err != nil {
+				Fatal("Error on read cachedir ", err.Error())
+			}
+
+			for _, file := range files {
+				if file.IsDir() {
+					continue
+				}
+
+				if config.LuetCfg.GetGeneral().Debug {
+					Info("Removing ", file.Name())
+				}
+
+				err := os.RemoveAll(
+					filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
+				if err != nil {
+					Fatal("Error on removing", file.Name())
+				}
+				cleaned++
+			}
+		}
+
+		Info("Cleaned: ", cleaned, "packages.")
+
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(cleanupCmd)
+}

cmd/config.go

@@ -0,0 +1,59 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+
+	config "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+var configCmd = &cobra.Command{
+	Use:   "config",
+	Short: "Print config",
+	Long:  `Show luet configuration`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println(config.LuetCfg.GetLogging())
+		fmt.Println(config.LuetCfg.GetGeneral())
+		fmt.Println(config.LuetCfg.GetSystem())
+		if len(config.LuetCfg.CacheRepositories) > 0 {
+			fmt.Println("repetitors:")
+			for _, r := range config.LuetCfg.CacheRepositories {
+				fmt.Println("  - ", r.String())
+			}
+		}
+		if len(config.LuetCfg.SystemRepositories) > 0 {
+			fmt.Println("repositories:")
+			for _, r := range config.LuetCfg.SystemRepositories {
+				fmt.Println("  - ", r.String())
+			}
+		}
+
+		if len(config.LuetCfg.RepositoriesConfDir) > 0 {
+			fmt.Println("repos_confdir:")
+			for _, dir := range config.LuetCfg.RepositoriesConfDir {
+				fmt.Println("  - ", dir)
+			}
+		}
+
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(configCmd)
+}

cmd/convert.go

@@ -16,8 +16,8 @@ package cmd
 
 import (
 	"io/ioutil"
-	"runtime"
 
+	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	tree "github.com/mudler/luet/pkg/tree"
@@ -33,13 +33,11 @@ var convertCmd = &cobra.Command{
 	Long:  `Parses external PM and produces a luet parsable tree`,
 	PreRun: func(cmd *cobra.Command, args []string) {
 		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
 		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
 		t := viper.GetString("type")
-		c := viper.GetInt("concurrency")
 		databaseType := viper.GetString("database")
 		var db pkg.PackageDatabase
 
@@ -54,9 +52,15 @@ var convertCmd = &cobra.Command{
 		var builder tree.Parser
 		switch t {
 		case "gentoo":
-			builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c, gentoo.InMemory)
+			builder = gentoo.NewGentooBuilder(
+				&gentoo.SimpleEbuildParser{},
+				LuetCfg.GetGeneral().Concurrency,
+				gentoo.InMemory)
 		default: // dup
-			builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c, gentoo.InMemory)
+			builder = gentoo.NewGentooBuilder(
+				&gentoo.SimpleEbuildParser{},
+				LuetCfg.GetGeneral().Concurrency,
+				gentoo.InMemory)
 		}
 
 		switch databaseType {
@@ -91,7 +95,6 @@ var convertCmd = &cobra.Command{
 
 func init() {
 	convertCmd.Flags().String("type", "gentoo", "source type")
-	convertCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
 	convertCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
 
 	RootCmd.AddCommand(convertCmd)

cmd/create-repo.go

@@ -17,8 +17,9 @@ package cmd
 import (
 	"os"
 
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
-
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -35,23 +36,71 @@ var createrepoCmd = &cobra.Command{
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("output", cmd.Flags().Lookup("output"))
 		viper.BindPFlag("name", cmd.Flags().Lookup("name"))
-		viper.BindPFlag("uri", cmd.Flags().Lookup("uri"))
+		viper.BindPFlag("descr", cmd.Flags().Lookup("descr"))
+		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
 		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
+		viper.BindPFlag("tree-compression", cmd.Flags().Lookup("tree-compression"))
+		viper.BindPFlag("tree-path", cmd.Flags().Lookup("tree-path"))
+		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
+		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
+		var err error
+		var repo installer.Repository
 
 		tree := viper.GetString("tree")
 		dst := viper.GetString("output")
 		packages := viper.GetString("packages")
 		name := viper.GetString("name")
-		uri := viper.GetString("uri")
+		descr := viper.GetString("descr")
+		urls := viper.GetStringSlice("urls")
 		t := viper.GetString("type")
+		reset := viper.GetBool("reset-revision")
+		treetype := viper.GetString("tree-compression")
+		treepath := viper.GetString("tree-path")
+		source_repo := viper.GetString("repo")
+
+		if source_repo != "" {
+			// Search for system repository
+			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
+			if tree == "" {
+				tree = lrepo.TreePath
+			}
+
+			if t == "" {
+				t = lrepo.Type
+			}
+
+			repo, err = installer.GenerateRepository(lrepo.Name,
+				lrepo.Description, t,
+				lrepo.Urls,
+				lrepo.Priority,
+				packages,
+				tree,
+				pkg.NewInMemoryDatabase(false))
+
+		} else {
+			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
+				tree, pkg.NewInMemoryDatabase(false))
+		}
 
-		repo, err := installer.GenerateRepository(name, uri, t, 1, packages, tree, pkg.NewInMemoryDatabase(false))
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
-		err = repo.Write(dst)
+
+		if treetype != "" {
+			repo.SetTreeCompressionType(compiler.CompressionImplementation(treetype))
+		}
+
+		if treepath != "" {
+			repo.SetTreePath(treepath)
+		}
+
+		err = repo.Write(dst, reset)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
@@ -67,8 +116,14 @@ func init() {
 	createrepoCmd.Flags().String("tree", path, "Source luet tree")
 	createrepoCmd.Flags().String("output", path, "Destination folder")
 	createrepoCmd.Flags().String("name", "luet", "Repository name")
-	createrepoCmd.Flags().String("uri", path, "Repository uri")
-	createrepoCmd.Flags().String("type", "local", "Repository type (local)")
+	createrepoCmd.Flags().String("descr", "luet", "Repository description")
+	createrepoCmd.Flags().StringSlice("urls", []string{}, "Repository URLs")
+	createrepoCmd.Flags().String("type", "disk", "Repository type (disk)")
+	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
+	createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
+
+	createrepoCmd.Flags().String("tree-compression", "none", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("tree-path", installer.TREE_TARBALL, "Repository tree filename")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

cmd/install.go

@@ -15,70 +15,93 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
-	"runtime"
 
 	installer "github.com/mudler/luet/pkg/installer"
 
+	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
+	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 var installCmd = &cobra.Command{
 	Use:   "install <pkg1> <pkg2> ...",
 	Short: "Install a package",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("system-dbpath", cmd.Flags().Lookup("system-dbpath"))
-		viper.BindPFlag("system-target", cmd.Flags().Lookup("system-target"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Long: `Install packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
-		c := []*installer.LuetRepository{}
-		err := viper.UnmarshalKey("system-repositories", &c)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-
 		var toInstall []pkg.Package
+		var systemDB pkg.PackageDatabase
 
 		for _, a := range args {
-			decodepackage, err := regexp.Compile(`^([<>]?\~?=?)((([^\/]+)\/)?(?U)(\S+))(-(\d+(\.\d+)*[a-z]?(_(alpha|beta|pre|rc|p)\d*)*(-r\d+)?))?$`)
+			gp, err := _gentoo.ParsePackageStr(a)
 			if err != nil {
-				Fatal("Error: " + err.Error())
+				Fatal("Invalid package string ", a, ": ", err.Error())
 			}
-			packageInfo := decodepackage.FindAllStringSubmatch(a, -1)
 
-			category := packageInfo[0][4]
-			name := packageInfo[0][5]
-			version := packageInfo[0][7]
-			toInstall = append(toInstall, &pkg.DefaultPackage{Name: name, Category: category, Version: version})
+			if gp.Version == "" {
+				gp.Version = "0"
+				gp.Condition = _gentoo.PkgCondGreaterEqual
+			}
 
+			pack := &pkg.DefaultPackage{
+				Name: gp.Name,
+				Version: fmt.Sprintf("%s%s%s",
+					pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+					gp.Version,
+					gp.VersionSuffix,
+				),
+				Category: gp.Category,
+				Uri:      make([]string, 0),
+			}
+			toInstall = append(toInstall, pack)
 		}
 
 		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
-		synced := installer.Repositories{}
-		for _, toSync := range c {
-			s, err := toSync.Sync()
-			if err != nil {
-				Fatal("Error: " + err.Error())
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
 			}
-			synced = append(synced, s)
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
 		}
 
-		inst := installer.NewLuetInstaller(viper.GetInt("concurrency"))
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 
-		inst.Repositories(synced)
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
 
-		os.MkdirAll(viper.GetString("system-dbpath"), os.ModePerm)
-		systemDB := pkg.NewBoltDatabase(filepath.Join(viper.GetString("system-dbpath"), "luet.db"))
-		system := &installer.System{Database: systemDB, Target: viper.GetString("system-target")}
-		err = inst.Install(toInstall, system)
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
+		inst.Repositories(repos)
+
+		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+			systemDB = pkg.NewBoltDatabase(
+				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+		} else {
+			systemDB = pkg.NewInMemoryDatabase(true)
+		}
+		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Install(toInstall, system)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
@@ -92,7 +115,10 @@ func init() {
 	}
 	installCmd.Flags().String("system-dbpath", path, "System db path")
 	installCmd.Flags().String("system-target", path, "System rootpath")
-	installCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
+	installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	installCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/repo.go

@@ -0,0 +1,37 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/repo"
+
+	"github.com/spf13/cobra"
+)
+
+var repoGroupCmd = &cobra.Command{
+	Use:   "repo [command] [OPTIONS]",
+	Short: "Manage repositories",
+}
+
+func init() {
+	RootCmd.AddCommand(repoGroupCmd)
+
+	repoGroupCmd.AddCommand(
+		NewRepoListCommand(),
+		NewRepoUpdateCommand(),
+	)
+}

cmd/repo/list.go

@@ -0,0 +1,101 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_repo
+
+import (
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	. "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+
+	. "github.com/logrusorgru/aurora"
+	"github.com/spf13/cobra"
+)
+
+func NewRepoListCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "list [OPTIONS]",
+		Short: "List of the configured repositories.",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var repoColor, repoText, repoRevision string
+
+			enable, _ := cmd.Flags().GetBool("enabled")
+			quiet, _ := cmd.Flags().GetBool("quiet")
+			repoType, _ := cmd.Flags().GetString("type")
+
+			for _, repo := range LuetCfg.SystemRepositories {
+				if enable && !repo.Enable {
+					continue
+				}
+
+				if repoType != "" && repo.Type != repoType {
+					continue
+				}
+
+				repoRevision = ""
+
+				if quiet {
+					fmt.Println(repo.Name)
+				} else {
+					if repo.Enable {
+						repoColor = Bold(BrightGreen(repo.Name)).String()
+					} else {
+						repoColor = Bold(BrightRed(repo.Name)).String()
+					}
+					if repo.Description != "" {
+						repoText = Yellow(repo.Description).String()
+					} else {
+						repoText = Yellow(repo.Urls[0]).String()
+					}
+
+					repobasedir := LuetCfg.GetSystem().GetRepoDatabaseDirPath(repo.Name)
+					if repo.Cached {
+
+						r := installer.NewSystemRepository(repo)
+						localRepo, _ := r.(*installer.LuetSystemRepository).ReadSpecFile(filepath.Join(repobasedir,
+							installer.REPOSITORY_SPECFILE), false)
+						if localRepo != nil {
+							tsec, _ := strconv.ParseInt(localRepo.GetLastUpdate(), 10, 64)
+							repoRevision = Bold(Red(localRepo.GetRevision())).String() +
+								" - " + Bold(Green(time.Unix(tsec, 0).String())).String()
+						}
+					}
+
+					if repoRevision != "" {
+						fmt.Println(
+							fmt.Sprintf("%s\n  %s\n  Revision %s", repoColor, repoText, repoRevision))
+					} else {
+						fmt.Println(
+							fmt.Sprintf("%s\n  %s", repoColor, repoText))
+					}
+				}
+			}
+		},
+	}
+
+	ans.Flags().Bool("enabled", false, "Show only enable repositories.")
+	ans.Flags().BoolP("quiet", "q", false, "Show only name of the repositories.")
+	ans.Flags().StringP("type", "t", "", "Filter repositories of a specific type")
+
+	return ans
+}

cmd/repo/update.go

@@ -0,0 +1,83 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_repo
+
+import (
+	. "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+func NewRepoUpdateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "update [repo1] [repo2] [OPTIONS]",
+		Short: "Update a specific cached repository or all cached repositories.",
+		Example: `
+# Update all cached repositories:
+$> luet repo update
+
+# Update only repo1 and repo2
+$> luet repo update repo1 repo2
+`,
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			ignore, _ := cmd.Flags().GetBool("ignore-errors")
+			force, _ := cmd.Flags().GetBool("force")
+
+			if len(args) > 0 {
+				for _, rname := range args {
+					repo, err := LuetCfg.GetSystemRepository(rname)
+					if err != nil && !ignore {
+						Fatal(err.Error())
+					} else if err != nil {
+						continue
+					}
+
+					r := installer.NewSystemRepository(*repo)
+					Spinner(32)
+					_, err = r.Sync(force)
+					if err != nil && !ignore {
+						Fatal("Error on sync repository " + rname + ": " + err.Error())
+					}
+					SpinnerStop()
+				}
+
+			} else {
+				for _, repo := range LuetCfg.SystemRepositories {
+					if repo.Cached {
+						r := installer.NewSystemRepository(repo)
+						Spinner(32)
+						_, err := r.Sync(force)
+						if err != nil && !ignore {
+							Fatal("Error on sync repository " + r.GetName() + ": " + err.Error())
+						}
+						SpinnerStop()
+					}
+				}
+			}
+		},
+	}
+
+	ans.Flags().BoolP("ignore-errors", "i", false, "Ignore errors on sync repositories.")
+	ans.Flags().BoolP("force", "f", false, "Force resync.")
+
+	return ans
+}

cmd/root.go

@@ -17,11 +17,15 @@ package cmd
 
 import (
 	"os"
-	"path"
+	"os/user"
 	"path/filepath"
+	"runtime"
+	"strings"
 
 	"github.com/marcsauter/single"
+	config "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
+	repo "github.com/mudler/luet/pkg/repository"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -30,7 +34,10 @@ import (
 var cfgFile string
 var Verbose bool
 
-const LuetCLIVersion = "0.2"
+const (
+	LuetCLIVersion = "0.5.1"
+	LuetEnvPrefix  = "LUET"
+)
 
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
@@ -38,6 +45,44 @@ var RootCmd = &cobra.Command{
 	Short:   "Package manager for the XXth century!",
 	Long:    `Package manager which uses containers to build packages`,
 	Version: LuetCLIVersion,
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		err := LoadConfig(config.LuetCfg)
+		if err != nil {
+			Fatal("failed to load configuration:", err.Error())
+		}
+	},
+}
+
+func LoadConfig(c *config.LuetConfig) error {
+	// If a config file is found, read it in.
+	if err := c.Viper.ReadInConfig(); err != nil {
+		Warning(err)
+	}
+
+	err := c.Viper.Unmarshal(&config.LuetCfg)
+	if err != nil {
+		return err
+	}
+
+	Debug("Using config file:", c.Viper.ConfigFileUsed())
+
+	NewSpinner()
+
+	if c.GetLogging().Path != "" {
+		// Init zap logger
+		err = ZapLogger()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Load repositories
+	err = repo.LoadRepositories(c)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // Execute adds all child commands to the root command sets flags appropriately.
@@ -62,8 +107,26 @@ func Execute() {
 
 func init() {
 	cobra.OnInitialize(initConfig)
-	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
-	RootCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
+	pflags := RootCmd.PersistentFlags()
+	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
+	pflags.BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
+	pflags.Bool("fatal", false, "Enables Warnings to exit")
+
+	u, err := user.Current()
+	if err != nil {
+		Fatal("failed to retrieve user identity:", err.Error())
+	}
+	sameOwner := false
+	if u.Uid == "0" {
+		sameOwner = true
+	}
+	pflags.Bool("same-owner", sameOwner, "Maintain same owner on uncompress.")
+	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
+
+	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
+	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("verbose"))
+	config.LuetCfg.Viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
+	config.LuetCfg.Viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
 }
 
 // initConfig reads in config file and ENV variables if set.
@@ -74,29 +137,23 @@ func initConfig() {
 		Error(err)
 		os.Exit(1)
 	}
+	viper.SetEnvPrefix(LuetEnvPrefix)
 	viper.SetConfigType("yaml")
 	viper.SetConfigName(".luet") // name of config file (without extension)
 	if cfgFile != "" {           // enable ability to specify config file via flag
 		Info(">>> cfgFile: ", cfgFile)
 		viper.SetConfigFile(cfgFile)
-		configDir := path.Dir(cfgFile)
-		if configDir != "." && configDir != dir {
-			viper.AddConfigPath(configDir)
-		}
+	} else {
+		viper.AddConfigPath(dir)
+		viper.AddConfigPath(".")
+		viper.AddConfigPath("$HOME")
+		viper.AddConfigPath("/etc/luet")
 	}
 
-	viper.AddConfigPath(dir)
-	viper.AddConfigPath(".")
-	viper.AddConfigPath("$HOME")
-	viper.AddConfigPath("/etc/luet")
-
 	viper.AutomaticEnv() // read in environment variables that match
 
-	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err == nil {
-		Info("Using config file:", viper.ConfigFileUsed())
-	} else {
-		Warning(err)
-	}
-
+	// Create EnvKey Replacer for handle complex structure
+	replacer := strings.NewReplacer(".", "__")
+	viper.SetEnvKeyReplacer(replacer)
+	viper.SetTypeByDefaultValue(true)
 }

cmd/search.go

@@ -18,15 +18,13 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-	"runtime"
 
+	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
-
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 var searchCmd = &cobra.Command{
@@ -34,43 +32,68 @@ var searchCmd = &cobra.Command{
 	Short: "Search packages",
 	Long:  `Search for installed and available packages`,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("system-dbpath", cmd.Flags().Lookup("system-dbpath"))
-		viper.BindPFlag("system-target", cmd.Flags().Lookup("system-target"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
-		viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		c := []*installer.LuetRepository{}
-		err := viper.UnmarshalKey("system-repositories", &c)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
+		var systemDB pkg.PackageDatabase
 
 		if len(args) != 1 {
 			Fatal("Wrong number of arguments (expected 1)")
 		}
-		installed := viper.GetBool("installed")
+		installed := LuetCfg.Viper.GetBool("installed")
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
 		if !installed {
-			synced := installer.Repositories{}
 
-			for _, toSync := range c {
-				s, err := toSync.Sync()
-				if err != nil {
-					Fatal("Error: " + err.Error())
+			repos := installer.Repositories{}
+			for _, repo := range LuetCfg.SystemRepositories {
+				if !repo.Enable {
+					continue
 				}
-				synced = append(synced, s)
+				r := installer.NewSystemRepository(repo)
+				repos = append(repos, r)
 			}
+
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
+
+			inst.Repositories(repos)
+			synced, err := inst.SyncRepositories(false)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
 			Info("--- Search results: ---")
 
 			matches := synced.Search(args[0])
 			for _, m := range matches {
-				Info(":package:", m.Package.GetCategory(), m.Package.GetName(), m.Package.GetVersion(), "repository:", m.Repo.GetName())
+				Info(":package:", m.Package.GetCategory(), m.Package.GetName(),
+					m.Package.GetVersion(), "repository:", m.Repo.GetName())
 			}
 		} else {
-			os.MkdirAll(viper.GetString("system-dbpath"), os.ModePerm)
-			systemDB := pkg.NewBoltDatabase(filepath.Join(viper.GetString("system-dbpath"), "luet.db"))
-			system := &installer.System{Database: systemDB, Target: viper.GetString("system-target")}
+
+			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+				systemDB = pkg.NewBoltDatabase(
+					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+			} else {
+				systemDB = pkg.NewInMemoryDatabase(true)
+			}
+			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
 			var term = regexp.MustCompile(args[0])
 
 			for _, k := range system.Database.GetPackages() {
@@ -91,7 +114,10 @@ func init() {
 	}
 	searchCmd.Flags().String("system-dbpath", path, "System db path")
 	searchCmd.Flags().String("system-target", path, "System rootpath")
-	searchCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
 	searchCmd.Flags().Bool("installed", false, "Search between system packages")
+	searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	searchCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	searchCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	searchCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	RootCmd.AddCommand(searchCmd)
 }

cmd/serve-repo.go

@@ -0,0 +1,59 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"net/http"
+	"os"
+
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var serverepoCmd = &cobra.Command{
+	Use:   "serve-repo",
+	Short: "Embedded micro-http server",
+	Long:  `Embedded mini http server for serving local repositories`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("dir", cmd.Flags().Lookup("dir"))
+		viper.BindPFlag("address", cmd.Flags().Lookup("address"))
+		viper.BindPFlag("port", cmd.Flags().Lookup("port"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+
+		dir := viper.GetString("dir")
+		port := viper.GetString("port")
+		address := viper.GetString("address")
+
+		http.Handle("/", http.FileServer(http.Dir(dir)))
+
+		Info("Serving ", dir, " on HTTP port: ", port)
+		Fatal(http.ListenAndServe(address+":"+port, nil))
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	serverepoCmd.Flags().String("dir", path, "Packages folder (output from build)")
+	serverepoCmd.Flags().String("port", "9090", "Listening port")
+	serverepoCmd.Flags().String("address", "0.0.0.0", "Listening address")
+
+	RootCmd.AddCommand(serverepoCmd)
+}

cmd/uninstall.go

@@ -15,52 +15,79 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
-	"runtime"
 
+	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
-
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
+	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 var uninstallCmd = &cobra.Command{
-	Use:   "uninstall <pkg>",
-	Short: "Uninstall a package",
+	Use:   "uninstall <pkg> <pkg2> ...",
+	Short: "Uninstall a package or a list of packages",
 	Long:  `Uninstall packages`,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("system-dbpath", cmd.Flags().Lookup("system-dbpath"))
-		viper.BindPFlag("system-target", cmd.Flags().Lookup("system-target"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		if len(args) != 1 {
-			Fatal("Wrong number of args")
-		}
+		var systemDB pkg.PackageDatabase
 
-		a := args[0]
-		decodepackage, err := regexp.Compile(`^([<>]?\~?=?)((([^\/]+)\/)?(?U)(\S+))(-(\d+(\.\d+)*[a-z]?(_(alpha|beta|pre|rc|p)\d*)*(-r\d+)?))?$`)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-		packageInfo := decodepackage.FindAllStringSubmatch(a, -1)
+		for _, a := range args {
+			gp, err := _gentoo.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			if gp.Version == "" {
+				gp.Version = "0"
+				gp.Condition = _gentoo.PkgCondGreaterEqual
+			}
+			pack := &pkg.DefaultPackage{
+				Name: gp.Name,
+				Version: fmt.Sprintf("%s%s%s",
+					pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+					gp.Version,
+					gp.VersionSuffix,
+				),
+				Category: gp.Category,
+				Uri:      make([]string, 0),
+			}
 
-		category := packageInfo[0][4]
-		name := packageInfo[0][5]
-		version := packageInfo[0][7]
+			stype := LuetCfg.Viper.GetString("solver.type")
+			discount := LuetCfg.Viper.GetFloat64("solver.discount")
+			rate := LuetCfg.Viper.GetFloat64("solver.rate")
+			attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 
-		inst := installer.NewLuetInstaller(viper.GetInt("concurrency"))
-		os.MkdirAll(viper.GetString("system-dbpath"), os.ModePerm)
-		systemDB := pkg.NewBoltDatabase(filepath.Join(viper.GetString("system-dbpath"), "luet.db"))
-		system := &installer.System{Database: systemDB, Target: viper.GetString("system-target")}
-		err = inst.Uninstall(&pkg.DefaultPackage{Name: name, Category: category, Version: version}, system)
-		if err != nil {
-			Fatal("Error: " + err.Error())
+			LuetCfg.GetSolverOptions().Type = stype
+			LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+			LuetCfg.GetSolverOptions().Discount = float32(discount)
+			LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
+
+			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+				systemDB = pkg.NewBoltDatabase(
+					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+			} else {
+				systemDB = pkg.NewInMemoryDatabase(true)
+			}
+			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+			err = inst.Uninstall(pack, system)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
 		}
 	},
 }
@@ -72,6 +99,9 @@ func init() {
 	}
 	uninstallCmd.Flags().String("system-dbpath", path, "System db path")
 	uninstallCmd.Flags().String("system-target", path, "System rootpath")
-	uninstallCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
+	uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	uninstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -17,50 +17,66 @@ package cmd
 import (
 	"os"
 	"path/filepath"
-	"runtime"
 
+	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
-
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 var upgradeCmd = &cobra.Command{
 	Use:   "upgrade",
 	Short: "Upgrades the system",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("system-dbpath", cmd.Flags().Lookup("system-dbpath"))
-		viper.BindPFlag("system-target", cmd.Flags().Lookup("system-target"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
+		LuetCfg.Viper.BindPFlag("system.database_path", installCmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", installCmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Long: `Upgrades packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
-		c := []*installer.LuetRepository{}
-		err := viper.UnmarshalKey("system-repositories", &c)
+		var systemDB pkg.PackageDatabase
+
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		Debug("Solver", LuetCfg.GetSolverOptions().String())
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
+		inst.Repositories(repos)
+		_, err := inst.SyncRepositories(false)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
 
-		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
-		synced := installer.Repositories{}
-		for _, toSync := range c {
-			s, err := toSync.Sync()
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-			synced = append(synced, s)
+		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+			systemDB = pkg.NewBoltDatabase(
+				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+		} else {
+			systemDB = pkg.NewInMemoryDatabase(true)
 		}
-
-		inst := installer.NewLuetInstaller(viper.GetInt("concurrency"))
-
-		inst.Repositories(synced)
-
-		os.MkdirAll(viper.GetString("system-dbpath"), os.ModePerm)
-		systemDB := pkg.NewBoltDatabase(filepath.Join(viper.GetString("system-dbpath"), "luet.db"))
-		system := &installer.System{Database: systemDB, Target: viper.GetString("system-target")}
+		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
 		err = inst.Upgrade(system)
 		if err != nil {
 			Fatal("Error: " + err.Error())
@@ -75,7 +91,9 @@ func init() {
 	}
 	upgradeCmd.Flags().String("system-dbpath", path, "System db path")
 	upgradeCmd.Flags().String("system-target", path, "System rootpath")
-	upgradeCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
-
+	upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	RootCmd.AddCommand(upgradeCmd)
 }

contrib/config/luet.yaml

@@ -0,0 +1,128 @@
+# Luet Configuration File
+#
+# ---------------------------------------------
+# Logging configuration section:
+# ---------------------------------------------
+# logging:
+#   Leave empty to skip logging to file.
+#   path: ""
+#
+#   Set logging level: error|warning|info|debug
+#   level: "info"
+#
+#   Enable JSON log format instead of console mode.
+#   json_format: false.
+#
+# ---------------------------------------------
+# General configuration section:
+# ---------------------------------------------
+# general:
+#   Define max concurrency processes. Default is based of arch: runtime.NumCPU()
+#   concurrency: 1
+#
+#   Enable Debug. If debug is active spinner is disabled.
+#   debug: false
+#
+#   Show output of build execution (docker, img, etc.)
+#   show_build_output: false
+#
+#   Define spinner ms
+#   spinner_ms: 200
+#
+#   Define spinner charset. See https://github.com/briandowns/spinner
+#   spinner_charset: 22
+#
+#   Enable warnings to exit
+#   fatal_warnings: false
+#
+#   Try extracting tree/packages with the same ownership as exists in the archive (default for superuser).
+#   same_owner: false
+#
+# ---------------------------------------------
+# System configuration section:
+# ---------------------------------------------
+# system:
+#
+#   Rootfs path of the luet system. Default is /.
+#   A specific path could be used for test installation to
+#   a chroot environment.
+#   rootfs: "/"
+#
+#   Choice database engine used for luet database.
+#   Supported values: boltdb|memory
+#   database_engine: boltdb
+#
+#   Database path directory where store luet database.
+#   The path is append to rootfs option path.
+#   database_path: "/var/cache/luet"
+#
+# ---------------------------------------------
+# Repositories configurations directories.
+# ---------------------------------------------
+# Define the list of directories where luet
+# try for files with .yml extension that define
+# luet repository.
+# repos_confdir:
+#   - /etc/luet/repos.conf.d
+#
+#
+# ---------------------------------------------
+# System repositories
+# ---------------------------------------------
+# In alternative to define repositories files
+# through repos_confdir option is possible
+# define directly the list of the repositories.
+#
+# repositories:
+#
+#     Name of the repository. It's better that this name is unique. Mandatory.
+#   - name: "repo1"
+#
+#     A user-friendly description of the repository
+#     description: "My luet repo"
+#
+#     Type of the repository. Supported types are: dir|http. Mandatory.
+#     type: "dir"
+#
+#     Define the priority of the repository on research packages. Default is 9999.
+#     priority: 9999
+#
+#     Enable/Disable of the repository.
+#     enable: false
+#
+#     Cached repository. If true a local cache of the remote repository tree is maintained
+#     locally in the $tree_path else it is used a temporary directory that is removed when
+#     installation of a package is completed. A cached repository reduce time on search/install
+#     packages. By default caching is disable.
+#     cached: false
+#
+#     Path where store tree of the specifications. Default path is $database_path/repos/$repo_name
+#     tree_path: "/var/cache/luet/repos/local"
+#
+#     Define the list of the URL where retrieve tree and packages.
+#     urls:
+#        - https://mydomain.local/luet/repo1
+#
+#     auth:
+#        Define Basic authentication header
+#        basic: "mybasicauth"
+#        Define token authentication header
+#        token: "mytoken"
+# ---------------------------------------------
+# Solver parameter configuration:
+# ---------------------------------------------
+# solver:
+#
+#   Solver strategy to solve possible conflicts during depedency
+#   solving. Defaults to empty (none). Available: qlearning
+#   type: ""
+#
+#   Solver agent learning rate. 0.1 to 1.0
+#   rate: 0.7
+#
+#   Learning discount factor.
+#   discount: 1.0
+#
+#   Number of overall attempts that the solver has available before bailing out.
+#   max_attempts: 9000
+#

go.mod

@@ -4,17 +4,18 @@ go 1.12
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56 // indirect
-	github.com/Sabayon/pkgs-checker v0.4.1
+	github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/briandowns/spinner v1.7.0
 	github.com/cavaliercoder/grab v2.0.0+incompatible
-	github.com/crillab/gophersat v1.1.7
+	github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3
 	github.com/docker/docker v0.7.3-0.20180827131323-0c5f8d2b9b23
+	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
 	github.com/ghodss/yaml v1.0.0
-	github.com/go-yaml/yaml v2.1.0+incompatible // indirect
 	github.com/hashicorp/go-version v1.2.0
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
+	github.com/klauspost/pgzip v1.2.1
+	github.com/kr/pretty v0.2.0 // indirect
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
@@ -32,8 +33,15 @@ require (
 	github.com/spf13/viper v1.5.0
 	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
 	go.etcd.io/bbolt v1.3.3
-	golang.org/x/crypto v0.0.0-20191028145041-f83a4685e152 // indirect
-	golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4 // indirect
-	gopkg.in/yaml.v2 v2.2.5
-	mvdan.cc/sh v2.6.4+incompatible
+	go.uber.org/atomic v1.5.1 // indirect
+	go.uber.org/multierr v1.4.0 // indirect
+	go.uber.org/zap v1.13.0
+	golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect
+	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
+	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
+	golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c // indirect
+	golang.org/x/tools v0.0.0-20200102200121-6de373a2766c // indirect
+	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
+	gopkg.in/yaml.v2 v2.2.7
+	mvdan.cc/sh/v3 v3.0.0-beta1
 )

go.sum

@@ -9,13 +9,11 @@ github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
-github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56 h1:XCZM9J5KqLsr5NqtrZuXiD3X5fe5IfgU7IIUZzpeFBk=
-github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56/go.mod h1:+Gbv6dg6TPHWq4oDjZY1vn978PLCEZ2hOu8kvn+S7t4=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/Sabayon/pkgs-checker v0.4.1 h1:NImZhA5Z9souyr9Ff3nDzP0Bs9SGuDLxRzduVqci3dQ=
-github.com/Sabayon/pkgs-checker v0.4.1/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
+github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7 h1:Vf80sSLu1ZWjjMmUKhw0FqM43lEOvT8O5B22NaHB6AQ=
+github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
 github.com/Sereal/Sereal v0.0.0-20181211220259-509a78ddbda3 h1:Xu7z47ZiE/J+sKXHZMGxEor/oY2q6dq51fkO0JqdSwY=
 github.com/Sereal/Sereal v0.0.0-20181211220259-509a78ddbda3/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -57,6 +55,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/crillab/gophersat v1.1.7 h1:f2Phe0W9jGyN1OefygKdcTdNM99q/goSjbWrFRjZGWc=
 github.com/crillab/gophersat v1.1.7/go.mod h1:S91tHga1PCZzYhCkStwZAhvp1rCc+zqtSi55I+vDWGc=
+github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3 h1:HO63LCf9kTXQgUnlvFeS2qSDQhZ/cLP8DAJO89CythY=
+github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3/go.mod h1:S91tHga1PCZzYhCkStwZAhvp1rCc+zqtSi55I+vDWGc=
 github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -77,6 +77,8 @@ github.com/docker/libnetwork v0.8.0-dev.2.0.20180608203834-19279f049241 h1:+ebE/
 github.com/docker/libnetwork v0.8.0-dev.2.0.20180608203834-19279f049241/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd h1:JWEotl+g5uCCn37eVAYLF3UjBqO5HJ0ezZ5Zgnsdoqc=
+github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd/go.mod h1:y0+kb0ORo7mC8lQbUzC4oa7ufu565J6SyUgWd39Z1Ic=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@@ -89,8 +91,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
-github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
@@ -107,6 +107,7 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
@@ -146,6 +147,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
@@ -197,13 +200,16 @@ github.com/opencontainers/runtime-spec v1.0.1 h1:wY4pOY8fBdSIvs9+IDHC55thBuEulhz
 github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc=
 github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
 github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
 github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f h1:WyCn68lTiytVSkk7W1K9nBiSGTSRlUOdyTnSjwrIlok=
 github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f/go.mod h1:/iRjX3DdSK956SzsUdV55J+wIsQ+2IBWmBrB4RvZfk4=
+github.com/pkg/diff v0.0.0-20190930165518-531926345625/go.mod h1:kFj35MyHn14a6pIgWhm46KVjJr5CHys3eEYxkuKD1EI=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -221,6 +227,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.5.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rootless-containers/proto v0.1.0 h1:gS1JOMEtk1YDYHCzBAf/url+olMJbac7MTrgSeP6zh4=
 github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZPu0UNHLXIQxhyqAFh8=
 github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
@@ -255,8 +263,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
-github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.5.0 h1:GpsTwfsQ27oS/Aha/6d1oD7tpKIqWnOA6tgOX9HHkt4=
 github.com/spf13/viper v1.5.0/go.mod h1:AkYRkVJF8TkSG/xet6PzXX+l39KhhXa2pdqVSxnTcn4=
 github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b h1:wJSBFlabo96ySlmSX0a02WAPyGxagzTo9c5sk3sHP3E=
@@ -294,18 +300,36 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
+go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E=
+go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU=
+go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191028145041-f83a4685e152 h1:ZC1Xn5A1nlpSmQCIva4bZ3ob3lmhYIefc+GU+DLg1Ow=
-golang.org/x/crypto v0.0.0-20191028145041-f83a4685e152/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc=
+golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -316,6 +340,8 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 h1:4dVFTC832rPn4pomLSz1vA+are2+dU19w1H8OngV7nc=
 golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -338,8 +364,8 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4 h1:Hynbrlo6LbYI3H1IqXpkVDOcX/3HiPdhVEuyj5a59RM=
-golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c h1:OYFUffxXPezb7BVTx9AaD4Vl0qtxmklBIkwCKH1YwDY=
+golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -348,8 +374,18 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190913181337-0240832f5c3d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200102200121-6de373a2766c h1:PBxLbymhzlh6kZuAXmeh8JK2tAJR0GM5Q/W71G2QJ40=
+golang.org/x/tools v0.0.0-20200102200121-6de373a2766c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -361,6 +397,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
@@ -372,10 +409,13 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.1.0+incompatible h1:5USw7CrJBYKqjg9R7QlA6jzqZKEAtvW82aNmsxxGPxw=
 gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-mvdan.cc/sh v2.6.4+incompatible h1:eD6tDeh0pw+/TOTI1BBEryZ02rD2nMcFsgcvde7jffM=
-mvdan.cc/sh v2.6.4+incompatible/go.mod h1:IeeQbZq+x2SUGBensq/jge5lLQbS3XT2ktyp3wrt4x8=
+honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+mvdan.cc/editorconfig v0.1.1-0.20191109213504-890940e3f00e/go.mod h1:Ge4atmRUYqueGppvJ7JNrtqpqokoJEFxYbP0Z+WeKS8=
+mvdan.cc/sh/v3 v3.0.0-beta1 h1:UqiwBEXEPzelaGxuvixaOtzc7WzKtrElePJ8HqvW7K8=
+mvdan.cc/sh/v3 v3.0.0-beta1/go.mod h1:rBIndNJFYPp8xSppiZcGIk6B5d1g3OEARxEaXjPxwVI=

pkg/compiler/artifact.go

@@ -16,22 +16,34 @@
 package compiler
 
 import (
+	"archive/tar"
+	"bufio"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 	"regexp"
 
+	gzip "github.com/klauspost/pgzip"
+
 	//"strconv"
 	"strings"
 	"sync"
 
+	. "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	"github.com/mudler/luet/pkg/solver"
-	yaml "gopkg.in/yaml.v2"
-
-	"github.com/mudler/luet/pkg/helpers"
 	"github.com/pkg/errors"
+	yaml "gopkg.in/yaml.v2"
+)
+
+type CompressionImplementation string
+
+const (
+	None CompressionImplementation = "none" // e.g. tar for standard packages
+	GZip CompressionImplementation = "gzip"
 )
 
 type ArtifactIndex []Artifact
@@ -40,7 +52,15 @@ func (i ArtifactIndex) CleanPath() ArtifactIndex {
 	newIndex := ArtifactIndex{}
 	for _, n := range i {
 		art := n.(*PackageArtifact)
-		newIndex = append(newIndex, &PackageArtifact{Path: path.Base(n.GetPath()), SourceAssertion: art.SourceAssertion, CompileSpec: art.CompileSpec, Dependencies: art.Dependencies})
+		// FIXME: This is a dup and makes difficult to add attributes to artifacts
+		newIndex = append(newIndex, &PackageArtifact{
+			Path:            path.Base(n.GetPath()),
+			SourceAssertion: art.SourceAssertion,
+			CompileSpec:     art.CompileSpec,
+			Dependencies:    art.Dependencies,
+			CompressionType: art.CompressionType,
+			Checksums:       art.Checksums,
+		})
 	}
 	return newIndex
 	//Update if exists, otherwise just create
@@ -50,19 +70,21 @@ func (i ArtifactIndex) CleanPath() ArtifactIndex {
 // which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact.
 // In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages.
 type PackageArtifact struct {
-	Path         string               `json:"path"`
-	Dependencies []*PackageArtifact   `json:"dependencies"`
-	CompileSpec  *LuetCompilationSpec `json:"compilationspec"`
+	Path string `json:"path"`
 
+	Dependencies    []*PackageArtifact        `json:"dependencies"`
+	CompileSpec     *LuetCompilationSpec      `json:"compilationspec"`
+	Checksums       Checksums                 `json:"checksums"`
 	SourceAssertion solver.PackagesAssertions `json:"-"`
+	CompressionType CompressionImplementation `json:"compressiontype"`
 }
 
 func NewPackageArtifact(path string) Artifact {
-	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}}
+	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: None}
 }
 
 func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
-	p := &PackageArtifact{}
+	p := &PackageArtifact{Checksums: Checksums{}}
 	err := yaml.Unmarshal(data, &p)
 	if err != nil {
 		return p, err
@@ -71,7 +93,56 @@ func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
 	return p, err
 }
 
+func LoadArtifactFromYaml(spec CompilationSpec) (Artifact, error) {
+
+	metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml"
+	dat, err := ioutil.ReadFile(spec.Rel(metaFile))
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading file "+metaFile)
+	}
+	art, err := NewPackageArtifactFromYaml(dat)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error writing file "+metaFile)
+	}
+	// It is relative, set it back to abs
+	art.SetPath(spec.Rel(art.GetPath()))
+	return art, nil
+}
+
+func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) {
+	a.CompressionType = t
+}
+
+func (a *PackageArtifact) GetChecksums() Checksums {
+	return a.Checksums
+}
+func (a *PackageArtifact) SetChecksums(c Checksums) {
+	a.Checksums = c
+}
+func (a *PackageArtifact) Hash() error {
+	return a.Checksums.Generate(a)
+}
+
+func (a *PackageArtifact) Verify() error {
+	sum := Checksums{}
+	err := sum.Generate(a)
+	if err != nil {
+		return err
+	}
+	err = sum.Compare(a.Checksums)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (a *PackageArtifact) WriteYaml(dst string) error {
+	// First compute checksum of artifact. When we write the yaml we want to write up-to-date informations.
+	err := a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for artifact")
+	}
+
 	//p := a.CompileSpec.GetPackage().GetPath()
 
 	//a.CompileSpec.GetPackage().SetPath("")
@@ -148,6 +219,155 @@ func (a *PackageArtifact) SetPath(p string) {
 	a.Path = p
 }
 
+// Compress Archives and compress (TODO) to the artifact path
+func (a *PackageArtifact) Compress(src string, concurrency int) error {
+	switch a.CompressionType {
+	case GZip:
+		err := helpers.Tar(src, a.Path)
+		if err != nil {
+			return err
+		}
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return err
+		}
+		defer original.Close()
+
+		gzipfile := a.Path + ".gz"
+		bufferedReader := bufio.NewReader(original)
+
+		// Open a file for writing.
+		dst, err := os.Create(gzipfile)
+		if err != nil {
+			return err
+		}
+		// Create gzip writer.
+		w := gzip.NewWriter(dst)
+		w.SetConcurrency(concurrency, 10)
+		defer w.Close()
+		defer dst.Close()
+		_, err = io.Copy(w, bufferedReader)
+		if err != nil {
+			return err
+		}
+		w.Close()
+		os.RemoveAll(a.Path) // Remove original
+		//	a.CompressedPath = gzipfile
+		a.Path = gzipfile
+		return nil
+		//a.Path = gzipfile
+
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		return helpers.Tar(src, a.Path)
+
+	}
+	return errors.New("Compression type must be supplied")
+}
+
+// Unpack Untar and decompress (TODO) to the given path
+func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+	switch a.CompressionType {
+	case GZip:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := gzip.NewReader(bufferedReader)
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+
+		_, err = io.Copy(archive, r)
+		if err != nil {
+			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
+		}
+
+		err = helpers.Untar(a.GetPath()+".uncompressed", dst,
+			LuetCfg.GetGeneral().SameOwner)
+		if err != nil {
+			return err
+		}
+		return nil
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		return helpers.Untar(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner)
+	}
+	return errors.New("Compression type must be supplied")
+}
+
+func (a *PackageArtifact) FileList() ([]string, error) {
+	var tr *tar.Reader
+	switch a.CompressionType {
+	case GZip:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		if err != nil {
+			return []string{}, err
+		}
+		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := gzip.NewReader(bufferedReader)
+		if err != nil {
+			return []string{}, err
+		}
+		defer r.Close()
+		tr = tar.NewReader(r)
+
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		tarFile, err := os.Open(a.GetPath())
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Could not open package archive")
+		}
+		defer tarFile.Close()
+		tr = tar.NewReader(tarFile)
+
+	}
+
+	var files []string
+	// untar each segment
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return []string{}, err
+		}
+		// determine proper file path info
+		finfo := hdr.FileInfo()
+		fileName := hdr.Name
+		if finfo.Mode().IsDir() {
+			continue
+		}
+		files = append(files, fileName)
+
+		// if a dir, create it, then go to next segment
+	}
+	return files, nil
+}
+
 type CopyJob struct {
 	Src, Dst string
 	Artifact string
@@ -175,7 +395,7 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 }
 
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string) (Artifact, error) {
+func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, t CompressionImplementation) (Artifact, error) {
 
 	archive, err := ioutil.TempDir(os.TempDir(), "archive")
 	if err != nil {
@@ -239,10 +459,46 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 
 	close(toCopy)
 	wg.Wait()
-
-	err = helpers.Tar(archive, dst)
+	a := NewPackageArtifact(dst)
+	a.SetCompressionType(t)
+	err = a.Compress(archive, concurrency)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}
-	return NewPackageArtifact(dst), nil
+	return a, nil
+}
+
+func ComputeArtifactLayerSummary(diffs []ArtifactLayer) ArtifactLayersSummary {
+
+	ans := ArtifactLayersSummary{
+		Layers: make([]ArtifactLayerSummary, 0),
+	}
+
+	for _, layer := range diffs {
+		sum := ArtifactLayerSummary{
+			FromImage:   layer.FromImage,
+			ToImage:     layer.ToImage,
+			AddFiles:    0,
+			AddSizes:    0,
+			DelFiles:    0,
+			DelSizes:    0,
+			ChangeFiles: 0,
+			ChangeSizes: 0,
+		}
+		for _, a := range layer.Diffs.Additions {
+			sum.AddFiles++
+			sum.AddSizes += int64(a.Size)
+		}
+		for _, d := range layer.Diffs.Deletions {
+			sum.DelFiles++
+			sum.DelSizes += int64(d.Size)
+		}
+		for _, c := range layer.Diffs.Changes {
+			sum.ChangeFiles++
+			sum.ChangeSizes += int64(c.Size)
+		}
+		ans.Layers = append(ans.Layers, sum)
+	}
+
+	return ans
 }

pkg/compiler/artifact_test.go

@@ -32,7 +32,7 @@ import (
 
 var _ = Describe("Artifact", func() {
 	Context("Simple package build definition", func() {
-		It("Generates a delta", func() {
+		It("Generates a verified delta", func() {
 
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
@@ -41,7 +41,7 @@ var _ = Describe("Artifact", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -125,7 +125,7 @@ RUN echo bar > /test2`))
 			err = b.ExtractRootfs(CompilerBackendOptions{SourcePath: filepath.Join(tmpdir, "output2.tar"), Destination: rootfs}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{})
+			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, None)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
 			err = helpers.Untar(artifact.GetPath(), unpacked, false)
@@ -138,6 +138,15 @@ RUN echo bar > /test2`))
 			content2, err := helpers.Read(filepath.Join(unpacked, "test2"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content2).To(Equal("bar\n"))
+
+			err = artifact.Hash()
+			Expect(err).ToNot(HaveOccurred())
+			err = artifact.Verify()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
+
+			err = artifact.Verify()
+			Expect(err).To(HaveOccurred())
 		})
 
 	})

pkg/compiler/backend/backend_suite_test.go

@@ -18,11 +18,15 @@ package backend_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Backend Suite")
 }

pkg/compiler/backend/simpledocker.go

@@ -17,6 +17,7 @@ package backend
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -26,6 +27,7 @@ import (
 	capi "github.com/mudler/docker-companion/api"
 
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -54,7 +56,12 @@ func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
 	}
 	Info(":whale: Building image " + name + " done")
 
-	//Info(string(out))
+	if config.LuetCfg.GetGeneral().ShowBuildOutput {
+		Info(string(out))
+	} else {
+		Debug(string(out))
+	}
+
 	return nil
 }
 
@@ -229,11 +236,27 @@ func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLaye
 		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
 	}
 
+	if config.LuetCfg.GetGeneral().ShowBuildOutput {
+		Info(string(out))
+	}
+
 	var diffs []compiler.ArtifactLayer
 
 	err = json.Unmarshal(out, &diffs)
 	if err != nil {
 		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
 	}
+
+	if config.LuetCfg.GetLogging().Level == "debug" {
+		summary := compiler.ComputeArtifactLayerSummary(diffs)
+		for _, l := range summary.Layers {
+			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
+				l.FromImage, l.ToImage,
+				l.AddFiles, l.AddSizes,
+				l.DelFiles, l.DelSizes,
+				l.ChangeFiles, l.ChangeSizes))
+		}
+	}
+
 	return diffs, nil
 }

pkg/compiler/backend/simpledocker_test.go

@@ -40,7 +40,7 @@ var _ = Describe("Docker backend", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 

pkg/compiler/checksum.go

@@ -0,0 +1,78 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+
+	//"strconv"
+
+	"crypto/sha256"
+	"fmt"
+	"hash"
+	"io"
+	"os"
+
+	//	. "github.com/mudler/luet/pkg/logger"
+	"github.com/pkg/errors"
+)
+
+type HashImplementation string
+
+const (
+	SHA256 HashImplementation = "sha256"
+)
+
+type Checksums map[string]string
+
+type HashOptions struct {
+	Hasher hash.Hash
+	Type   HashImplementation
+}
+
+// Generate generates all Checksums supported for the artifact
+func (c *Checksums) Generate(a Artifact) error {
+	return c.generateSHA256(a)
+}
+
+func (c Checksums) Compare(d Checksums) error {
+	for t, sum := range d {
+		if v, ok := c[t]; ok && v != sum {
+			return errors.New("Checksum mismsatch")
+		}
+	}
+	return nil
+}
+
+func (c *Checksums) generateSHA256(a Artifact) error {
+	return c.generateSum(a, HashOptions{Hasher: sha256.New(), Type: SHA256})
+}
+
+func (c *Checksums) generateSum(a Artifact, opts HashOptions) error {
+
+	f, err := os.Open(a.GetPath())
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := io.Copy(opts.Hasher, f); err != nil {
+		return err
+	}
+
+	sum := fmt.Sprintf("%x", opts.Hasher.Sum(nil))
+
+	(*c)[string(opts.Type)] = sum
+	return nil
+}

pkg/compiler/checksum_test.go

@@ -0,0 +1,61 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler_test
+
+import (
+	"io/ioutil"
+	"os"
+
+	. "github.com/mudler/luet/pkg/compiler"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Checksum", func() {
+	Context("Generation", func() {
+		It("Compares successfully", func() {
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			buildsum := Checksums{}
+			definitionsum := Checksums{}
+			definitionsum2 := Checksums{}
+
+			Expect(len(buildsum)).To(Equal(0))
+			Expect(len(definitionsum)).To(Equal(0))
+			Expect(len(definitionsum2)).To(Equal(0))
+
+			err = buildsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/build.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = definitionsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = definitionsum2.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(buildsum)).To(Equal(1))
+			Expect(len(definitionsum)).To(Equal(1))
+			Expect(len(definitionsum2)).To(Equal(1))
+
+			Expect(definitionsum.Compare(buildsum)).To(HaveOccurred())
+			Expect(definitionsum.Compare(definitionsum2)).ToNot(HaveOccurred())
+		})
+	})
+
+})

pkg/compiler/compiler.go

@@ -36,13 +36,16 @@ const BuildFile = "build.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
-	Backend            CompilerBackend
-	Database           pkg.PackageDatabase
-	ImageRepository    string
-	PullFirst, KeepImg bool
+	Backend                   CompilerBackend
+	Database                  pkg.PackageDatabase
+	ImageRepository           string
+	PullFirst, KeepImg, Clean bool
+	Concurrency               int
+	CompressionType           CompressionImplementation
+	Options                   CompilerOptions
 }
 
-func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase) Compiler {
+func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions) Compiler {
 	// The CompilerRecipe will gives us a tree with only build deps listed.
 	return &LuetCompiler{
 		Backend: backend,
@@ -50,12 +53,24 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase) Compiler {
 			tree.Recipe{Database: db},
 		},
 		Database:        db,
-		ImageRepository: "luet/cache",
-		PullFirst:       true,
-		KeepImg:         true,
+		ImageRepository: opt.ImageRepository,
+		PullFirst:       opt.PullFirst,
+		CompressionType: opt.CompressionType,
+		KeepImg:         opt.KeepImg,
+		Concurrency:     opt.Concurrency,
+		Clean:           opt.Clean,
+		Options:         *opt,
 	}
 }
 
+func (cs *LuetCompiler) SetConcurrency(i int) {
+	cs.Concurrency = i
+}
+
+func (cs *LuetCompiler) SetCompressionType(t CompressionImplementation) {
+	cs.CompressionType = t
+}
+
 func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan CompilationSpec, a *[]Artifact, m *sync.Mutex, concurrency int, keepPermissions bool, errors chan error) {
 	defer wg.Done()
 
@@ -70,8 +85,9 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan Co
 		m.Unlock()
 	}
 }
-func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
-	artifacts, err := cs.CompileParallel(concurrency, keepPermissions, ps)
+
+func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
+	artifacts, err := cs.CompileParallel(keepPermissions, ps)
 	if len(err) != 0 {
 		return artifacts, err
 	}
@@ -119,11 +135,11 @@ func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions
 		Info(" :arrow_right_hook:", u.GetPackage().GetName(), ":leaves:", u.GetPackage().GetVersion(), "(", u.GetPackage().GetCategory(), ")")
 	}
 
-	artifacts2, err := cs.CompileParallel(concurrency, keepPermissions, uniques)
+	artifacts2, err := cs.CompileParallel(keepPermissions, uniques)
 	return append(artifacts, artifacts2...), err
 }
 
-func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
+func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
 	Spinner(22)
 	defer SpinnerStop()
 	all := make(chan CompilationSpec)
@@ -131,9 +147,9 @@ func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, p
 	mutex := &sync.Mutex{}
 	errors := make(chan error, ps.Len())
 	var wg = new(sync.WaitGroup)
-	for i := 0; i < concurrency; i++ {
+	for i := 0; i < cs.Concurrency; i++ {
 		wg.Add(1)
-		go cs.compilerWorker(i, wg, all, &artifacts, mutex, concurrency, keepPermissions, errors)
+		go cs.compilerWorker(i, wg, all, &artifacts, mutex, cs.Concurrency, keepPermissions, errors)
 	}
 
 	for _, p := range ps.All() {
@@ -213,6 +229,12 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 }
 
 func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec) (Artifact, error) {
+	if !cs.Clean {
+		if art, err := LoadArtifactFromYaml(p); err == nil {
+			Debug("Artifact reloaded. Skipping build")
+			return art, err
+		}
+	}
 	pkgTag := ":package:  " + p.GetPackage().GetName()
 
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
@@ -321,13 +343,11 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		// TODO: Handle caching and optionally do not remove things
 		err = cs.Backend.RemoveImage(builderOpts)
 		if err != nil {
-			// TODO: Have a --fatal flag which enables Warnings to exit.
 			Warning("Could not remove image ", builderOpts.ImageName)
 			//	return nil, errors.Wrap(err, "Could not remove image")
 		}
 		err = cs.Backend.RemoveImage(runnerOpts)
 		if err != nil {
-			// TODO: Have a --fatal flag which enables Warnings to exit.
 			Warning("Could not remove image ", builderOpts.ImageName)
 			//	return nil, errors.Wrap(err, "Could not remove image")
 		}
@@ -339,21 +359,22 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 			// strip from includes
 			cs.stripIncludesFromRootfs(p.GetIncludes(), rootfs)
 		}
-
-		err = helpers.Tar(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"))
+		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
+		artifact.SetCompressionType(cs.CompressionType)
+		err = artifact.Compress(rootfs, concurrency)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating package archive")
 		}
 
-		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 		artifact.SetCompileSpec(p)
 	} else {
 		Info(pkgTag, "Generating delta")
 
-		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes())
+		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not generate deltas")
 		}
+
 		artifact.SetCompileSpec(p)
 	}
 
@@ -366,7 +387,13 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	return artifact, nil
 }
 
-func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions, keepImg bool) (Artifact, error) {
+func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions, keepImg bool, concurrency int) (Artifact, error) {
+	if !cs.Clean {
+		if art, err := LoadArtifactFromYaml(p); err == nil {
+			Debug("Artifact reloaded. Skipping build")
+			return art, err
+		}
+	}
 	pkgTag := ":package:  " + p.GetPackage().GetName()
 
 	Info(pkgTag, "   🍩 Build starts 🔨 🔨 🔨 ")
@@ -404,8 +431,11 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not extract rootfs")
 	}
+	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
+	artifact.SetCompileSpec(p)
+	artifact.SetCompressionType(cs.CompressionType)
 
-	err = helpers.Tar(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"))
+	err = artifact.Compress(rootfs, concurrency)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}
@@ -415,15 +445,13 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 		// TODO: Handle caching and optionally do not remove things
 		err = cs.Backend.RemoveImage(builderOpts)
 		if err != nil {
-			// TODO: Have a --fatal flag which enables Warnings to exit.
 			Warning("Could not remove image ", builderOpts.ImageName)
 			//	return nil, errors.Wrap(err, "Could not remove image")
 		}
 	}
 
 	Info(pkgTag, "   :white_check_mark: Done")
-	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
-	artifact.SetCompileSpec(p)
+
 	err = artifact.WriteYaml(p.GetOutputPath())
 	if err != nil {
 		return artifact, err
@@ -433,7 +461,7 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 
 func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
 
-	s := solver.NewSolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false))
+	s := solver.NewResolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
 
 	solution, err := s.Install([]pkg.Package{p.GetPackage()})
 	if err != nil {
@@ -441,18 +469,15 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
 	}
 
 	dependencies := solution.Order(cs.Database, p.GetPackage().GetFingerPrint())
-	assertions := solver.PackagesAssertions{}
 
+	assertions := solver.PackagesAssertions{}
 	for _, assertion := range dependencies { //highly dependent on the order
 		if assertion.Value {
-			nthsolution, err := s.Install([]pkg.Package{assertion.Package})
-			if err != nil {
-				return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
-			}
+			nthsolution := dependencies.Cut(assertion.Package)
 
 			assertion.Hash = solver.PackageHash{
-				BuildHash:   nthsolution.Order(cs.Database, assertion.Package.GetFingerPrint()).Drop(assertion.Package).AssertionHash(),
-				PackageHash: nthsolution.Order(cs.Database, assertion.Package.GetFingerPrint()).AssertionHash(),
+				BuildHash:   nthsolution.Drop(assertion.Package).AssertionHash(),
+				PackageHash: nthsolution.AssertionHash(),
 			}
 			assertions = append(assertions, assertion)
 		}
@@ -462,13 +487,13 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
 }
 
 // Compile is non-parallel
-func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
+func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifact, error) {
 	asserts, err := cs.ComputeDepTree(p)
 	if err != nil {
 		panic(err)
 	}
 	p.SetSourceAssertion(asserts)
-	return cs.compile(concurrency, keepPermissions, p)
+	return cs.compile(cs.Concurrency, keepPermissions, p)
 }
 
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
@@ -483,7 +508,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
 		if p.ImageUnpack() { // If it is just an entire image, create a package from it
-			return cs.packageFromImage(p, "", keepPermissions, cs.KeepImg)
+			return cs.packageFromImage(p, "", keepPermissions, cs.KeepImg, concurrency)
 		}
 
 		return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, cs.KeepImg, p)
@@ -530,7 +555,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 					return nil, errors.New("No image defined for package: " + assertion.Package.GetName())
 				}
 				Info(pkgTag, ":whale: Sourcing package from image", compileSpec.GetImage())
-				artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions, cs.KeepImg)
+				artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions, cs.KeepImg, concurrency)
 				if err != nil {
 					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
 				}

pkg/compiler/compiler_suite_test.go

@@ -18,11 +18,15 @@ package compiler_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Compiler Suite")
 }

pkg/compiler/compiler_test.go

@@ -38,7 +38,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -53,7 +53,9 @@ var _ = Describe("Compiler", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			artifact, err := compiler.Compile(2, false, spec)
+			compiler.SetConcurrency(2)
+
+			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -80,7 +82,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -95,7 +97,8 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec, spec2))
+			compiler.SetConcurrency(2)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
 				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
@@ -117,7 +120,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -134,8 +137,9 @@ var _ = Describe("Compiler", func() {
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(2)
 
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec, spec2, spec3))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2, spec3))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(3))
 
@@ -173,7 +177,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -181,12 +185,12 @@ var _ = Describe("Compiler", func() {
 			Expect(err).ToNot(HaveOccurred())
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
-			artifacts2, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec2))
+			artifacts2, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts2)).To(Equal(1))
 
@@ -215,7 +219,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -224,8 +228,9 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
@@ -249,7 +254,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -258,8 +263,8 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
@@ -287,7 +292,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -296,8 +301,9 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
@@ -316,6 +322,92 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
+		It("Compiles with provides support", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/provides")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
+		})
+
+		It("Compiles with provides and selectors support", func() {
+			// Same test as before, but fixtures differs
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/provides_selector")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
+		})
 		It("Compiles revdeps", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "revdep")
@@ -327,7 +419,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -337,7 +429,7 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(2))
 
@@ -354,6 +446,45 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
+		It("Compiles complex dependencies trees with best matches", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "complex")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/complex/selection")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(6))
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("build-sabayon-overlays-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("linux-sabayon-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("sabayon-sources-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("vhba"))).To(BeTrue())
+		})
+
 		It("Compiles revdeps with seeds", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
@@ -365,13 +496,13 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(4))
 
@@ -417,7 +548,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -429,7 +560,9 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(2)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
 				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
@@ -458,7 +591,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -470,7 +603,9 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
@@ -479,4 +614,40 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
 	})
+
+	Context("Compression", func() {
+		It("Builds packages in gzip", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/packagelayers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			compiler.SetCompressionType(GZip)
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
+			Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred())
+			//	Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
+		})
+	})
 })

pkg/compiler/interface.go

@@ -16,20 +16,24 @@
 package compiler
 
 import (
+	"runtime"
+
+	"github.com/mudler/luet/pkg/config"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 )
 
 type Compiler interface {
-	Compile(int, bool, CompilationSpec) (Artifact, error)
-	CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
+	Compile(bool, CompilationSpec) (Artifact, error)
+	CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
+	CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
-
+	SetConcurrency(i int)
 	FromPackage(pkg.Package) (CompilationSpec, error)
 
 	SetBackend(CompilerBackend)
 	GetBackend() CompilerBackend
+	SetCompressionType(t CompressionImplementation)
 }
 
 type CompilerBackendOptions struct {
@@ -39,6 +43,27 @@ type CompilerBackendOptions struct {
 	Destination    string
 }
 
+type CompilerOptions struct {
+	ImageRepository    string
+	PullFirst, KeepImg bool
+	Concurrency        int
+	CompressionType    CompressionImplementation
+	Clean              bool
+
+	SolverOptions config.LuetSolverOptions
+}
+
+func NewDefaultCompilerOptions() *CompilerOptions {
+	return &CompilerOptions{
+		ImageRepository: "luet/cache",
+		PullFirst:       true,
+		CompressionType: None,
+		KeepImg:         true,
+		Concurrency:     runtime.NumCPU(),
+		Clean:           true,
+	}
+}
+
 type CompilerBackend interface {
 	BuildImage(CompilerBackendOptions) error
 	ExportImage(CompilerBackendOptions) error
@@ -62,6 +87,15 @@ type Artifact interface {
 	SetCompileSpec(as CompilationSpec)
 	GetCompileSpec() CompilationSpec
 	WriteYaml(dst string) error
+	Unpack(dst string, keepPerms bool) error
+	Compress(src string, concurrency int) error
+	SetCompressionType(t CompressionImplementation)
+	FileList() ([]string, error)
+	Hash() error
+	Verify() error
+
+	GetChecksums() Checksums
+	SetChecksums(c Checksums)
 }
 
 type ArtifactNode struct {
@@ -78,6 +112,19 @@ type ArtifactLayer struct {
 	ToImage   string        `json:"Image2"`
 	Diffs     ArtifactDiffs `json:"Diff"`
 }
+type ArtifactLayerSummary struct {
+	FromImage   string `json:"image1"`
+	ToImage     string `json:"image2"`
+	AddFiles    int    `json:"add_files"`
+	AddSizes    int64  `json:"add_sizes"`
+	DelFiles    int    `json:"del_files"`
+	DelSizes    int64  `json:"del_sizes"`
+	ChangeFiles int    `json:"change_files"`
+	ChangeSizes int64  `json:"change_sizes"`
+}
+type ArtifactLayersSummary struct {
+	Layers []ArtifactLayerSummary `json:"summary"`
+}
 
 // CompilationSpec represent a compilation specification derived from a package
 type CompilationSpec interface {

pkg/compiler/spec_test.go

@@ -61,7 +61,7 @@ var _ = Describe("Spec", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 

pkg/config/config.go

@@ -0,0 +1,326 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	solver "github.com/mudler/luet/pkg/solver"
+	v "github.com/spf13/viper"
+)
+
+var LuetCfg = NewLuetConfig(v.GetViper())
+var AvailableResolvers = strings.Join([]string{solver.QLearningResolverType}, " ")
+
+type LuetLoggingConfig struct {
+	Path       string `mapstructure:"path"`
+	JsonFormat bool   `mapstructure:"json_format"`
+	Level      string `mapstructure:"level"`
+}
+
+type LuetGeneralConfig struct {
+	SameOwner       bool `mapstructure:"same_owner"`
+	Concurrency     int  `mapstructure:"concurrency"`
+	Debug           bool `mapstructure:"debug"`
+	ShowBuildOutput bool `mapstructure:"show_build_output"`
+	SpinnerMs       int  `mapstructure:"spinner_ms"`
+	SpinnerCharset  int  `mapstructure:"spinner_charset"`
+	FatalWarns      bool `mapstructure:"fatal_warnings"`
+}
+
+type LuetSolverOptions struct {
+	Type        string  `mapstructure:"type"`
+	LearnRate   float32 `mapstructure:"rate"`
+	Discount    float32 `mapstructure:"discount"`
+	MaxAttempts int     `mapstructure:"max_attempts"`
+}
+
+func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		if opts.LearnRate != 0.0 {
+			return solver.NewQLearningResolver(opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+
+		}
+		return solver.SimpleQLearningSolver()
+	}
+
+	return &solver.DummyPackageResolver{}
+}
+
+func (opts *LuetSolverOptions) CompactString() string {
+	return fmt.Sprintf("type: %s rate: %f, discount: %f, attempts: %d, initialobserved: %d",
+		opts.Type, opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+}
+
+type LuetSystemConfig struct {
+	DatabaseEngine string `yaml:"database_engine" mapstructure:"database_engine"`
+	DatabasePath   string `yaml:"database_path" mapstructure:"database_path"`
+	Rootfs         string `yaml:"rootfs" mapstructure:"rootfs"`
+	PkgsCachePath  string `yaml:"pkgs_cache_path" mapstructure:"pkgs_cache_path"`
+}
+
+func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
+	dbpath = filepath.Join(dbpath, "repos/"+name)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+	dbpath := filepath.Join(sc.Rootfs,
+		sc.DatabasePath)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+	var cachepath string
+	if sc.PkgsCachePath != "" {
+		cachepath = sc.PkgsCachePath
+	} else {
+		// Create dynamic cache for test suites
+		cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
+	}
+
+	if filepath.IsAbs(cachepath) {
+		ans = cachepath
+	} else {
+		ans = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
+	}
+
+	return
+}
+
+type LuetRepository struct {
+	Name           string            `json:"name" yaml:"name" mapstructure:"name"`
+	Description    string            `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
+	Urls           []string          `json:"urls" yaml:"urls" mapstructure:"urls"`
+	Type           string            `json:"type" yaml:"type" mapstructure:"type"`
+	Mode           string            `json:"mode,omitempty" yaml:"mode,omitempty" mapstructure:"mode,omitempty"`
+	Priority       int               `json:"priority,omitempty" yaml:"priority,omitempty" mapstructure:"priority"`
+	Enable         bool              `json:"enable" yaml:"enable" mapstructure:"enable"`
+	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
+	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
+	TreePath       string            `json:"tree_path,omitempty" yaml:"tree_path,omitempty" mapstructure:"tree_path"`
+
+	// Serialized options not used in repository configuration
+
+	// Incremented value that identify revision of the repository in a user-friendly way.
+	Revision int `json:"revision,omitempty" yaml:"-,omitempty" mapstructure:"-,omitempty"`
+	// Epoch time in seconds
+	LastUpdate string `json:"last_update,omitempty" yaml:"-,omitempty" mapstructure:"-,omitempty"`
+}
+
+func NewLuetRepository(name, t, descr string, urls []string, priority int, enable, cached bool) *LuetRepository {
+	return &LuetRepository{
+		Name:        name,
+		Description: descr,
+		Urls:        urls,
+		Type:        t,
+		// Used in cached repositories
+		Mode:           "",
+		Priority:       priority,
+		Enable:         enable,
+		Cached:         cached,
+		Authentication: make(map[string]string, 0),
+		TreePath:       "",
+	}
+}
+
+func NewEmptyLuetRepository() *LuetRepository {
+	return &LuetRepository{
+		Name:           "",
+		Description:    "",
+		Urls:           []string{},
+		Type:           "",
+		Priority:       9999,
+		TreePath:       "",
+		Enable:         false,
+		Cached:         false,
+		Authentication: make(map[string]string, 0),
+	}
+}
+
+func (r *LuetRepository) String() string {
+	return fmt.Sprintf("[%s] prio: %d, type: %s, enable: %t, cached: %t",
+		r.Name, r.Priority, r.Type, r.Enable, r.Cached)
+}
+
+type LuetConfig struct {
+	Viper *v.Viper
+
+	Logging LuetLoggingConfig `mapstructure:"logging"`
+	General LuetGeneralConfig `mapstructure:"general"`
+	System  LuetSystemConfig  `mapstructure:"system"`
+	Solver  LuetSolverOptions `mapstructure:"solver"`
+
+	RepositoriesConfDir []string         `mapstructure:"repos_confdir"`
+	CacheRepositories   []LuetRepository `mapstructure:"repetitors"`
+	SystemRepositories  []LuetRepository `mapstructure:"repositories"`
+}
+
+func NewLuetConfig(viper *v.Viper) *LuetConfig {
+	if viper == nil {
+		viper = v.New()
+	}
+
+	GenDefault(viper)
+	return &LuetConfig{Viper: viper}
+}
+
+func GenDefault(viper *v.Viper) {
+	viper.SetDefault("logging.level", "info")
+	viper.SetDefault("logging.path", "")
+	viper.SetDefault("logging.json_format", false)
+
+	viper.SetDefault("general.concurrency", runtime.NumCPU())
+	viper.SetDefault("general.debug", false)
+	viper.SetDefault("general.show_build_output", false)
+	viper.SetDefault("general.spinner_ms", 100)
+	viper.SetDefault("general.spinner_charset", 22)
+	viper.SetDefault("general.fatal_warnings", false)
+
+	u, _ := user.Current()
+	if u.Uid == "0" {
+		viper.SetDefault("general.same_owner", true)
+	} else {
+		viper.SetDefault("general.same_owner", false)
+	}
+
+	viper.SetDefault("system.database_engine", "boltdb")
+	viper.SetDefault("system.database_path", "/var/cache/luet")
+	viper.SetDefault("system.rootfs", "/")
+	viper.SetDefault("system.pkgs_cache_path", "packages")
+
+	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
+	viper.SetDefault("cache_repositories", []string{})
+	viper.SetDefault("system_repositories", []string{})
+
+	viper.SetDefault("solver.type", "")
+	viper.SetDefault("solver.rate", 0.7)
+	viper.SetDefault("solver.discount", 1.0)
+	viper.SetDefault("solver.max_attempts", 9000)
+}
+
+func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
+	c.SystemRepositories = append(c.SystemRepositories, r)
+}
+
+func (c *LuetConfig) GetLogging() *LuetLoggingConfig {
+	return &c.Logging
+}
+
+func (c *LuetConfig) GetGeneral() *LuetGeneralConfig {
+	return &c.General
+}
+
+func (c *LuetConfig) GetSystem() *LuetSystemConfig {
+	return &c.System
+}
+
+func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
+	return &c.Solver
+}
+
+func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
+	var ans *LuetRepository = nil
+
+	for idx, repo := range c.SystemRepositories {
+		if repo.Name == name {
+			ans = &c.SystemRepositories[idx]
+			break
+		}
+	}
+	if ans == nil {
+		return nil, errors.New("Repository " + name + " not found")
+	}
+
+	return ans, nil
+}
+
+func (c *LuetSolverOptions) String() string {
+	ans := fmt.Sprintf(`
+solver:
+  type: %s
+  rate: %f
+  discount: %f
+  max_attempts: %d`, c.Type, c.LearnRate, c.Discount,
+		c.MaxAttempts)
+
+	return ans
+}
+
+func (c *LuetGeneralConfig) String() string {
+	ans := fmt.Sprintf(`
+general:
+  concurrency: %d
+  same_owner: %t
+  debug: %t
+  fatal_warnings: %t
+  show_build_output: %t
+  spinner_ms: %d
+  spinner_charset: %d`, c.Concurrency, c.SameOwner, c.Debug,
+		c.FatalWarns, c.ShowBuildOutput,
+		c.SpinnerMs, c.SpinnerCharset)
+
+	return ans
+}
+
+func (c *LuetGeneralConfig) GetSpinnerMs() time.Duration {
+	duration, err := time.ParseDuration(fmt.Sprintf("%dms", c.SpinnerMs))
+	if err != nil {
+		return 100 * time.Millisecond
+	}
+	return duration
+}
+
+func (c *LuetLoggingConfig) String() string {
+	ans := fmt.Sprintf(`
+logging:
+  path: %s
+  json_format: %t
+  level: %s`, c.Path, c.JsonFormat, c.Level)
+
+	return ans
+}
+
+func (c *LuetSystemConfig) String() string {
+	ans := fmt.Sprintf(`
+system:
+  database_engine: %s
+  database_path: %s
+  pkgs_cache_path: %s
+  rootfs: %s`,
+		c.DatabaseEngine, c.DatabasePath, c.PkgsCachePath, c.Rootfs)
+
+	return ans
+}

pkg/helpers/archive.go

@@ -16,8 +16,10 @@
 package helpers
 
 import (
+	"archive/tar"
 	"io"
 	"os"
+	"path/filepath"
 
 	"github.com/docker/docker/pkg/archive"
 )
@@ -49,14 +51,82 @@ func Tar(src, dest string) error {
 
 // Untar just a wrapper around the docker functions
 func Untar(src, dest string, sameOwner bool) error {
+	var ans error
+
 	in, err := os.Open(src)
 	if err != nil {
 		return err
 	}
 	defer in.Close()
 
-	return archive.Untar(in, dest, &archive.TarOptions{
-		NoLchown:        !sameOwner,
-		ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
-	})
+	if sameOwner {
+		// PRE: i have root privileged.
+
+		opts := &archive.TarOptions{
+			// NOTE: NoLchown boolean is used for chmod of the symlink
+			// Probably it's needed set this always to true.
+			NoLchown:        true,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+		}
+
+		ans = archive.Untar(in, dest, opts)
+	} else {
+
+		var fileReader io.ReadCloser = in
+
+		tr := tar.NewReader(fileReader)
+		for {
+			header, err := tr.Next()
+
+			switch {
+			case err == io.EOF:
+				goto tarEof
+			case err != nil:
+				return err
+			case header == nil:
+				continue
+			}
+
+			// the target location where the dir/file should be created
+			target := filepath.Join(dest, header.Name)
+
+			// Check the file type
+			switch header.Typeflag {
+
+			// if its a dir and it doesn't exist create it
+			case tar.TypeDir:
+				if _, err := os.Stat(target); err != nil {
+					if err := os.MkdirAll(target, 0755); err != nil {
+						return err
+					}
+				}
+
+				// handle creation of file
+			case tar.TypeReg:
+				f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+				if err != nil {
+					return err
+				}
+
+				// copy over contents
+				if _, err := io.Copy(f, tr); err != nil {
+					return err
+				}
+
+				// manually close here after each file operation; defering would cause each
+				// file close to wait until all operations have completed.
+				f.Close()
+
+			case tar.TypeSymlink:
+				source := header.Linkname
+				err := os.Symlink(source, target)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	tarEof:
+	}
+
+	return ans
 }

pkg/helpers/helpers_suite_test.go

@@ -18,11 +18,15 @@ package helpers_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Helpers Suite")
 }

pkg/helpers/math.go

@@ -0,0 +1,24 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+func Factorial(n uint64) (result uint64) {
+	if n > 0 {
+		result = n * Factorial(n-1)
+		return result
+	}
+	return 1
+}

pkg/installer/client/client_suite_test.go

@@ -18,11 +18,19 @@ package client_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestClient(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	// Set temporary directory for rootfs
+	config.LuetCfg.GetSystem().Rootfs = "/tmp/luet-root"
+	// Force dynamic path for packages cache
+	config.LuetCfg.GetSystem().PkgsCachePath = ""
 	RunSpecs(t, "Client Suite")
 }

pkg/installer/client/http.go

@@ -16,7 +16,9 @@
 package client
 
 import (
+	"fmt"
 	"io/ioutil"
+	"math"
 	"net/url"
 	"os"
 	"path"
@@ -25,6 +27,7 @@ import (
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 
 	"github.com/cavaliercoder/grab"
@@ -38,60 +41,151 @@ func NewHttpClient(r RepoData) *HttpClient {
 	return &HttpClient{RepoData: r}
 }
 
+func (c *HttpClient) PrepareReq(dst, url string) (*grab.Request, error) {
+
+	req, err := grab.NewRequest(dst, url)
+	if err != nil {
+		return nil, err
+	}
+
+	if val, ok := c.RepoData.Authentication["token"]; ok {
+		req.HTTPRequest.Header.Set("Authorization", "token "+val)
+	} else if val, ok := c.RepoData.Authentication["basic"]; ok {
+		req.HTTPRequest.Header.Set("Authorization", "Basic "+val)
+	}
+
+	return req, err
+}
+
+func Round(input float64) float64 {
+	if input < 0 {
+		return math.Ceil(input - 0.5)
+	}
+	return math.Floor(input + 0.5)
+}
+
 func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+	var u *url.URL = nil
+	var err error
+	var req *grab.Request
+	var temp string
+
 	artifactName := path.Base(artifact.GetPath())
-	Info("Downloading artifact", artifactName, "from", c.RepoData.Uri)
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
+	ok := false
 
-	temp, err := ioutil.TempDir(os.TempDir(), "tree")
-	if err != nil {
-		return nil, err
+	// Check if file is already in cache
+	if helpers.Exists(cacheFile) {
+		Info("Use artifact", artifactName, "from cache.")
+	} else {
+
+		temp, err = ioutil.TempDir(os.TempDir(), "tree")
+		if err != nil {
+			return nil, err
+		}
+		defer os.RemoveAll(temp)
+
+		client := grab.NewClient()
+
+		for _, uri := range c.RepoData.Urls {
+			Info("Downloading artifact", artifactName, "from", uri)
+
+			u, err = url.Parse(uri)
+			if err != nil {
+				continue
+			}
+			u.Path = path.Join(u.Path, artifactName)
+
+			req, err = c.PrepareReq(temp, u.String())
+			if err != nil {
+				continue
+			}
+
+			resp := client.Do(req)
+			if err = resp.Err(); err != nil {
+				continue
+			}
+
+			Info("Downloaded", artifactName, "of",
+				fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
+				fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
+
+			Debug("Copying file ", filepath.Join(temp, artifactName), "to", cacheFile)
+			err = helpers.CopyFile(filepath.Join(temp, artifactName), cacheFile)
+			if err != nil {
+				continue
+			}
+
+			ok = true
+			break
+		}
+
+		if !ok {
+			return nil, err
+		}
 	}
 
-	file, err := ioutil.TempFile(temp, "HttpClient")
-	if err != nil {
-		return nil, err
-	}
-
-	u, err := url.Parse(c.RepoData.Uri)
-	if err != nil {
-		return nil, err
-	}
-	u.Path = path.Join(u.Path, artifactName)
-
-	_, err = grab.Get(temp, u.String())
-	if err != nil {
-		return nil, err
-	}
-
-	err = helpers.CopyFile(filepath.Join(temp, artifactName), file.Name())
-
-	return compiler.NewPackageArtifact(file.Name()), nil
+	newart := artifact
+	newart.SetPath(cacheFile)
+	return newart, nil
 }
 
 func (c *HttpClient) DownloadFile(name string) (string, error) {
-	temp, err := ioutil.TempDir(os.TempDir(), "tree")
-	if err != nil {
-		return "", err
-	}
-	file, err := ioutil.TempFile(os.TempDir(), "HttpClient")
-	if err != nil {
-		return "", err
-	}
-	//defer os.Remove(file.Name())
-	u, err := url.Parse(c.RepoData.Uri)
-	if err != nil {
-		return "", err
-	}
-	u.Path = path.Join(u.Path, name)
+	var file *os.File = nil
+	var u *url.URL = nil
+	var err error
+	var req *grab.Request
+	var temp string
 
-	Info("Downloading", u.String())
+	ok := false
 
-	_, err = grab.Get(temp, u.String())
+	temp, err = ioutil.TempDir(os.TempDir(), "tree")
 	if err != nil {
 		return "", err
 	}
 
-	err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+	client := grab.NewClient()
+
+	for _, uri := range c.RepoData.Urls {
+
+		file, err = ioutil.TempFile(os.TempDir(), "HttpClient")
+		if err != nil {
+			continue
+		}
+
+		u, err = url.Parse(uri)
+		if err != nil {
+			continue
+		}
+		u.Path = path.Join(u.Path, name)
+
+		Info("Downloading", u.String())
+
+		req, err = c.PrepareReq(temp, u.String())
+		if err != nil {
+			continue
+		}
+
+		resp := client.Do(req)
+		if err = resp.Err(); err != nil {
+			continue
+		}
+
+		Info("Downloaded", filepath.Base(resp.Filename), "of",
+			fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
+			fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
+
+		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
+
+	if !ok {
+		return "", err
+	}
 
 	return file.Name(), err
 }

pkg/installer/client/http_test.go

@@ -44,7 +44,7 @@ var _ = Describe("Http client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewHttpClient(RepoData{Uri: ts.URL})
+			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
 			path, err := c.DownloadFile("test.txt")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path)).To(Equal("test"))
@@ -62,7 +62,7 @@ var _ = Describe("Http client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewHttpClient(RepoData{Uri: ts.URL})
+			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
 			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path.GetPath())).To(Equal("test"))

pkg/installer/client/interface.go

@@ -16,5 +16,6 @@
 package client
 
 type RepoData struct {
-	Uri string
-}
+	Urls           []string
+	Authentication map[string]string
+}

pkg/installer/client/local.go

@@ -21,6 +21,7 @@ import (
 	"path"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/mudler/luet/pkg/compiler"
@@ -36,28 +37,62 @@ func NewLocalClient(r RepoData) *LocalClient {
 }
 
 func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+	var err error
+
 	artifactName := path.Base(artifact.GetPath())
-	Info("Downloading artifact", artifactName, "from", c.RepoData.Uri)
-	file, err := ioutil.TempFile(os.TempDir(), "localclient")
-	if err != nil {
-		return nil, err
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
+
+	// Check if file is already in cache
+	if helpers.Exists(cacheFile) {
+		Info("Use artifact", artifactName, "from cache.")
+	} else {
+		ok := false
+		for _, uri := range c.RepoData.Urls {
+			Info("Downloading artifact", artifactName, "from", uri)
+
+			//defer os.Remove(file.Name())
+			err = helpers.CopyFile(filepath.Join(uri, artifactName), cacheFile)
+			if err != nil {
+				continue
+			}
+			ok = true
+			break
+		}
+
+		if !ok {
+			return nil, err
+		}
 	}
-	//defer os.Remove(file.Name())
 
-	err = helpers.CopyFile(filepath.Join(c.RepoData.Uri, artifactName), file.Name())
-
-	return compiler.NewPackageArtifact(file.Name()), nil
+	newart := artifact
+	newart.SetPath(cacheFile)
+	return newart, nil
 }
+
 func (c *LocalClient) DownloadFile(name string) (string, error) {
-	Info("Downloading file", name, "from", c.RepoData.Uri)
+	var err error
+	var file *os.File = nil
 
-	file, err := ioutil.TempFile(os.TempDir(), "localclient")
-	if err != nil {
-		return "", err
+	ok := false
+	for _, uri := range c.RepoData.Urls {
+		Info("Downloading file", name, "from", uri)
+		file, err = ioutil.TempFile(os.TempDir(), "localclient")
+		if err != nil {
+			continue
+		}
+		//defer os.Remove(file.Name())
+
+		err = helpers.CopyFile(filepath.Join(uri, name), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
 	}
-	//defer os.Remove(file.Name())
 
-	err = helpers.CopyFile(filepath.Join(c.RepoData.Uri, name), file.Name())
+	if ok {
+		return file.Name(), nil
+	}
 
-	return file.Name(), err
+	return "", err
 }

pkg/installer/client/local_test.go

@@ -39,7 +39,7 @@ var _ = Describe("Local client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewLocalClient(RepoData{Uri: tmpdir})
+			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
 			path, err := c.DownloadFile("test.txt")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path)).To(Equal("test"))
@@ -55,7 +55,7 @@ var _ = Describe("Local client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewLocalClient(RepoData{Uri: tmpdir})
+			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
 			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path.GetPath())).To(Equal("test"))

pkg/installer/installer.go

@@ -16,8 +16,6 @@
 package installer
 
 import (
-	"archive/tar"
-	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -27,6 +25,7 @@ import (
 
 	"github.com/ghodss/yaml"
 	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -36,9 +35,15 @@ import (
 	"github.com/pkg/errors"
 )
 
+type LuetInstallerOptions struct {
+	SolverOptions config.LuetSolverOptions
+	Concurrency   int
+}
+
 type LuetInstaller struct {
 	PackageRepositories Repositories
-	Concurrency         int
+
+	Options LuetInstallerOptions
 }
 
 type ArtifactMatch struct {
@@ -60,7 +65,7 @@ func (f *LuetFinalizer) RunInstall() error {
 		if err != nil {
 			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
 		}
-		Info(stdoutStderr)
+		Info(string(stdoutStderr))
 	}
 	return nil
 }
@@ -74,7 +79,7 @@ func (f *LuetFinalizer) RunUnInstall() error {
 		if err != nil {
 			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
 		}
-		Info(stdoutStderr)
+		Info(string(stdoutStderr))
 	}
 	return nil
 }
@@ -88,32 +93,21 @@ func NewLuetFinalizerFromYaml(data []byte) (*LuetFinalizer, error) {
 	return &p, err
 }
 
-func NewLuetInstaller(concurrency int) Installer {
-	return &LuetInstaller{Concurrency: concurrency}
+func NewLuetInstaller(opts LuetInstallerOptions) Installer {
+	return &LuetInstaller{Options: opts}
 }
 
 func (l *LuetInstaller) Upgrade(s *System) error {
-	Spinner(32)
-	defer SpinnerStop()
-	syncedRepos := Repositories{}
-	for _, r := range l.PackageRepositories {
-		repo, err := r.Sync()
-		if err != nil {
-			return errors.Wrap(err, "Failed syncing repository: "+r.GetName())
-		}
-		syncedRepos = append(syncedRepos, repo)
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
 	}
-
-	// compute what to install and from where
-	sort.Sort(syncedRepos)
-
 	// First match packages against repositories by priority
 	//	matches := syncedRepos.PackageMatches(p)
-
-	// compute a "big" world
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
-	solv := solver.NewSolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false))
+	// compute a "big" world
+	solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 	uninstall, solution, err := solv.Upgrade()
 	if err != nil {
 		return errors.Wrap(err, "Failed solving solution for upgrade")
@@ -136,16 +130,14 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 	return l.Install(toInstall, s)
 }
 
-func (l *LuetInstaller) Install(p []pkg.Package, s *System) error {
-	// First get metas from all repos (and decodes trees)
-
+func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
 	Spinner(32)
 	defer SpinnerStop()
 	syncedRepos := Repositories{}
 	for _, r := range l.PackageRepositories {
-		repo, err := r.Sync()
+		repo, err := r.Sync(false)
 		if err != nil {
-			return errors.Wrap(err, "Failed syncing repository: "+r.GetName())
+			return nil, errors.Wrap(err, "Failed syncing repository: "+r.GetName())
 		}
 		syncedRepos = append(syncedRepos, repo)
 	}
@@ -153,24 +145,60 @@ func (l *LuetInstaller) Install(p []pkg.Package, s *System) error {
 	// compute what to install and from where
 	sort.Sort(syncedRepos)
 
+	if !inMemory {
+		l.PackageRepositories = syncedRepos
+	}
+
+	return syncedRepos, nil
+}
+
+func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
+	var p []pkg.Package
+
+	// Check if the package is installed first
+	for _, pi := range cp {
+
+		vers, _ := s.Database.FindPackageVersions(pi)
+
+		if len(vers) >= 1 {
+			Warning("Filtering out package " + pi.GetFingerPrint() + ", it has other versions already installed. Uninstall one of them first ")
+			continue
+			//return errors.New("Package " + pi.GetFingerPrint() + " has other versions already installed. Uninstall one of them first: " + strings.Join(vers, " "))
+
+		}
+		p = append(p, pi)
+	}
+
+	if len(p) == 0 {
+		Warning("No package to install, bailing out with no errors")
+		return nil
+	}
+	// First get metas from all repos (and decodes trees)
+
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
 	// First match packages against repositories by priority
 	//	matches := syncedRepos.PackageMatches(p)
 
 	// compute a "big" world
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
+	p = syncedRepos.ResolveSelectors(p)
 
-	solv := solver.NewSolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false))
+	solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 	solution, err := solv.Install(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed solving solution for package")
 	}
+
 	// Gathers things to install
 	toInstall := map[string]ArtifactMatch{}
 	for _, assertion := range solution {
 		if assertion.Value {
 			matches := syncedRepos.PackageMatches([]pkg.Package{assertion.Package})
-			if len(matches) != 1 {
+			if len(matches) == 0 {
 				return errors.New("Failed matching solutions against repository - where are definitions coming from?!")
 			}
 		A:
@@ -194,7 +222,7 @@ func (l *LuetInstaller) Install(p []pkg.Package, s *System) error {
 	all := make(chan ArtifactMatch)
 
 	var wg = new(sync.WaitGroup)
-	for i := 0; i < l.Concurrency; i++ {
+	for i := 0; i < l.Options.Concurrency; i++ {
 		wg.Add(1)
 		go l.installerWorker(i, wg, all, s)
 	}
@@ -264,39 +292,22 @@ func (l *LuetInstaller) Install(p []pkg.Package, s *System) error {
 
 func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
 
-	// FIXME: Implement
 	artifact, err := a.Repository.Client().DownloadArtifact(a.Artifact)
-	defer os.Remove(artifact.GetPath())
+	if err != nil {
+		return errors.Wrap(err, "Error on download artifact")
+	}
 
-	tarFile, err := os.Open(artifact.GetPath())
+	err = artifact.Verify()
+	if err != nil {
+		return errors.Wrap(err, "Artifact integrity check failure")
+	}
+
+	files, err := artifact.FileList()
 	if err != nil {
 		return errors.Wrap(err, "Could not open package archive")
 	}
-	defer tarFile.Close()
-	tr := tar.NewReader(tarFile)
 
-	var files []string
-	// untar each segment
-	for {
-		hdr, err := tr.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return err
-		}
-		// determine proper file path info
-		finfo := hdr.FileInfo()
-		fileName := hdr.Name
-		if finfo.Mode().IsDir() {
-			continue
-		}
-		files = append(files, fileName)
-
-		// if a dir, create it, then go to next segment
-	}
-
-	err = helpers.Untar(artifact.GetPath(), s.Target, true)
+	err = artifact.Unpack(s.Target, true)
 	if err != nil {
 		return errors.Wrap(err, "Error met while unpacking rootfs")
 	}
@@ -353,8 +364,7 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
 	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) - mark the uninstallation in db
 	// Get installed definition
-
-	solv := solver.NewSolver(s.Database, s.Database, pkg.NewInMemoryDatabase(false))
+	solv := solver.NewResolver(s.Database, s.Database, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 	solution, err := solv.Uninstall(p)
 	if err != nil {
 		return errors.Wrap(err, "Uninstall failed")

pkg/installer/installer_suite_test.go

@@ -18,11 +18,19 @@ package installer_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestInstaller(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	// Set temporary directory for rootfs
+	config.LuetCfg.GetSystem().Rootfs = "/tmp/luet-root"
+	// Force dynamic path for packages cache
+	config.LuetCfg.GetSystem().PkgsCachePath = ""
 	RunSpecs(t, "Installer Suite")
 }

pkg/installer/installer_test.go

@@ -34,7 +34,7 @@ import (
 var _ = Describe("Installer", func() {
 	Context("Writes a repository definition", func() {
 		It("Writes a repo and can install packages from it", func() {
-			//repo:=NewLuetRepository()
+			//repo:=NewLuetSystemRepository()
 
 			tmpdir, err := ioutil.TempDir("", "tree")
 			Expect(err).ToNot(HaveOccurred())
@@ -47,7 +47,7 @@ var _ = Describe("Installer", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -62,7 +62,9 @@ var _ = Describe("Installer", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			artifact, err := compiler.Compile(2, false, spec)
+			compiler.SetConcurrency(2)
+
+			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -80,34 +82,35 @@ var _ = Describe("Installer", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", tmpdir, "local", 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir)
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(fakeroot) // clean up
 
-			inst := NewLuetInstaller(1)
-			repo2, err := NewLuetRepositoryFromYaml([]byte(`
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
 name: "test"
-type: "local"
-uri: "`+tmpdir+`"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
 `), pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 
 			inst.Repositories(Repositories{repo2})
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 			systemDB := pkg.NewInMemoryDatabase(false)
 			system := &System{Database: systemDB, Target: fakeroot}
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
@@ -144,7 +147,7 @@ uri: "`+tmpdir+`"
 
 	Context("Installation", func() {
 		It("Installs in a system with a persistent db", func() {
-			//repo:=NewLuetRepository()
+			//repo:=NewLuetSystemRepository()
 
 			tmpdir, err := ioutil.TempDir("", "tree")
 			Expect(err).ToNot(HaveOccurred())
@@ -157,7 +160,7 @@ uri: "`+tmpdir+`"
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -172,7 +175,9 @@ uri: "`+tmpdir+`"
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			artifact, err := compiler.Compile(2, false, spec)
+			compiler.SetConcurrency(2)
+
+			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -190,34 +195,35 @@ uri: "`+tmpdir+`"
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", tmpdir, "local", 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir)
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(fakeroot) // clean up
 
-			inst := NewLuetInstaller(1)
-			repo2, err := NewLuetRepositoryFromYaml([]byte(`
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
 name: "test"
-type: "local"
-uri: "`+tmpdir+`"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
 `), pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 
 			inst.Repositories(Repositories{repo2})
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 
 			bolt, err := ioutil.TempDir("", "db")
 			Expect(err).ToNot(HaveOccurred())
@@ -260,7 +266,7 @@ uri: "`+tmpdir+`"
 
 	Context("Simple upgrades", func() {
 		It("Installs packages and Upgrades a system with a persistent db", func() {
-			//repo:=NewLuetRepository()
+			//repo:=NewLuetSystemRepository()
 
 			tmpdir, err := ioutil.TempDir("", "tree")
 			Expect(err).ToNot(HaveOccurred())
@@ -273,7 +279,7 @@ uri: "`+tmpdir+`"
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -291,38 +297,156 @@ uri: "`+tmpdir+`"
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			_, errs := c.CompileParallel(2, false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+			c.SetConcurrency(2)
+
+			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", tmpdir, "local", 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir)
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(fakeroot) // clean up
 
-			inst := NewLuetInstaller(1)
-			repo2, err := NewLuetRepositoryFromYaml([]byte(`
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
 name: "test"
-type: "local"
-uri: "`+tmpdir+`"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
 `), pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 
 			inst.Repositories(Repositories{repo2})
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.GetPackages())).To(Equal(1))
+			p, err := system.Database.GetPackage(system.Database.GetPackages()[0])
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetName()).To(Equal("b"))
+
+			files, err := systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(files).To(Equal([]string{"artifact42", "test5", "test6"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = inst.Upgrade(system)
+			Expect(err).ToNot(HaveOccurred())
+
+			// Nothing should be there anymore (files, packagedb entry)
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+
+			// New version - new files
+			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+
+			// New package should be there
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+		})
+
+	})
+
+	Context("Compressed packages", func() {
+		It("Installs", func() {
+			//repo:=NewLuetSystemRepository()
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/upgrade")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec3, err := c.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+			spec3.SetOutputPath(tmpdir)
+			c.SetConcurrency(2)
+			c.SetCompressionType(compiler.GZip)
+			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+
+			Expect(errs).To(BeEmpty())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+
+			inst.Repositories(Repositories{repo2})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 
 			bolt, err := ioutil.TempDir("", "db")
 			Expect(err).ToNot(HaveOccurred())

pkg/installer/interface.go

@@ -27,6 +27,7 @@ type Installer interface {
 	Uninstall(pkg.Package, *System) error
 	Upgrade(s *System) error
 	Repositories([]Repository)
+	SyncRepositories(bool) (Repositories, error)
 }
 
 type Client interface {
@@ -38,17 +39,30 @@ type Repositories []Repository
 
 type Repository interface {
 	GetName() string
-	GetUri() string
-	SetUri(string)
+	GetDescription() string
+	GetUrls() []string
+	SetUrls([]string)
+	AddUrl(string)
 	GetPriority() int
 	GetIndex() compiler.ArtifactIndex
 	GetTree() tree.Builder
 	SetTree(tree.Builder)
-	Write(path string) error
-	Sync() (Repository, error)
+	Write(path string, resetRevision bool) error
+	Sync(bool) (Repository, error)
 	GetTreePath() string
 	SetTreePath(string)
 	GetType() string
 	SetType(string)
+	SetAuthentication(map[string]string)
+	GetAuthentication() map[string]string
+	GetRevision() int
+	IncrementRevision()
+	GetLastUpdate() string
+	SetLastUpdate(string)
 	Client() Client
+
+	GetTreeChecksums() compiler.Checksums
+	GetTreeCompressionType() compiler.CompressionImplementation
+	SetTreeCompressionType(c compiler.CompressionImplementation)
+	SetTreeChecksums(c compiler.Checksums)
 }

pkg/installer/repository.go

@@ -16,42 +16,60 @@
 package installer
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
+	"time"
 
-	"github.com/mudler/luet/pkg/installer/client"
-
-	"github.com/ghodss/yaml"
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/installer/client"
+	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/ghodss/yaml"
+	. "github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 )
 
-type LuetRepository struct {
-	Name     string                 `json:"name"`
-	Uri      string                 `json:"uri"`
-	Priority int                    `json:"priority"`
-	Index    compiler.ArtifactIndex `json:"index"`
-	Tree     tree.Builder           `json:"-"`
-	TreePath string                 `json:"-"`
-	Type     string                 `json:"type"`
+const (
+	REPOSITORY_SPECFILE = "repository.yaml"
+	TREE_TARBALL        = "tree.tar"
+)
+
+type LuetSystemRepository struct {
+	*config.LuetRepository
+
+	Index               compiler.ArtifactIndex             `json:"index"`
+	Tree                tree.Builder                       `json:"-"`
+	TreePath            string                             `json:"treepath"`
+	TreeCompressionType compiler.CompressionImplementation `json:"treecompressiontype"`
+	TreeChecksums       compiler.Checksums                 `json:"treechecksums"`
 }
 
-type LuetRepositorySerialized struct {
-	Name     string                      `json:"name"`
-	Uri      string                      `json:"uri"`
-	Priority int                         `json:"priority"`
-	Index    []*compiler.PackageArtifact `json:"index"`
-	Type     string                      `json:"type"`
+type LuetSystemRepositorySerialized struct {
+	Name                string                             `json:"name"`
+	Description         string                             `json:"description,omitempty"`
+	Urls                []string                           `json:"urls"`
+	Priority            int                                `json:"priority"`
+	Index               []*compiler.PackageArtifact        `json:"index"`
+	Type                string                             `json:"type"`
+	Revision            int                                `json:"revision,omitempty"`
+	LastUpdate          string                             `json:"last_update,omitempty"`
+	TreePath            string                             `json:"treepath"`
+	TreeCompressionType compiler.CompressionImplementation `json:"treecompressiontype"`
+	TreeChecksums       compiler.Checksums                 `json:"treechecksums"`
 }
 
-func GenerateRepository(name, uri, t string, priority int, src, treeDir string, db pkg.PackageDatabase) (Repository, error) {
+func GenerateRepository(name, descr, t string, urls []string, priority int, src, treeDir string, db pkg.PackageDatabase) (Repository, error) {
 
 	art, err := buildPackageIndex(src)
 	if err != nil {
@@ -63,24 +81,51 @@ func GenerateRepository(name, uri, t string, priority int, src, treeDir string,
 		return nil, err
 	}
 
-	return NewLuetRepository(name, uri, t, priority, art, tr), nil
+	return NewLuetSystemRepository(
+		config.NewLuetRepository(name, t, descr, urls, priority, true, false),
+		art, tr), nil
 }
 
-func NewLuetRepository(name, uri, t string, priority int, art []compiler.Artifact, builder tree.Builder) Repository {
-	return &LuetRepository{Index: art, Type: t, Tree: builder, Name: name, Uri: uri, Priority: priority}
+func NewSystemRepository(repo config.LuetRepository) Repository {
+	return &LuetSystemRepository{
+		LuetRepository: &repo,
+	}
 }
 
-func NewLuetRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, error) {
-	var p *LuetRepositorySerialized
-	r := &LuetRepository{}
+func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder) Repository {
+	return &LuetSystemRepository{
+		LuetRepository: repo,
+		Index:          art,
+		Tree:           builder,
+	}
+}
+
+func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, error) {
+	var p *LuetSystemRepositorySerialized
 	err := yaml.Unmarshal(data, &p)
 	if err != nil {
 		return nil, err
 	}
-	r.Name = p.Name
-	r.Uri = p.Uri
-	r.Priority = p.Priority
-	r.Type = p.Type
+	r := &LuetSystemRepository{
+		LuetRepository: config.NewLuetRepository(
+			p.Name,
+			p.Type,
+			p.Description,
+			p.Urls,
+			p.Priority,
+			true,
+			false,
+		),
+		TreeCompressionType: p.TreeCompressionType,
+		TreeChecksums:       p.TreeChecksums,
+		TreePath:            p.TreePath,
+	}
+	if p.Revision > 0 {
+		r.Revision = p.Revision
+	}
+	if p.LastUpdate != "" {
+		r.LastUpdate = p.LastUpdate
+	}
 	i := compiler.ArtifactIndex{}
 	for _, ii := range p.Index {
 		i = append(i, ii)
@@ -122,56 +167,129 @@ func buildPackageIndex(path string) ([]compiler.Artifact, error) {
 	return art, nil
 }
 
-func (r *LuetRepository) GetName() string {
-	return r.Name
+func (r *LuetSystemRepository) GetName() string {
+	return r.LuetRepository.Name
 }
-func (r *LuetRepository) GetTreePath() string {
+func (r *LuetSystemRepository) GetDescription() string {
+	return r.LuetRepository.Description
+}
+
+func (r *LuetSystemRepository) GetAuthentication() map[string]string {
+	return r.LuetRepository.Authentication
+}
+
+func (r *LuetSystemRepository) GetTreeCompressionType() compiler.CompressionImplementation {
+	return r.TreeCompressionType
+}
+
+func (r *LuetSystemRepository) GetTreeChecksums() compiler.Checksums {
+	return r.TreeChecksums
+}
+
+func (r *LuetSystemRepository) SetTreeCompressionType(c compiler.CompressionImplementation) {
+	r.TreeCompressionType = c
+}
+
+func (r *LuetSystemRepository) SetTreeChecksums(c compiler.Checksums) {
+	r.TreeChecksums = c
+}
+
+func (r *LuetSystemRepository) GetType() string {
+	return r.LuetRepository.Type
+}
+func (r *LuetSystemRepository) SetType(p string) {
+	r.LuetRepository.Type = p
+}
+func (r *LuetSystemRepository) AddUrl(p string) {
+	r.LuetRepository.Urls = append(r.LuetRepository.Urls, p)
+}
+func (r *LuetSystemRepository) GetUrls() []string {
+	return r.LuetRepository.Urls
+}
+func (r *LuetSystemRepository) SetUrls(urls []string) {
+	r.LuetRepository.Urls = urls
+}
+func (r *LuetSystemRepository) GetPriority() int {
+	return r.LuetRepository.Priority
+}
+func (r *LuetSystemRepository) GetTreePath() string {
 	return r.TreePath
 }
-func (r *LuetRepository) SetTreePath(p string) {
+func (r *LuetSystemRepository) SetTreePath(p string) {
 	r.TreePath = p
 }
-
-func (r *LuetRepository) SetTree(b tree.Builder) {
+func (r *LuetSystemRepository) SetTree(b tree.Builder) {
 	r.Tree = b
 }
-
-func (r *LuetRepository) GetType() string {
-	return r.Type
-}
-func (r *LuetRepository) SetType(p string) {
-	r.Type = p
-}
-
-func (r *LuetRepository) SetUri(p string) {
-	r.Uri = p
-}
-func (r *LuetRepository) GetUri() string {
-	return r.Uri
-}
-func (r *LuetRepository) GetPriority() int {
-	return r.Priority
-}
-func (r *LuetRepository) GetIndex() compiler.ArtifactIndex {
+func (r *LuetSystemRepository) GetIndex() compiler.ArtifactIndex {
 	return r.Index
 }
-func (r *LuetRepository) GetTree() tree.Builder {
+func (r *LuetSystemRepository) GetTree() tree.Builder {
 	return r.Tree
 }
+func (r *LuetSystemRepository) GetRevision() int {
+	return r.LuetRepository.Revision
+}
+func (r *LuetSystemRepository) GetLastUpdate() string {
+	return r.LuetRepository.LastUpdate
+}
+func (r *LuetSystemRepository) SetLastUpdate(u string) {
+	r.LuetRepository.LastUpdate = u
+}
+func (r *LuetSystemRepository) IncrementRevision() {
+	r.LuetRepository.Revision++
+}
 
-func (r *LuetRepository) Write(dst string) error {
+func (r *LuetSystemRepository) SetAuthentication(auth map[string]string) {
+	r.LuetRepository.Authentication = auth
+}
 
-	os.MkdirAll(dst, os.ModePerm)
+func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repository, error) {
+	dat, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading file "+file)
+	}
+	if removeFile {
+		defer os.Remove(file)
+	}
+
+	var repo Repository
+	repo, err = NewLuetSystemRepositoryFromYaml(dat, pkg.NewInMemoryDatabase(false))
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading repository from file "+file)
+	}
+
+	return repo, err
+}
+
+func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
+	err := os.MkdirAll(dst, os.ModePerm)
+	if err != nil {
+		return err
+	}
 	r.Index = r.Index.CleanPath()
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
 
-	data, err := yaml.Marshal(r)
-	if err != nil {
-		return err
-	}
-	err = ioutil.WriteFile(filepath.Join(dst, "repository.yaml"), data, os.ModePerm)
-	if err != nil {
-		return err
+	repospec := filepath.Join(dst, REPOSITORY_SPECFILE)
+	if resetRevision {
+		r.Revision = 0
+	} else {
+		if _, err := os.Stat(repospec); !os.IsNotExist(err) {
+			// Read existing file for retrieve revision
+			spec, err := r.ReadSpecFile(repospec, false)
+			if err != nil {
+				return err
+			}
+			r.Revision = spec.GetRevision()
+		}
 	}
+	r.Revision++
+
+	Info(fmt.Sprintf(
+		"For repository %s creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
 	archive, err := ioutil.TempDir(os.TempDir(), "archive")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for archive")
@@ -181,67 +299,148 @@ func (r *LuetRepository) Write(dst string) error {
 	if err != nil {
 		return errors.Wrap(err, "Error met while saving the tree")
 	}
-	err = helpers.Tar(archive, filepath.Join(dst, "tree.tar"))
+	tpath := r.GetTreePath()
+	if tpath == "" {
+		tpath = TREE_TARBALL
+	}
+
+	a := compiler.NewPackageArtifact(filepath.Join(dst, tpath))
+	a.SetCompressionType(r.TreeCompressionType)
+	err = a.Compress(archive, 1)
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating package archive")
 	}
-	return nil
-}
 
-func (r *LuetRepository) Client() Client {
-	switch r.GetType() {
-	case "local":
-		return client.NewLocalClient(client.RepoData{Uri: r.GetUri()})
-	case "http":
-		return client.NewHttpClient(client.RepoData{Uri: r.GetUri()})
+	r.TreePath = path.Base(a.GetPath())
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	r.TreeChecksums = a.GetChecksums()
+
+	data, err := yaml.Marshal(r)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(repospec, data, os.ModePerm)
+	if err != nil {
+		return err
 	}
 
 	return nil
 }
-func (r *LuetRepository) Sync() (Repository, error) {
+
+func (r *LuetSystemRepository) Client() Client {
+	switch r.GetType() {
+	case "disk":
+		return client.NewLocalClient(client.RepoData{Urls: r.GetUrls()})
+	case "http":
+		return client.NewHttpClient(
+			client.RepoData{
+				Urls:           r.GetUrls(),
+				Authentication: r.GetAuthentication(),
+			})
+	}
+
+	return nil
+}
+func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
+	var repoUpdated bool = false
+	var treefs string
+
+	Debug("Sync of the repository", r.Name, "in progress...")
 	c := r.Client()
 	if c == nil {
 		return nil, errors.New("No client could be generated from repository.")
 	}
-	file, err := c.DownloadFile("repository.yaml")
-	if err != nil {
-		return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri())
-	}
-	dat, err := ioutil.ReadFile(file)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error reading file "+file)
-	}
-	defer os.Remove(file)
 
-	// TODO: make it swappable
-	repo, err := NewLuetRepositoryFromYaml(dat, pkg.NewInMemoryDatabase(false))
+	// Retrieve remote repository.yaml for retrieve revision and date
+	file, err := c.DownloadFile(REPOSITORY_SPECFILE)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error reading repository from file "+file)
-
+		return nil, errors.Wrap(err, "While downloading "+REPOSITORY_SPECFILE)
 	}
 
-	archivetree, err := c.DownloadFile("tree.tar")
+	repobasedir := config.LuetCfg.GetSystem().GetRepoDatabaseDirPath(r.GetName())
+	repo, err := r.ReadSpecFile(file, false)
 	if err != nil {
-		return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri())
+		return nil, err
 	}
-	defer os.RemoveAll(archivetree) // clean up
+	// Remove temporary file that contains repository.html.
+	// Example: /tmp/HttpClient236052003
+	defer os.RemoveAll(file)
 
-	treefs, err := ioutil.TempDir(os.TempDir(), "treefs")
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	if r.Cached {
+		if !force {
+			localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir, REPOSITORY_SPECFILE), false)
+			if localRepo != nil {
+				if localRepo.GetRevision() == repo.GetRevision() &&
+					localRepo.GetLastUpdate() == repo.GetLastUpdate() {
+					repoUpdated = true
+				}
+			}
+		}
+		if r.GetTreePath() == "" {
+			treefs = filepath.Join(repobasedir, "treefs")
+		} else {
+			treefs = r.GetTreePath()
+		}
+
+	} else {
+		treefs, err = ioutil.TempDir(os.TempDir(), "treefs")
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
 	}
-	//defer os.RemoveAll(treefs) // clean up
 
-	// TODO: Following as option if archive as output?
-	// archive, err := ioutil.TempDir(os.TempDir(), "archive")
-	// if err != nil {
-	// 	return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	// }
-	// defer os.RemoveAll(archive) // clean up
+	if !repoUpdated {
+		tpath := repo.GetTreePath()
+		if tpath == "" {
+			tpath = TREE_TARBALL
+		}
+		a := compiler.NewPackageArtifact(tpath)
 
-	err = helpers.Untar(archivetree, treefs, false)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while unpacking rootfs")
+		artifact, err := c.DownloadArtifact(a)
+		if err != nil {
+			return nil, errors.Wrap(err, "While downloading "+tpath)
+		}
+		defer os.Remove(artifact.GetPath())
+
+		artifact.SetChecksums(repo.GetTreeChecksums())
+		artifact.SetCompressionType(repo.GetTreeCompressionType())
+
+		err = artifact.Verify()
+		if err != nil {
+			return nil, errors.Wrap(err, "Tree integrity check failure")
+		}
+
+		Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
+
+		if r.Cached {
+			// Copy updated repository.yaml file to repo dir now that the tree is synced.
+			err = helpers.CopyFile(file, filepath.Join(repobasedir, REPOSITORY_SPECFILE))
+			if err != nil {
+				return nil, errors.Wrap(err, "Error on update "+REPOSITORY_SPECFILE)
+			}
+			// Remove previous tree
+			os.RemoveAll(treefs)
+		}
+		Debug("Decompress tree of the repository " + r.Name + "...")
+
+		err = artifact.Unpack(treefs, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while unpacking tree")
+		}
+
+		tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64)
+
+		InfoC(
+			Bold(Red(":house: Repository "+r.GetName()+" revision: ")).String() +
+				Bold(Green(repo.GetRevision())).String() + " - " +
+				Bold(Green(time.Unix(tsec, 0).String())).String(),
+		)
+
+	} else {
+		Info("Repository", r.GetName(), "is already up to date.")
 	}
 
 	reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
@@ -249,9 +448,11 @@ func (r *LuetRepository) Sync() (Repository, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while unpacking rootfs")
 	}
+
 	repo.SetTree(reciper)
 	repo.SetTreePath(treefs)
-	repo.SetUri(r.GetUri())
+	repo.SetUrls(r.GetUrls())
+	repo.SetAuthentication(r.GetAuthentication())
 
 	return repo, nil
 }
@@ -323,6 +524,29 @@ PACKAGE:
 
 }
 
+func (re Repositories) ResolveSelectors(p []pkg.Package) []pkg.Package {
+	// If a selector is given, get the best from each repo
+	sort.Sort(re) // respect prio
+	var matches []pkg.Package
+PACKAGE:
+	for _, pack := range p {
+		for _, r := range re {
+			if pack.IsSelector() {
+				c, err := r.GetTree().GetDatabase().FindPackageCandidate(pack)
+				if err == nil {
+					matches = append(matches, c)
+					continue PACKAGE
+				}
+			} else {
+				matches = append(matches, pack)
+			}
+		}
+	}
+
+	return matches
+
+}
+
 func (re Repositories) Search(s string) []PackageMatch {
 	sort.Sort(re)
 	var term = regexp.MustCompile(s)

pkg/installer/repository_test.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
+	config "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -46,7 +47,7 @@ var _ = Describe("Repository", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -61,7 +62,9 @@ var _ = Describe("Repository", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			artifact, err := compiler.Compile(2, false, spec)
+			compiler.SetConcurrency(1)
+
+			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -79,16 +82,16 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", tmpdir, "local", 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir)
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
 		})
 	})
 	Context("Matching packages", func() {
@@ -103,8 +106,8 @@ var _ = Describe("Repository", func() {
 
 			_, err = builder2.GetDatabase().CreatePackage(package2)
 			Expect(err).ToNot(HaveOccurred())
-			repo1 := &LuetRepository{Name: "test1", Tree: builder1}
-			repo2 := &LuetRepository{Name: "test2", Tree: builder2}
+			repo1 := &LuetSystemRepository{LuetRepository: &config.LuetRepository{Name: "test1"}, Tree: builder1}
+			repo2 := &LuetSystemRepository{LuetRepository: &config.LuetRepository{Name: "test2"}, Tree: builder2}
 			repositories := Repositories{repo1, repo2}
 			matches := repositories.PackageMatches([]pkg.Package{package1})
 			Expect(matches).To(Equal([]PackageMatch{{Repo: repo1, Package: package1}}))

pkg/logger/logger.go

@@ -3,17 +3,54 @@ package logger
 import (
 	"fmt"
 	"os"
-	"time"
+
+	. "github.com/mudler/luet/pkg/config"
 
 	"github.com/briandowns/spinner"
 	"github.com/kyokomi/emoji"
 	. "github.com/logrusorgru/aurora"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
-var s *spinner.Spinner = spinner.New(spinner.CharSets[22], 100*time.Millisecond)
+var s *spinner.Spinner = nil
+var z *zap.Logger = nil
 
-// TODO: handle this from configuration
-var debug = false
+func NewSpinner() {
+	if s == nil {
+		s = spinner.New(
+			spinner.CharSets[LuetCfg.GetGeneral().SpinnerCharset],
+			LuetCfg.GetGeneral().GetSpinnerMs())
+	}
+}
+
+func ZapLogger() error {
+	var err error
+	if z == nil {
+		// TODO: test permission for open logfile.
+		cfg := zap.NewProductionConfig()
+		cfg.OutputPaths = []string{LuetCfg.GetLogging().Path}
+		cfg.Level = level2AtomicLevel(LuetCfg.GetLogging().Level)
+		cfg.ErrorOutputPaths = []string{}
+		if LuetCfg.GetLogging().JsonFormat {
+			cfg.Encoding = "json"
+		} else {
+			cfg.Encoding = "console"
+		}
+		cfg.DisableCaller = true
+		cfg.DisableStacktrace = true
+		cfg.EncoderConfig.TimeKey = "time"
+		cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+
+		z, err = cfg.Build()
+		if err != nil {
+			fmt.Fprint(os.Stderr, "Error on initialize file logger: "+err.Error()+"\n")
+			return err
+		}
+	}
+
+	return nil
+}
 
 func Spinner(i int) {
 
@@ -21,7 +58,7 @@ func Spinner(i int) {
 		i = 43
 	}
 
-	if !debug && !s.Active() {
+	if !LuetCfg.GetGeneral().Debug && !s.Active() {
 		//	s.UpdateCharSet(spinner.CharSets[i])
 		s.Start() // Start the spinner
 	}
@@ -30,7 +67,7 @@ func Spinner(i int) {
 func SpinnerText(suffix, prefix string) {
 	s.Lock()
 	defer s.Unlock()
-	if debug {
+	if LuetCfg.GetGeneral().Debug {
 		fmt.Println(fmt.Sprintf("%s %s",
 			Bold(Cyan(prefix)).String(),
 			Bold(Magenta(suffix)).BgBlack().String(),
@@ -42,59 +79,119 @@ func SpinnerText(suffix, prefix string) {
 }
 
 func SpinnerStop() {
-	if !debug {
+	if !LuetCfg.GetGeneral().Debug {
 		s.Stop()
 	}
 }
 
-func msg(level string, msg ...interface{}) {
+func level2Number(level string) int {
+	switch level {
+	case "error":
+		return 0
+	case "warning":
+		return 1
+	case "info":
+		return 2
+	default:
+		return 3
+	}
+}
+
+func log2File(level, msg string) {
+	switch level {
+	case "error":
+		z.Error(msg)
+	case "warning":
+		z.Warn(msg)
+	case "info":
+		z.Info(msg)
+	default:
+		z.Debug(msg)
+	}
+}
+
+func level2AtomicLevel(level string) zap.AtomicLevel {
+	switch level {
+	case "error":
+		return zap.NewAtomicLevelAt(zap.ErrorLevel)
+	case "warning":
+		return zap.NewAtomicLevelAt(zap.WarnLevel)
+	case "info":
+		return zap.NewAtomicLevelAt(zap.InfoLevel)
+	default:
+		return zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+}
+
+func msg(level string, withoutColor bool, msg ...interface{}) {
 	var message string
+	var confLevel, msgLevel int
+
+	if LuetCfg.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = level2Number(LuetCfg.GetLogging().Level)
+	}
+	msgLevel = level2Number(level)
+	if msgLevel > confLevel {
+		return
+	}
+
 	for _, m := range msg {
 		message += " " + fmt.Sprintf("%v", m)
 	}
 
 	var levelMsg string
-	switch level {
-	case "warning":
-		levelMsg = Bold(Yellow(":construction: " + message)).BgBlack().String()
-	case "debug":
-		levelMsg = White(message).BgBlack().String()
-	case "info":
-		levelMsg = Bold(White(message)).BgBlack().String()
-	case "error":
-		levelMsg = Bold(Red(":bomb: " + message + ":fire:")).BgBlack().String()
+
+	if withoutColor {
+		levelMsg = message
+	} else {
+		switch level {
+		case "warning":
+			levelMsg = Bold(Yellow(":construction: " + message)).BgBlack().String()
+		case "debug":
+			levelMsg = White(message).BgBlack().String()
+		case "info":
+			levelMsg = Bold(White(message)).BgBlack().String()
+		case "error":
+			levelMsg = Bold(Red(":bomb: " + message + ":fire:")).BgBlack().String()
+		}
 	}
 
 	levelMsg = emoji.Sprint(levelMsg)
 
-	//if s.Active() {
-	//		SpinnerText(levelMsg, "")
-	//	return
-	//	}
-
-	cmd := []interface{}{}
-	for _, f := range msg {
-		cmd = append(cmd, f)
+	if z != nil {
+		log2File(level, message)
 	}
 
 	fmt.Println(levelMsg)
-	//fmt.Println(cmd...)
 }
 
 func Warning(mess ...interface{}) {
-	msg("warning", mess...)
+	msg("warning", false, mess...)
+	if LuetCfg.GetGeneral().FatalWarns {
+		os.Exit(2)
+	}
 }
 
 func Debug(mess ...interface{}) {
-	msg("debug", mess...)
+	msg("debug", false, mess...)
+}
+
+func DebugC(mess ...interface{}) {
+	msg("debug", true, mess...)
 }
 
 func Info(mess ...interface{}) {
-	msg("info", mess...)
+	msg("info", false, mess...)
+}
+
+func InfoC(mess ...interface{}) {
+	msg("info", true, mess...)
 }
 
 func Error(mess ...interface{}) {
-	msg("error", mess...)
+	msg("error", false, mess...)
 }
 
 func Fatal(mess ...interface{}) {

pkg/package/database.go

@@ -41,7 +41,7 @@ type PackageSet interface {
 	GetPackageFiles(Package) ([]string, error)
 	SetPackageFiles(*PackageFile) error
 	RemovePackageFiles(Package) error
-
+	FindPackageVersions(p Package) ([]Package, error)
 	World() []Package
 
 	FindPackageCandidate(p Package) (Package, error)

pkg/package/database_boltdb.go

@@ -22,7 +22,6 @@ import (
 	"sync"
 	"time"
 
-	version "github.com/hashicorp/go-version"
 	"github.com/pkg/errors"
 
 	storm "github.com/asdine/storm"
@@ -34,7 +33,8 @@ import (
 
 type BoltDatabase struct {
 	sync.Mutex
-	Path string
+	Path             string
+	ProvidesDatabase map[string]map[string]Package
 }
 
 func NewBoltDatabase(path string) PackageDatabase {
@@ -42,7 +42,7 @@ func NewBoltDatabase(path string) PackageDatabase {
 	// 	BoltInstance = &BoltDatabase{Path: path}
 	// }
 	//return BoltInstance, nil
-	return &BoltDatabase{Path: path}
+	return &BoltDatabase{Path: path, ProvidesDatabase: map[string]map[string]Package{}}
 }
 
 func (db *BoltDatabase) Get(s string) (string, error) {
@@ -85,6 +85,11 @@ func (db *BoltDatabase) Retrieve(ID string) ([]byte, error) {
 }
 
 func (db *BoltDatabase) FindPackage(tofind Package) (Package, error) {
+	// Provides: Return the replaced package here
+	if provided, err := db.getProvide(tofind); err == nil {
+		return provided, nil
+	}
+
 	p := &DefaultPackage{}
 	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
@@ -196,9 +201,57 @@ func (db *BoltDatabase) CreatePackage(p Package) (string, error) {
 		return "", errors.Wrap(err, "Error saving package to "+db.Path)
 	}
 
+	// Create extra cache between package -> []versions
+	db.Lock()
+	defer db.Unlock()
+	// TODO: Replace with a bolt implementation (and not in memory)
+	// Provides: Store package provides, we will reuse this when walking deps
+	for _, provide := range dp.Provides {
+		if _, ok := db.ProvidesDatabase[provide.GetPackageName()]; !ok {
+			db.ProvidesDatabase[provide.GetPackageName()] = make(map[string]Package)
+
+		}
+
+		db.ProvidesDatabase[provide.GetPackageName()][provide.GetVersion()] = p
+	}
+
 	return strconv.Itoa(dp.ID), err
 }
 
+// Dup from memory implementation
+func (db *BoltDatabase) getProvide(p Package) (Package, error) {
+	db.Lock()
+	pa, ok := db.ProvidesDatabase[p.GetPackageName()][p.GetVersion()]
+	if !ok {
+		versions, ok := db.ProvidesDatabase[p.GetPackageName()]
+		db.Unlock()
+
+		if !ok {
+			return nil, errors.New("No versions found for package")
+		}
+
+		for ve, _ := range versions {
+
+			match, err := p.VersionMatchSelector(ve)
+			if err != nil {
+				return nil, errors.Wrap(err, "Error on match version")
+			}
+			if match {
+				pa, ok := db.ProvidesDatabase[p.GetPackageName()][ve]
+				if !ok {
+					return nil, errors.New("No versions found for package")
+				}
+				return pa, nil //pick the first (we shouldn't have providers that are conflicting)
+				// TODO: A find dbcall here would recurse, but would give chance to have providers of providers
+			}
+		}
+
+		return nil, errors.New("No package provides this")
+	}
+	db.Unlock()
+	return db.FindPackage(pa)
+}
+
 func (db *BoltDatabase) Clean() error {
 	db.Lock()
 	defer db.Unlock()
@@ -295,26 +348,39 @@ func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
 
 }
 
-// FindPackages return the list of the packages beloging to cat/name (any versions)
+// FindPackages return the list of the packages beloging to cat/name  (any versions in requested range)
 // FIXME: Optimize, see inmemorydb
 func (db *BoltDatabase) FindPackages(p Package) ([]Package, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
 	var versionsInWorld []Package
 	for _, w := range db.World() {
 		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
 			continue
 		}
 
-		v, err := version.NewVersion(w.GetVersion())
+		match, err := p.SelectorMatchVersion(w.GetVersion())
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Error on match selector")
 		}
-		constraints, err := version.NewConstraint(p.GetVersion())
-		if err != nil {
-			return nil, err
-		}
-		if constraints.Check(v) {
+		if match {
 			versionsInWorld = append(versionsInWorld, w)
 		}
 	}
 	return versionsInWorld, nil
 }
+
+// FindPackageVersions return the list of the packages beloging to cat/name
+func (db *BoltDatabase) FindPackageVersions(p Package) ([]Package, error) {
+	var versionsInWorld []Package
+	for _, w := range db.World() {
+		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
+			continue
+		}
+
+		versionsInWorld = append(versionsInWorld, w)
+	}
+	return versionsInWorld, nil
+}

pkg/package/database_mem.go

@@ -20,32 +20,34 @@ import (
 	"encoding/json"
 	"sync"
 
-	version "github.com/hashicorp/go-version"
 	"github.com/pkg/errors"
 )
 
 var DBInMemoryInstance = &InMemoryDatabase{
-	Mutex:          &sync.Mutex{},
-	FileDatabase:   map[string][]string{},
-	Database:       map[string]string{},
-	CacheNoVersion: map[string]map[string]interface{}{},
+	Mutex:            &sync.Mutex{},
+	FileDatabase:     map[string][]string{},
+	Database:         map[string]string{},
+	CacheNoVersion:   map[string]map[string]interface{}{},
+	ProvidesDatabase: map[string]map[string]Package{},
 }
 
 type InMemoryDatabase struct {
 	*sync.Mutex
-	Database       map[string]string
-	FileDatabase   map[string][]string
-	CacheNoVersion map[string]map[string]interface{}
+	Database         map[string]string
+	FileDatabase     map[string][]string
+	CacheNoVersion   map[string]map[string]interface{}
+	ProvidesDatabase map[string]map[string]Package
 }
 
 func NewInMemoryDatabase(singleton bool) PackageDatabase {
 	// In memoryDB is a singleton
 	if !singleton {
 		return &InMemoryDatabase{
-			Mutex:          &sync.Mutex{},
-			FileDatabase:   map[string][]string{},
-			Database:       map[string]string{},
-			CacheNoVersion: map[string]map[string]interface{}{},
+			Mutex:            &sync.Mutex{},
+			FileDatabase:     map[string][]string{},
+			Database:         map[string]string{},
+			CacheNoVersion:   map[string]map[string]interface{}{},
+			ProvidesDatabase: map[string]map[string]Package{},
 		}
 	}
 	return DBInMemoryInstance
@@ -142,6 +144,17 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 	// Create extra cache between package -> []versions
 	db.Lock()
 	defer db.Unlock()
+
+	// Provides: Store package provides, we will reuse this when walking deps
+	for _, provide := range pd.Provides {
+		if _, ok := db.ProvidesDatabase[provide.GetPackageName()]; !ok {
+			db.ProvidesDatabase[provide.GetPackageName()] = make(map[string]Package)
+
+		}
+
+		db.ProvidesDatabase[provide.GetPackageName()][provide.GetVersion()] = p
+	}
+
 	_, ok = db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
 		db.CacheNoVersion[p.GetPackageName()] = make(map[string]interface{})
@@ -151,6 +164,38 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 	return ID, nil
 }
 
+func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
+	db.Lock()
+	pa, ok := db.ProvidesDatabase[p.GetPackageName()][p.GetVersion()]
+	if !ok {
+		versions, ok := db.ProvidesDatabase[p.GetPackageName()]
+		db.Unlock()
+
+		if !ok {
+			return nil, errors.New("No versions found for package")
+		}
+
+		for ve, _ := range versions {
+
+			match, err := p.VersionMatchSelector(ve)
+			if err != nil {
+				return nil, errors.Wrap(err, "Error on match version")
+			}
+			if match {
+				pa, ok := db.ProvidesDatabase[p.GetPackageName()][ve]
+				if !ok {
+					return nil, errors.New("No versions found for package")
+				}
+				return pa, nil
+			}
+		}
+
+		return nil, errors.New("No package provides this")
+	}
+	db.Unlock()
+	return db.FindPackage(pa)
+}
+
 func (db *InMemoryDatabase) encodePackage(p Package) (string, string, error) {
 	pd, ok := p.(*DefaultPackage)
 	if !ok {
@@ -167,26 +212,51 @@ func (db *InMemoryDatabase) encodePackage(p Package) (string, string, error) {
 }
 
 func (db *InMemoryDatabase) FindPackage(p Package) (Package, error) {
+
+	// Provides: Return the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		return provided, nil
+	}
+
 	return db.GetPackage(p.GetFingerPrint())
 }
 
-// FindPackages return the list of the packages beloging to cat/name (any versions)
-func (db *InMemoryDatabase) FindPackages(p Package) ([]Package, error) {
+// FindPackages return the list of the packages beloging to cat/name
+func (db *InMemoryDatabase) FindPackageVersions(p Package) ([]Package, error) {
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
 		return nil, errors.New("No versions found for package")
 	}
 	var versionsInWorld []Package
 	for ve, _ := range versions {
-		v, err := version.NewVersion(ve)
+		w, err := db.FindPackage(&DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")
 		}
-		constraints, err := version.NewConstraint(p.GetVersion())
+		versionsInWorld = append(versionsInWorld, w)
+	}
+	return versionsInWorld, nil
+}
+
+// FindPackages return the list of the packages beloging to cat/name (any versions in requested range)
+func (db *InMemoryDatabase) FindPackages(p Package) ([]Package, error) {
+
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+	versions, ok := db.CacheNoVersion[p.GetPackageName()]
+	if !ok {
+		return nil, errors.New("No versions found for package")
+	}
+	var versionsInWorld []Package
+	for ve, _ := range versions {
+		match, err := p.SelectorMatchVersion(ve)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Error on match selector")
 		}
-		if constraints.Check(v) {
+
+		if match {
 			w, err := db.FindPackage(&DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
 			if err != nil {
 				return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")

pkg/package/database_test.go

@@ -76,6 +76,52 @@ var _ = Describe("Database", func() {
 			Expect(pack).To(Equal(a3))
 
 		})
+
+		It("Find specific package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Provides replaces definitions", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+			a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+			Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+
+			s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackage(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+		})
 	})
 
 })

pkg/package/package.go

@@ -21,13 +21,13 @@ import (
 	"fmt"
 	"path/filepath"
 	"sort"
+	"strings"
 
 	//	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/crillab/gophersat/bf"
 	version "github.com/hashicorp/go-version"
 	"github.com/jinzhu/copier"
-	"github.com/pkg/errors"
 
 	"github.com/ghodss/yaml"
 )
@@ -46,6 +46,9 @@ type Package interface {
 	Conflicts([]*DefaultPackage) Package
 	Revdeps(PackageDatabase) []Package
 
+	GetProvides() []*DefaultPackage
+	SetProvides([]*DefaultPackage) Package
+
 	GetRequires() []*DefaultPackage
 	GetConflicts() []*DefaultPackage
 	Expand(PackageDatabase) ([]Package, error)
@@ -57,6 +60,7 @@ type Package interface {
 	GetVersion() string
 	RequiresContains(PackageDatabase, Package) (bool, error)
 	Matches(m Package) bool
+	Bigger(m Package) bool
 
 	AddUse(use string)
 	RemoveUse(use string)
@@ -68,6 +72,21 @@ type Package interface {
 	SetPath(string)
 	GetPath() string
 	Rel(string) string
+
+	GetDescription() string
+	SetDescription(string)
+
+	AddURI(string)
+	GetURI() []string
+
+	SetLicense(string)
+	GetLicense() string
+
+	IsSelector() bool
+	VersionMatchSelector(string) (bool, error)
+	SelectorMatchVersion(string) (bool, error)
+
+	String() string
 }
 
 type Tree interface {
@@ -111,20 +130,25 @@ func (t *DefaultPackage) JSON() ([]byte, error) {
 
 // DefaultPackage represent a standard package definition
 type DefaultPackage struct {
-	ID               int      `json:"-" storm:"id,increment"` // primary key with auto increment
-	Name             string   `json:"name"`                   // Affects YAML field names too.
-	Version          string   `json:"version"`                // Affects YAML field names too.
-	Category         string   `json:"category"`               // Affects YAML field names too.
-	UseFlags         []string `json:"use_flags"`              // Affects YAML field names too.
+	ID               int      `storm:"id,increment" json:"id"` // primary key with auto increment
+	Name             string   `json:"name"`                    // Affects YAML field names too.
+	Version          string   `json:"version"`                 // Affects YAML field names too.
+	Category         string   `json:"category"`                // Affects YAML field names too.
+	UseFlags         []string `json:"use_flags"`               // Affects YAML field names too.
 	State            State
 	PackageRequires  []*DefaultPackage `json:"requires"`  // Affects YAML field names too.
 	PackageConflicts []*DefaultPackage `json:"conflicts"` // Affects YAML field names too.
 	IsSet            bool              `json:"set"`       // Affects YAML field names too.
+	Provides         []*DefaultPackage `json:"provides"`  // Affects YAML field names too.
 
 	// TODO: Annotations?
 
 	// Path is set only internally when tree is loaded from disk
 	Path string `json:"path,omitempty"`
+
+	Description string   `json:"description"`
+	Uri         []string `json:"uri"`
+	License     string   `json:"license"`
 }
 
 // State represent the package state
@@ -138,7 +162,7 @@ func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*D
 func (p *DefaultPackage) String() string {
 	b, err := p.JSON()
 	if err != nil {
-		return fmt.Sprintf("{ id: \"%d\", name: \"%s\" }", p.ID, p.Name)
+		return fmt.Sprintf("{ id: \"%d\", name: \"%s\", version: \"%s\", category: \"%s\"  }", p.ID, p.Name, p.Version, p.Category)
 	}
 	return fmt.Sprintf("%s", string(b))
 }
@@ -149,6 +173,16 @@ func (p *DefaultPackage) GetFingerPrint() string {
 	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version)
 }
 
+func FromString(s string) Package {
+	var unescaped DefaultPackage
+
+	err := json.Unmarshal([]byte(s), &unescaped)
+	if err != nil {
+		return &unescaped
+	}
+	return &unescaped
+}
+
 func (p *DefaultPackage) GetPackageName() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.Category)
 }
@@ -166,6 +200,10 @@ func (p *DefaultPackage) SetPath(s string) {
 	p.Path = s
 }
 
+func (p *DefaultPackage) IsSelector() bool {
+	return strings.ContainsAny(p.GetVersion(), "<>=")
+}
+
 // AddUse adds a use to a package
 func (p *DefaultPackage) AddUse(use string) {
 	for _, v := range p.UseFlags {
@@ -223,18 +261,40 @@ func (p *DefaultPackage) GetName() string {
 func (p *DefaultPackage) GetVersion() string {
 	return p.Version
 }
-
+func (p *DefaultPackage) GetDescription() string {
+	return p.Description
+}
+func (p *DefaultPackage) SetDescription(s string) {
+	p.Description = s
+}
+func (p *DefaultPackage) GetLicense() string {
+	return p.License
+}
+func (p *DefaultPackage) SetLicense(s string) {
+	p.License = s
+}
+func (p *DefaultPackage) AddURI(s string) {
+	p.Uri = append(p.Uri, s)
+}
+func (p *DefaultPackage) GetURI() []string {
+	return p.Uri
+}
 func (p *DefaultPackage) GetCategory() string {
 	return p.Category
 }
-
 func (p *DefaultPackage) SetCategory(s string) {
 	p.Category = s
 }
 func (p *DefaultPackage) GetUses() []string {
 	return p.UseFlags
 }
-
+func (p *DefaultPackage) GetProvides() []*DefaultPackage {
+	return p.Provides
+}
+func (p *DefaultPackage) SetProvides(req []*DefaultPackage) Package {
+	p.Provides = req
+	return p
+}
 func (p *DefaultPackage) GetRequires() []*DefaultPackage {
 	return p.PackageRequires
 }
@@ -254,13 +314,19 @@ func (p *DefaultPackage) Clone() Package {
 	copier.Copy(&new, &p)
 	return new
 }
-
 func (p *DefaultPackage) Matches(m Package) bool {
 	if p.GetFingerPrint() == m.GetFingerPrint() {
 		return true
 	}
 	return false
 }
+func (p *DefaultPackage) Bigger(m Package) bool {
+	low := Lower([]Package{p, m})
+	if low.Matches(m) {
+		return true
+	}
+	return false
+}
 
 func (p *DefaultPackage) Expand(definitiondb PackageDatabase) ([]Package, error) {
 	var versionsInWorld []Package
@@ -270,15 +336,11 @@ func (p *DefaultPackage) Expand(definitiondb PackageDatabase) ([]Package, error)
 		return nil, err
 	}
 	for _, w := range all {
-		v, err := version.NewVersion(w.GetVersion())
+		match, err := p.SelectorMatchVersion(w.GetVersion())
 		if err != nil {
 			return nil, err
 		}
-		constraints, err := version.NewConstraint(p.GetVersion())
-		if err != nil {
-			return nil, err
-		}
-		if constraints.Check(v) {
+		if match {
 			versionsInWorld = append(versionsInWorld, w)
 		}
 	}
@@ -332,7 +394,29 @@ func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Pac
 
 	return false, nil
 }
+func Lower(set []Package) Package {
+	var versionsMap map[string]Package = make(map[string]Package)
+	if len(set) == 0 {
+		panic("Best needs a list with elements")
+	}
 
+	versionsRaw := []string{}
+	for _, p := range set {
+		versionsRaw = append(versionsRaw, p.GetVersion())
+		versionsMap[p.GetVersion()] = p
+	}
+
+	versions := make([]*version.Version, len(versionsRaw))
+	for i, raw := range versionsRaw {
+		v, _ := version.NewVersion(raw)
+		versions[i] = v
+	}
+
+	// After this, the versions are properly sorted
+	sort.Sort(version.Collection(versions))
+
+	return versionsMap[versions[0].Original()]
+}
 func Best(set []Package) Package {
 	var versionsMap map[string]Package = make(map[string]Package)
 	if len(set) == 0 {
@@ -358,7 +442,6 @@ func Best(set []Package) Package {
 }
 
 func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db PackageDatabase) ([]bf.Formula, error) {
-	// TODO: Expansion needs to go here - and so we ditch Resolvedeps()
 	p, err := definitiondb.FindPackage(pack)
 	if err != nil {
 		p = pack // Relax failures and trust the def
@@ -372,9 +455,72 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 
 	var formulas []bf.Formula
 	for _, requiredDef := range p.GetRequires() {
-		required, err := definitiondb.FindPackageCandidate(requiredDef)
-		if err != nil {
-			return nil, errors.Wrap(err, "Couldn't find required package in db definition")
+		required, err := definitiondb.FindPackage(requiredDef)
+		if err != nil || requiredDef.IsSelector() {
+			if err == nil {
+				requiredDef = required.(*DefaultPackage)
+			}
+
+			packages, err := definitiondb.FindPackages(requiredDef)
+			if err != nil || len(packages) == 0 {
+				required = requiredDef
+			} else {
+				if len(packages) == 1 {
+					required = packages[0]
+				} else {
+					var ALO, priorityConstraints, priorityALO []bf.Formula
+
+					// Try to prio best match
+					// Force the solver to consider first our candidate (if does exists).
+					// Then builds ALO and AMO for the requires.
+					c, candidateErr := definitiondb.FindPackageCandidate(requiredDef)
+					var C bf.Formula
+					if candidateErr == nil {
+						// We have a desired candidate, try to look a solution with that included first
+						for _, o := range packages {
+							encodedB, err := o.Encode(db)
+							if err != nil {
+								return nil, err
+							}
+							B := bf.Var(encodedB)
+							if !o.Matches(c) {
+								priorityConstraints = append(priorityConstraints, bf.Not(B))
+								priorityALO = append(priorityALO, B)
+							}
+						}
+						encodedC, err := c.Encode(db)
+						if err != nil {
+							return nil, err
+						}
+						C = bf.Var(encodedC)
+						// Or the Candidate is true, or all the others might be not true
+						// This forces the CDCL sat implementation to look first at a solution with C=true
+						formulas = append(formulas, bf.Or(bf.Or(C, bf.Or(priorityConstraints...)), bf.Or(bf.Not(C), bf.Or(priorityALO...))))
+					}
+
+					// AMO - At most one
+					for _, o := range packages {
+						encodedB, err := o.Encode(db)
+						if err != nil {
+							return nil, err
+						}
+						B := bf.Var(encodedB)
+						ALO = append(ALO, B)
+						for _, i := range packages {
+							encodedI, err := i.Encode(db)
+							if err != nil {
+								return nil, err
+							}
+							I := bf.Var(encodedI)
+							if !o.Matches(i) {
+								formulas = append(formulas, bf.Or(bf.Not(I), bf.Not(B)))
+							}
+						}
+					}
+					formulas = append(formulas, bf.Or(ALO...)) // ALO - At least one
+					continue
+				}
+			}
 		}
 
 		encodedB, err := required.Encode(db)
@@ -394,30 +540,37 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 
 	for _, requiredDef := range p.GetConflicts() {
 		required, err := definitiondb.FindPackage(requiredDef)
-		if err != nil {
-			packages, err := requiredDef.Expand(definitiondb)
+		if err != nil || requiredDef.IsSelector() {
+			if err == nil {
+				requiredDef = required.(*DefaultPackage)
+			}
+			packages, err := definitiondb.FindPackages(requiredDef)
 			if err != nil || len(packages) == 0 {
 				required = requiredDef
 			} else {
-				for _, p := range packages {
-					encodedB, err := p.Encode(db)
-					if err != nil {
-						return nil, err
-					}
-					B := bf.Var(encodedB)
-					formulas = append(formulas, bf.Or(bf.Not(A),
-						bf.Not(B)))
+				if len(packages) == 1 {
+					required = packages[0]
+				} else {
+					for _, p := range packages {
+						encodedB, err := p.Encode(db)
+						if err != nil {
+							return nil, err
+						}
+						B := bf.Var(encodedB)
+						formulas = append(formulas, bf.Or(bf.Not(A),
+							bf.Not(B)))
 
-					f, err := p.BuildFormula(definitiondb, db)
-					if err != nil {
-						return nil, err
+						f, err := p.BuildFormula(definitiondb, db)
+						if err != nil {
+							return nil, err
+						}
+						formulas = append(formulas, f...)
 					}
-					formulas = append(formulas, f...)
+					continue
 				}
 			}
 		}
 
-		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
 		encodedB, err := required.Encode(db)
 		if err != nil {
 			return nil, err

pkg/package/package_suite_test.go

@@ -18,11 +18,15 @@ package pkg_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Package Suite")
 }

pkg/package/package_test.go

@@ -23,6 +23,15 @@ import (
 
 var _ = Describe("Package", func() {
 
+	Context("Encoding/Decoding", func() {
+		a := &DefaultPackage{Name: "test", Version: "1", Category: "t"}
+		It("Encodes and decodes correctly", func() {
+			Expect(a.String()).ToNot(Equal(""))
+			p := FromString(a.String())
+			Expect(p).To(Equal(a))
+		})
+	})
+
 	Context("Simple package", func() {
 		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
@@ -45,6 +54,35 @@ var _ = Describe("Package", func() {
 		})
 	})
 
+	Context("Check description", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.SetDescription("Description A")
+
+		It("Set and get correctly a description", func() {
+			Expect(a.GetDescription()).To(Equal("Description A"))
+		})
+	})
+
+	Context("Check licenses", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.SetLicense("MIT")
+
+		It("Set and get correctly a license", func() {
+			Expect(a.GetLicense()).To(Equal("MIT"))
+		})
+	})
+
+	Context("Check URI", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.AddURI("ftp://ftp.freeradius.org/pub/radius/freearadius-server-3.0.20.tar.gz")
+
+		It("Set and get correctly an uri", func() {
+			Expect(a.GetURI()).To(Equal([]string{
+				"ftp://ftp.freeradius.org/pub/radius/freearadius-server-3.0.20.tar.gz",
+			}))
+		})
+	})
+
 	Context("revdeps", func() {
 		a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		b := NewPackage("B", "1.0", []*DefaultPackage{a}, []*DefaultPackage{})
@@ -100,6 +138,9 @@ var _ = Describe("Package", func() {
 			Expect(a01.RequiresContains(definitions, a11)).To(BeTrue())
 			Expect(a01.RequiresContains(definitions, a)).To(BeTrue())
 			Expect(a.RequiresContains(definitions, a11)).ToNot(BeTrue())
+			Expect(a.IsSelector()).To(BeTrue())
+			Expect(a1.IsSelector()).To(BeFalse())
+
 		})
 	})
 

pkg/package/version.go

@@ -0,0 +1,333 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>,
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	version "github.com/hashicorp/go-version"
+)
+
+// Package Selector Condition
+type PkgSelectorCondition int
+
+type PkgVersionSelector struct {
+	Version       string
+	VersionSuffix string
+	Condition     PkgSelectorCondition
+	// TODO: Integrate support for multiple repository
+}
+
+const (
+	PkgCondInvalid = 0
+	// >
+	PkgCondGreater = 1
+	// >=
+	PkgCondGreaterEqual = 2
+	// <
+	PkgCondLess = 3
+	// <=
+	PkgCondLessEqual = 4
+	// =
+	PkgCondEqual = 5
+	// !
+	PkgCondNot = 6
+	// ~
+	PkgCondAnyRevision = 7
+	// =<pkg>*
+	PkgCondMatchVersion = 8
+)
+
+func PkgSelectorConditionFromInt(c int) (ans PkgSelectorCondition) {
+	if c == PkgCondGreater {
+		ans = PkgCondGreater
+	} else if c == PkgCondGreaterEqual {
+		ans = PkgCondGreaterEqual
+	} else if c == PkgCondLess {
+		ans = PkgCondLess
+	} else if c == PkgCondLessEqual {
+		ans = PkgCondLessEqual
+	} else if c == PkgCondNot {
+		ans = PkgCondNot
+	} else if c == PkgCondAnyRevision {
+		ans = PkgCondAnyRevision
+	} else if c == PkgCondMatchVersion {
+		ans = PkgCondMatchVersion
+	} else {
+		ans = PkgCondInvalid
+	}
+	return
+}
+
+func (p PkgSelectorCondition) String() (ans string) {
+	if p == PkgCondInvalid {
+		ans = ""
+	} else if p == PkgCondGreater {
+		ans = ">"
+	} else if p == PkgCondGreaterEqual {
+		ans = ">="
+	} else if p == PkgCondLess {
+		ans = "<"
+	} else if p == PkgCondLessEqual {
+		ans = "<="
+	} else if p == PkgCondEqual {
+		// To permit correct matching on database
+		// we currently use directly package version without =
+		ans = ""
+	} else if p == PkgCondNot {
+		ans = "!"
+	} else if p == PkgCondAnyRevision {
+		ans = "~"
+	} else if p == PkgCondMatchVersion {
+		ans = "=*"
+	}
+	return
+}
+
+func (p PkgSelectorCondition) Int() (ans int) {
+	if p == PkgCondInvalid {
+		ans = PkgCondInvalid
+	} else if p == PkgCondGreater {
+		ans = PkgCondGreater
+	} else if p == PkgCondGreaterEqual {
+		ans = PkgCondGreaterEqual
+	} else if p == PkgCondLess {
+		ans = PkgCondLess
+	} else if p == PkgCondLessEqual {
+		ans = PkgCondLessEqual
+	} else if p == PkgCondEqual {
+		// To permit correct matching on database
+		// we currently use directly package version without =
+		ans = PkgCondEqual
+	} else if p == PkgCondNot {
+		ans = PkgCondNot
+	} else if p == PkgCondAnyRevision {
+		ans = PkgCondAnyRevision
+	} else if p == PkgCondMatchVersion {
+		ans = PkgCondMatchVersion
+	}
+	return
+}
+
+func ParseVersion(v string) (PkgVersionSelector, error) {
+	var ans PkgVersionSelector = PkgVersionSelector{
+		Version:       "",
+		VersionSuffix: "",
+		Condition:     PkgCondInvalid,
+	}
+
+	if strings.HasPrefix(v, ">=") {
+		v = v[2:]
+		ans.Condition = PkgCondGreaterEqual
+	} else if strings.HasPrefix(v, ">") {
+		v = v[1:]
+		ans.Condition = PkgCondGreater
+	} else if strings.HasPrefix(v, "<=") {
+		v = v[2:]
+		ans.Condition = PkgCondLessEqual
+	} else if strings.HasPrefix(v, "<") {
+		v = v[1:]
+		ans.Condition = PkgCondLess
+	} else if strings.HasPrefix(v, "=") {
+		v = v[1:]
+		if strings.HasSuffix(v, "*") {
+			ans.Condition = PkgCondMatchVersion
+			v = v[0 : len(v)-1]
+		} else {
+			ans.Condition = PkgCondEqual
+		}
+	} else if strings.HasPrefix(v, "~") {
+		v = v[1:]
+		ans.Condition = PkgCondAnyRevision
+	} else if strings.HasPrefix(v, "!") {
+		v = v[1:]
+		ans.Condition = PkgCondNot
+	}
+
+	regexPkg := regexp.MustCompile(
+		fmt.Sprintf("(%s|%s|%s|%s|%s|%s)((%s|%s|%s|%s|%s|%s|%s)+)*$",
+			// Version regex
+			// 1.1
+			"[0-9]+[.][0-9]+[a-z]*",
+			// 1
+			"[0-9]+[a-z]*",
+			// 1.1.1
+			"[0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+			// 1.1.1.1
+			"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+			// 1.1.1.1.1
+			"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+			// 1.1.1.1.1.1
+			"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+			// suffix
+			"-r[0-9]+",
+			"_p[0-9]+",
+			"_pre[0-9]*",
+			"_rc[0-9]+",
+			// handle also rc without number
+			"_rc",
+			"_alpha",
+			"_beta",
+		),
+	)
+	matches := regexPkg.FindAllString(v, -1)
+
+	if len(matches) > 0 {
+		// Check if there patch
+		if strings.Contains(matches[0], "_p") {
+			ans.Version = matches[0][0:strings.Index(matches[0], "_p")]
+			ans.VersionSuffix = matches[0][strings.Index(matches[0], "_p"):]
+		} else if strings.Contains(matches[0], "_rc") {
+			ans.Version = matches[0][0:strings.Index(matches[0], "_rc")]
+			ans.VersionSuffix = matches[0][strings.Index(matches[0], "_rc"):]
+		} else if strings.Contains(matches[0], "_alpha") {
+			ans.Version = matches[0][0:strings.Index(matches[0], "_alpha")]
+			ans.VersionSuffix = matches[0][strings.Index(matches[0], "_alpha"):]
+		} else if strings.Contains(matches[0], "_beta") {
+			ans.Version = matches[0][0:strings.Index(matches[0], "_beta")]
+			ans.VersionSuffix = matches[0][strings.Index(matches[0], "_beta"):]
+		} else if strings.Contains(matches[0], "-r") {
+			ans.Version = matches[0][0:strings.Index(matches[0], "-r")]
+			ans.VersionSuffix = matches[0][strings.Index(matches[0], "-r"):]
+		} else {
+			ans.Version = matches[0]
+		}
+	}
+
+	// Set condition if there isn't a prefix but only a version
+	if ans.Condition == PkgCondInvalid && ans.Version != "" {
+		ans.Condition = PkgCondEqual
+	}
+
+	// NOTE: Now suffix complex like _alpha_rc1 are not supported.
+	return ans, nil
+}
+
+func PackageAdmit(selector, i PkgVersionSelector) (bool, error) {
+	var v1 *version.Version = nil
+	var v2 *version.Version = nil
+	var ans bool
+	var err error
+
+	if selector.Version != "" {
+		v1, err = version.NewVersion(selector.Version)
+		if err != nil {
+			return false, err
+		}
+	}
+	if i.Version != "" {
+		v2, err = version.NewVersion(i.Version)
+		if err != nil {
+			return false, err
+		}
+	} else {
+		// If version is not defined match always package
+		ans = true
+	}
+
+	// If package doesn't define version admit all versions of the package.
+	if selector.Version == "" {
+		ans = true
+	} else {
+		if selector.Condition == PkgCondInvalid || selector.Condition == PkgCondEqual {
+			// case 1: source-pkg-1.0 and dest-pkg-1.0 or dest-pkg without version
+			if i.Version != "" && i.Version == selector.Version && selector.VersionSuffix == i.VersionSuffix {
+				ans = true
+			}
+		} else if selector.Condition == PkgCondAnyRevision {
+			if v1 != nil && v2 != nil {
+				ans = v1.Equal(v2)
+			}
+		} else if selector.Condition == PkgCondMatchVersion {
+			// TODO: case of 7.3* where 7.30 is accepted.
+			if v1 != nil && v2 != nil {
+				segments := v1.Segments()
+				n := strings.Count(selector.Version, ".")
+				switch n {
+				case 0:
+					segments[0]++
+				case 1:
+					segments[1]++
+				case 2:
+					segments[2]++
+				default:
+					segments[len(segments)-1]++
+				}
+				nextVersion := strings.Trim(strings.Replace(fmt.Sprint(segments), " ", ".", -1), "[]")
+				constraints, err := version.NewConstraint(
+					fmt.Sprintf(">= %s, < %s", selector.Version, nextVersion),
+				)
+				if err != nil {
+					return false, err
+				}
+				ans = constraints.Check(v2)
+			}
+		} else if v1 != nil && v2 != nil {
+
+			// TODO: Integrate check of version suffix
+			switch selector.Condition {
+			case PkgCondGreaterEqual:
+				ans = v2.GreaterThanOrEqual(v1)
+			case PkgCondLessEqual:
+				ans = v2.LessThanOrEqual(v1)
+			case PkgCondGreater:
+				ans = v2.GreaterThan(v1)
+			case PkgCondLess:
+				ans = v2.LessThan(v1)
+			case PkgCondNot:
+				ans = !v2.Equal(v1)
+			}
+		}
+	}
+
+	return ans, nil
+}
+
+func (p *DefaultPackage) SelectorMatchVersion(v string) (bool, error) {
+	if !p.IsSelector() {
+		return false, errors.New("Package is not a selector")
+	}
+
+	vS, err := ParseVersion(p.GetVersion())
+	if err != nil {
+		return false, err
+	}
+
+	vSI, err := ParseVersion(v)
+	if err != nil {
+		return false, err
+	}
+
+	return PackageAdmit(vS, vSI)
+}
+
+func (p *DefaultPackage) VersionMatchSelector(selector string) (bool, error) {
+	vS, err := ParseVersion(selector)
+	if err != nil {
+		return false, err
+	}
+
+	vSI, err := ParseVersion(p.GetVersion())
+	if err != nil {
+		return false, err
+	}
+
+	return PackageAdmit(vS, vSI)
+}

pkg/package/version_test.go

@@ -0,0 +1,197 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Versions", func() {
+
+	Context("Versions Parser1", func() {
+		v, err := ParseVersion(">=1.0")
+		It("ParseVersion1", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser2", func() {
+		v, err := ParseVersion(">1.0")
+		It("ParseVersion2", func() {
+			var c PkgSelectorCondition = PkgCondGreater
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser3", func() {
+		v, err := ParseVersion("<=1.0")
+		It("ParseVersion3", func() {
+			var c PkgSelectorCondition = PkgCondLessEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser4", func() {
+		v, err := ParseVersion("<1.0")
+		It("ParseVersion4", func() {
+			var c PkgSelectorCondition = PkgCondLess
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser5", func() {
+		v, err := ParseVersion("=1.0")
+		It("ParseVersion5", func() {
+			var c PkgSelectorCondition = PkgCondEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser6", func() {
+		v, err := ParseVersion("!1.0")
+		It("ParseVersion6", func() {
+			var c PkgSelectorCondition = PkgCondNot
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser7", func() {
+		v, err := ParseVersion("")
+		It("ParseVersion7", func() {
+			var c PkgSelectorCondition = PkgCondInvalid
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal(""))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser8", func() {
+		v, err := ParseVersion("=12.1.0.2_p1")
+		It("ParseVersion8", func() {
+			var c PkgSelectorCondition = PkgCondEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("12.1.0.2"))
+			Expect(v.VersionSuffix).Should(Equal("_p1"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser9", func() {
+		v, err := ParseVersion(">=0.0.20190406.4.9.172-r1")
+		It("ParseVersion9", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.0.20190406.4.9.172"))
+			Expect(v.VersionSuffix).Should(Equal("-r1"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser10", func() {
+		v, err := ParseVersion(">=0.0.20190406.4.9.172_alpha")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.0.20190406.4.9.172"))
+			Expect(v.VersionSuffix).Should(Equal("_alpha"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Selector1", func() {
+		v1, err := ParseVersion(">=0.0.20190406.4.9.172-r1")
+		v2, err2 := ParseVersion("1.0.111")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector1", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(true))
+		})
+	})
+
+	Context("Selector2", func() {
+		v1, err := ParseVersion(">=0.0.20190406.4.9.172-r1")
+		v2, err2 := ParseVersion("0")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector2", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(false))
+		})
+	})
+
+	Context("Selector3", func() {
+		v1, err := ParseVersion(">0")
+		v2, err2 := ParseVersion("0.0.40-alpha")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector3", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(true))
+		})
+	})
+
+	Context("Selector4", func() {
+		v1, err := ParseVersion(">0")
+		v2, err2 := ParseVersion("")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector4", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(true))
+		})
+	})
+
+	Context("Condition Converter 1", func() {
+		gp, err := gentoo.ParsePackageStr("=layer/build-1.0")
+		var cond gentoo.PackageCond = gentoo.PkgCondEqual
+		It("Converter1", func() {
+			Expect(err).Should(BeNil())
+			Expect((*gp).Condition).Should(Equal(cond))
+			Expect(PkgSelectorConditionFromInt((*gp).Condition.Int()).String()).Should(Equal(""))
+		})
+	})
+
+})

pkg/repository/loader.go

@@ -0,0 +1,85 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package repository
+
+import (
+	"io/ioutil"
+	"path"
+	"regexp"
+
+	"github.com/ghodss/yaml"
+
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+func LoadRepositories(c *LuetConfig) error {
+	var regexRepo = regexp.MustCompile(`.yml$`)
+
+	for _, rdir := range c.RepositoriesConfDir {
+		Debug("Parsing Repository Directory", rdir, "...")
+
+		files, err := ioutil.ReadDir(rdir)
+		if err != nil {
+			Warning("Skip dir", rdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexRepo.MatchString(file.Name()) {
+				Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(rdir, file.Name()))
+			if err != nil {
+				Warning("On read file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := LoadRepository(content)
+			if err != nil {
+				Warning("On parse file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Urls) == 0 || r.Type == "" {
+				Warning("Invalid repository ", file.Name())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddSystemRepository(*r)
+		}
+	}
+	return nil
+}
+
+func LoadRepository(data []byte) (*LuetRepository, error) {
+	ans := NewEmptyLuetRepository()
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}

pkg/repository/repository_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package repository_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Repository Suite")
+}

pkg/repository/repository_test.go

@@ -0,0 +1,46 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package repository_test
+
+import (
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/repository"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	viper "github.com/spf13/viper"
+)
+
+var _ = Describe("Repository", func() {
+	Context("Load Repository1", func() {
+		cfg := NewLuetConfig(viper.New())
+		cfg.RepositoriesConfDir = []string{
+			"../../tests/fixtures/repos.conf.d",
+		}
+		err := LoadRepositories(cfg)
+
+		It("Chec Load Repository 1", func() {
+			Expect(err).Should(BeNil())
+			Expect(len(cfg.SystemRepositories)).Should(Equal(1))
+			Expect(cfg.SystemRepositories[0].Name).Should(Equal("test1"))
+			Expect(cfg.SystemRepositories[0].Priority).Should(Equal(999))
+			Expect(cfg.SystemRepositories[0].Type).Should(Equal("disk"))
+			Expect(len(cfg.SystemRepositories[0].Urls)).Should(Equal(1))
+			Expect(cfg.SystemRepositories[0].Urls[0]).Should(Equal("tests/repos/test1"))
+		})
+	})
+})

pkg/solver/decoder.go

@@ -122,6 +122,29 @@ func (assertions PackagesAssertions) EnsureOrder() PackagesAssertions {
 	return orderedAssertions
 }
 
+func (assertions PackagesAssertions) SearchByName(f string) *PackageAssert {
+	for _, a := range assertions {
+		if a.Value {
+			if a.Package.GetPackageName() == f {
+				return &a
+			}
+		}
+	}
+
+	return nil
+}
+func (assertions PackagesAssertions) Search(f string) *PackageAssert {
+	for _, a := range assertions {
+		if a.Value {
+			if a.Package.GetFingerPrint() == f {
+				return &a
+			}
+		}
+	}
+
+	return nil
+}
+
 func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fingerprint string) PackagesAssertions {
 
 	orderedAssertions := PackagesAssertions{}
@@ -145,19 +168,25 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 	}
 
 	sort.Sort(unorderedAssertions)
-
 	// Build a topological graph
 	//graph := toposort.NewGraph(len(unorderedAssertions))
 	//	graph.AddNodes(fingerprints...)
 	for _, a := range unorderedAssertions {
-		for _, requiredDef := range a.Package.GetRequires() {
-			req, err := definitiondb.FindPackageCandidate(requiredDef)
-			if err != nil {
-				req = requiredDef
+		currentPkg := a.Package
+		for _, requiredDef := range currentPkg.GetRequires() {
+			if def, err := definitiondb.FindPackage(requiredDef); err == nil { // Provides: Get a chance of being override here
+				requiredDef = def.(*pkg.DefaultPackage)
+			}
+
+			// We cannot search for fingerprint, as we could have selector in versions.
+			// We know that the assertions are unique for packages, so look for a package with such name in the assertions
+			req := assertions.SearchByName(requiredDef.GetPackageName())
+			if req != nil {
+				requiredDef = req.Package
 			}
 
 			// Expand also here, as we need to order them (or instead the solver should give back the dep correctly?)
-			graph.AddEdge(a.Package.GetFingerPrint(), req.GetFingerPrint())
+			graph.AddEdge(currentPkg.GetFingerPrint(), requiredDef.GetFingerPrint())
 		}
 	}
 	result, err := graph.TopSort(fingerprint)
@@ -167,7 +196,7 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 	for _, res := range result {
 		a, ok := tmpMap[res]
 		if !ok {
-			panic("fail")
+			panic("fail looking for " + res)
 			//	continue
 		}
 		orderedAssertions = append(orderedAssertions, a)
@@ -239,3 +268,18 @@ func (assertions PackagesAssertions) Drop(p pkg.Package) PackagesAssertions {
 	}
 	return ass
 }
+
+// Cut returns an assertion list of installed (filter by Value) "cutted" until the package is found (included)
+func (assertions PackagesAssertions) Cut(p pkg.Package) PackagesAssertions {
+	ass := PackagesAssertions{}
+
+	for _, a := range assertions {
+		if a.Value {
+			ass = append(ass, a)
+			if a.Package.Matches(p) {
+				break
+			}
+		}
+	}
+	return ass
+}

pkg/solver/decoder_test.go

@@ -246,5 +246,31 @@ var _ = Describe("Decoder", func() {
 			Expect(solution.Order(dbDefinitions, Y.GetFingerPrint()).Drop(Y).AssertionHash() == solution2.Order(dbDefinitions, Z.GetFingerPrint()).Drop(Z).AssertionHash()).To(BeTrue())
 		})
 
+		It("Hashes them, Cuts them and could be used for comparison", func() {
+
+			X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+			Z := pkg.NewPackage("Z", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{X, Y, Z} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			solution, err := s.Install([]pkg.Package{Y})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution2, err := s.Install([]pkg.Package{Z})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(solution.Order(dbDefinitions, Y.GetFingerPrint()).Cut(Y).Drop(Y)).To(Equal(solution2.Order(dbDefinitions, Z.GetFingerPrint()).Cut(Z).Drop(Z)))
+
+			Expect(solution.Order(dbDefinitions, Y.GetFingerPrint()).Cut(Y).Drop(Y).AssertionHash()).To(Equal(solution2.Order(dbDefinitions, Z.GetFingerPrint()).Cut(Z).Drop(Z).AssertionHash()))
+		})
+
 	})
 })

pkg/solver/resolver.go

@@ -0,0 +1,336 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/crillab/gophersat/bf"
+	"github.com/mudler/luet/pkg/helpers"
+	"gopkg.in/yaml.v2"
+
+	"github.com/ecooper/qlearning"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/pkg/errors"
+)
+
+type ActionType int
+
+const (
+	NoAction      = 0
+	Solved        = iota
+	NoSolution    = iota
+	Going         = iota
+	ActionRemoved = iota
+	ActionAdded   = iota
+
+	DoNoop = false
+
+	ActionDomains = 3 // Bump it if you increase the number of actions
+
+	DefaultMaxAttempts     = 9000
+	DefaultLearningRate    = 0.7
+	DefaultDiscount        = 1.0
+	DefaultInitialObserved = 999999
+
+	QLearningResolverType = "qlearning"
+)
+
+//. "github.com/mudler/luet/pkg/logger"
+
+// PackageResolver assists PackageSolver on unsat cases
+type PackageResolver interface {
+	Solve(bf.Formula, PackageSolver) (PackagesAssertions, error)
+}
+
+type DummyPackageResolver struct {
+}
+
+func (*DummyPackageResolver) Solve(bf.Formula, PackageSolver) (PackagesAssertions, error) {
+	return nil, errors.New("Could not satisfy the constraints. Try again by removing deps ")
+}
+
+type QLearningResolver struct {
+	Attempts int
+
+	ToAttempt int
+
+	attempts int
+
+	Attempted map[string]bool
+
+	Solver  PackageSolver
+	Formula bf.Formula
+
+	Targets []pkg.Package
+	Current []pkg.Package
+
+	observedDelta       int
+	observedDeltaChoice []pkg.Package
+
+	Agent *qlearning.SimpleAgent
+}
+
+func SimpleQLearningSolver() PackageResolver {
+	return NewQLearningResolver(DefaultLearningRate, DefaultDiscount, DefaultMaxAttempts, DefaultInitialObserved)
+}
+
+// Defaults LearningRate 0.7, Discount 1.0
+func NewQLearningResolver(LearningRate, Discount float32, MaxAttempts, initialObservedDelta int) PackageResolver {
+	return &QLearningResolver{
+		Agent:         qlearning.NewSimpleAgent(LearningRate, Discount),
+		observedDelta: initialObservedDelta,
+		Attempts:      MaxAttempts,
+	}
+}
+
+func (resolver *QLearningResolver) Solve(f bf.Formula, s PackageSolver) (PackagesAssertions, error) {
+	//	Info("Using QLearning solver to resolve conflicts. Please be patient.")
+	resolver.Solver = s
+
+	s.SetResolver(&DummyPackageResolver{}) // Set dummy. Otherwise the attempts will run again a QLearning instance.
+	defer s.SetResolver(resolver)          // Set back ourselves as resolver
+
+	resolver.Formula = f
+
+	// Our agent by default has a learning rate of 0.7 and discount of 1.0.
+	if resolver.Agent == nil {
+		resolver.Agent = qlearning.NewSimpleAgent(DefaultLearningRate, DefaultDiscount) // FIXME: Remove hardcoded values
+	}
+
+	// 3 are the action domains, counting noop regardless if enabled or not
+	// get the permutations to attempt
+	resolver.ToAttempt = int(helpers.Factorial(uint64(len(resolver.Solver.(*Solver).Wanted)-1) * ActionDomains)) // TODO: type assertions must go away
+	resolver.Targets = resolver.Solver.(*Solver).Wanted
+
+	resolver.attempts = resolver.Attempts
+
+	resolver.Attempted = make(map[string]bool, len(resolver.Targets))
+
+	for resolver.IsComplete() == Going {
+		// Pick the next move, which is going to be a letter choice.
+		action := qlearning.Next(resolver.Agent, resolver)
+
+		// Whatever that choice is, let's update our model for its
+		// impact. If the package chosen makes the formula sat,
+		// then this action will be positive. Otherwise, it will be
+		// negative.
+		resolver.Agent.Learn(action, resolver)
+
+		// Reward doesn't change state so we can check what the
+		// reward would be for this action, and report how the
+		// env changed.
+		//	score := resolver.Reward(action)
+		//	if score > 0.0 {
+		//	resolver.Log("%s was correct", action.Action.String())
+		//	} else {
+		//	resolver.Log("%s was incorrect", action.Action.String())
+		//	}
+	}
+
+	// If we get good result, take it
+	// Take the result also if we did  reached overall maximum attempts
+	if resolver.IsComplete() == Solved || resolver.IsComplete() == NoSolution {
+
+		if len(resolver.observedDeltaChoice) != 0 {
+			// Take the minimum delta observed choice result, and consume it (Try sets the wanted list)
+			resolver.Solver.(*Solver).Wanted = resolver.observedDeltaChoice
+		}
+
+		return resolver.Solver.Solve()
+	} else {
+		return nil, errors.New("QLearning resolver failed ")
+	}
+
+}
+
+// Returns the current state.
+func (resolver *QLearningResolver) IsComplete() int {
+	if resolver.attempts < 1 {
+		return NoSolution
+	}
+
+	if resolver.ToAttempt > 0 {
+		return Going
+	}
+
+	return Solved
+}
+
+func (resolver *QLearningResolver) Try(c Choice) error {
+	pack := c.Package
+	packtoAdd := pkg.FromString(pack)
+	resolver.Attempted[pack+strconv.Itoa(int(c.Action))] = true // increase the count
+	s, _ := resolver.Solver.(*Solver)
+	var filtered []pkg.Package
+
+	switch c.Action {
+	case ActionAdded:
+		found := false
+		for _, p := range s.Wanted {
+			if p.String() == pack {
+				found = true
+				break
+			}
+		}
+		if !found {
+			resolver.Solver.(*Solver).Wanted = append(resolver.Solver.(*Solver).Wanted, packtoAdd)
+		}
+
+	case ActionRemoved:
+		for _, p := range s.Wanted {
+			if p.String() != pack {
+				filtered = append(filtered, p)
+			}
+		}
+
+		resolver.Solver.(*Solver).Wanted = filtered
+	}
+
+	_, err := resolver.Solver.Solve()
+
+	return err
+}
+
+// Choose applies a pack attempt, returning
+// true if the formula returns sat.
+//
+// Choose updates the resolver's state.
+func (resolver *QLearningResolver) Choose(c Choice) bool {
+	//pack := pkg.FromString(c.Package)
+
+	err := resolver.Try(c)
+
+	if err == nil {
+		resolver.ToAttempt--
+		resolver.attempts-- // Decrease attempts - it's a barrier. We could also do not decrease it here, allowing more attempts to be made
+	} else {
+		resolver.attempts--
+		return false
+	}
+
+	return true
+}
+
+// Reward returns a score for a given qlearning.StateAction. Reward is a
+// member of the qlearning.Rewarder interface. If the choice will make sat the formula, a positive score is returned.
+// Otherwise, a static -1000 is returned.
+func (resolver *QLearningResolver) Reward(action *qlearning.StateAction) float32 {
+	choice := action.Action.(*Choice)
+
+	//_, err := resolver.Solver.Solve()
+	err := resolver.Try(*choice)
+
+	toBeInstalled := len(resolver.Solver.(*Solver).Wanted)
+	originalTarget := len(resolver.Targets)
+	noaction := choice.Action == NoAction
+	delta := originalTarget - toBeInstalled
+
+	if err == nil {
+		// if toBeInstalled == originalTarget { // Base case: all the targets matches (it shouldn't happen, but lets put a higher)
+		// 	Debug("Target match, maximum score")
+		// 	return 24.0 / float32(len(resolver.Attempted))
+
+		// }
+		if DoNoop {
+			if noaction && toBeInstalled == 0 { // We decided to stay in the current state, and no targets have been chosen
+				return -100
+			}
+		}
+
+		if delta <= resolver.observedDelta { // Try to maximise observedDelta
+			resolver.observedDelta = delta
+			resolver.observedDeltaChoice = resolver.Solver.(*Solver).Wanted // we store it as this is our return value at the end
+			return 24.0 / float32(len(resolver.Attempted))
+		} else if toBeInstalled > 0 { // If we installed something, at least give a good score
+			return 24.0 / float32(len(resolver.Attempted))
+		}
+
+	}
+
+	return -1000
+}
+
+// Next creates a new slice of qlearning.Action instances. A possible
+// action is created for each package that could be removed from the formula's target
+func (resolver *QLearningResolver) Next() []qlearning.Action {
+	actions := make([]qlearning.Action, 0, (len(resolver.Targets)-1)*3)
+
+TARGETS:
+	for _, pack := range resolver.Targets {
+		for _, current := range resolver.Solver.(*Solver).Wanted {
+			if current.String() == pack.String() {
+				actions = append(actions, &Choice{Package: pack.String(), Action: ActionRemoved})
+				continue TARGETS
+			}
+
+		}
+		actions = append(actions, &Choice{Package: pack.String(), Action: ActionAdded})
+	}
+
+	if DoNoop {
+		actions = append(actions, &Choice{Package: "", Action: NoAction}) // NOOP
+	}
+
+	return actions
+}
+
+// Log is a wrapper of fmt.Printf. If Game.debug is true, Log will print
+// to stdout.
+func (resolver *QLearningResolver) Log(msg string, args ...interface{}) {
+	logMsg := fmt.Sprintf("(%d moves, %d remaining attempts) %s\n", len(resolver.Attempted), resolver.attempts, msg)
+	fmt.Println(fmt.Sprintf(logMsg, args...))
+}
+
+// String returns a consistent hash for the current env state to be
+// used in a qlearning.Agent.
+func (resolver *QLearningResolver) String() string {
+	return fmt.Sprintf("%v", resolver.Solver.(*Solver).Wanted)
+}
+
+// Choice implements qlearning.Action for a package choice for removal from wanted targets
+type Choice struct {
+	Package string     `json:"pack"`
+	Action  ActionType `json:"action"`
+}
+
+func ChoiceFromString(s string) (*Choice, error) {
+	var p *Choice
+	err := yaml.Unmarshal([]byte(s), &p)
+	if err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+// String returns the character for the current action.
+func (choice *Choice) String() string {
+	data, err := json.Marshal(choice)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+// Apply updates the state of the solver for the package choice.
+func (choice *Choice) Apply(state qlearning.State) qlearning.State {
+	resolver := state.(*QLearningResolver)
+	resolver.Choose(*choice)
+
+	return resolver
+}

pkg/solver/resolver_test.go

@@ -0,0 +1,182 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver_test
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "github.com/mudler/luet/pkg/solver"
+)
+
+var _ = Describe("Resolver", func() {
+
+	db := pkg.NewInMemoryDatabase(false)
+	dbInstalled := pkg.NewInMemoryDatabase(false)
+	dbDefinitions := pkg.NewInMemoryDatabase(false)
+	s := NewSolver(dbInstalled, dbDefinitions, db)
+
+	BeforeEach(func() {
+		db = pkg.NewInMemoryDatabase(false)
+		dbInstalled = pkg.NewInMemoryDatabase(false)
+		dbDefinitions = pkg.NewInMemoryDatabase(false)
+		s = NewSolver(dbInstalled, dbDefinitions, db)
+	})
+
+	Context("Conflict set", func() {
+		Context("DummyPackageResolver", func() {
+			It("is unsolvable - as we something we ask to install conflict with system stuff", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A})
+				Expect(len(solution)).To(Equal(0))
+				Expect(err).To(HaveOccurred())
+			})
+			It("succeeds to install D and F if explictly requested", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D, E, F} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{D, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(len(solution)).To(Equal(6))
+
+				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
+
+			})
+
+		})
+		Context("QLearningResolver", func() {
+			It("will find out that we can install D by ignoring A", func() {
+				s.SetResolver(SimpleQLearningSolver())
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A, D})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+
+				Expect(len(solution)).To(Equal(4))
+			})
+
+			It("will find out that we can install D and F by ignoring E and A", func() {
+				s.SetResolver(SimpleQLearningSolver())
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D, E, F} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A, D, E, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) // Was already installed
+				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
+				Expect(len(solution)).To(Equal(6))
+
+			})
+		})
+
+		Context("DummyPackageResolver", func() {
+			It("cannot find a solution", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A, D})
+				Expect(err).To(HaveOccurred())
+
+				Expect(len(solution)).To(Equal(0))
+			})
+
+		})
+	})
+
+})

pkg/solver/solver.go

@@ -33,6 +33,10 @@ type PackageSolver interface {
 	ConflictsWith(p pkg.Package, ls []pkg.Package) (bool, error)
 	World() []pkg.Package
 	Upgrade() ([]pkg.Package, PackagesAssertions, error)
+
+	SetResolver(PackageResolver)
+
+	Solve() (PackagesAssertions, error)
 }
 
 // Solver is the default solver for luet
@@ -41,20 +45,33 @@ type Solver struct {
 	SolverDatabase     pkg.PackageDatabase
 	Wanted             []pkg.Package
 	InstalledDatabase  pkg.PackageDatabase
+
+	Resolver PackageResolver
 }
 
 // NewSolver accepts as argument two lists of packages, the first is the initial set,
 // the second represent all the known packages.
 func NewSolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase) PackageSolver {
-	return &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb}
+	return NewResolver(installed, definitiondb, solverdb, &DummyPackageResolver{})
 }
 
-// SetWorld is a setter for the list of all known packages to the solver
+// NewReSolver accepts as argument two lists of packages, the first is the initial set,
+// the second represent all the known packages.
+func NewResolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase, re PackageResolver) PackageSolver {
+	return &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb, Resolver: re}
+}
+
+// SetDefinitionDatabase is a setter for the definition Database
 
 func (s *Solver) SetDefinitionDatabase(db pkg.PackageDatabase) {
 	s.DefinitionDatabase = db
 }
 
+// SetResolver is a setter for the unsat resolver backend
+func (s *Solver) SetResolver(r PackageResolver) {
+	s.Resolver = r
+}
+
 func (s *Solver) World() []pkg.Package {
 	return s.DefinitionDatabase.World()
 }
@@ -221,6 +238,7 @@ func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
 	}
 
 	s2 := NewSolver(installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+	s2.SetResolver(s.Resolver)
 	// Then try to uninstall the versions in the system, and store that tree
 	for _, p := range toUninstall {
 		r, err := s.Uninstall(p)
@@ -361,13 +379,20 @@ func (s *Solver) solve(f bf.Formula) (map[string]bool, bf.Formula, error) {
 
 // Solve builds the formula given the current state and returns package assertions
 func (s *Solver) Solve() (PackagesAssertions, error) {
+	var model map[string]bool
+	var err error
+
 	f, err := s.BuildFormula()
 
 	if err != nil {
 		return nil, err
 	}
 
-	model, _, err := s.solve(f)
+	model, _, err = s.solve(f)
+	if err != nil && s.Resolver != nil {
+		return s.Resolver.Solve(f, s)
+	}
+
 	if err != nil {
 		return nil, err
 	}

pkg/solver/solver_suite_test.go

@@ -18,11 +18,15 @@ package solver_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Solver Suite")
 }

pkg/solver/solver_test.go

@@ -326,6 +326,195 @@ var _ = Describe("Solver", func() {
 			Expect(len(solution)).To(Equal(3))
 			Expect(err).ToNot(HaveOccurred())
 		})
+		It("Selects one version", func() {
+
+			E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D2 := pkg.NewPackage("D", "1.9", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("D", "1.8", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D1 := pkg.NewPackage("D", "1.4", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B := pkg.NewPackage("B", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: "1.4"}}, []*pkg.DefaultPackage{})
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{A, B, C, D, D1, D2, E} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			s = NewSolver(dbInstalled, dbDefinitions, db)
+
+			solution, err := s.Install([]pkg.Package{A, B})
+			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D2, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D2, Value: false}))
+
+			Expect(len(solution)).To(Equal(5))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Selects best version", func() {
+
+			E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D2 := pkg.NewPackage("D", "1.9", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("D", "1.8", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D1 := pkg.NewPackage("D", "1.4", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B := pkg.NewPackage("B", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{A, B, C, D, D1, D2, E} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			s = NewSolver(dbInstalled, dbDefinitions, db)
+
+			solution, err := s.Install([]pkg.Package{A, B})
+			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D1, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D2, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
+
+			Expect(len(solution)).To(Equal(5))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Support provides", func() {
+
+			E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D2 := pkg.NewPackage("D", "1.9", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("D", "1.8", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D1 := pkg.NewPackage("D", "1.4", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B := pkg.NewPackage("B", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+
+			D2.SetProvides([]*pkg.DefaultPackage{{Name: "E"}})
+			A2 := pkg.NewPackage("A", "1.3", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "E", Version: ""}}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{A, B, C, D, D1, D2, A2, E} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			s = NewSolver(dbInstalled, dbDefinitions, db)
+
+			solution, err := s.Install([]pkg.Package{A2, B})
+			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D1, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D2, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
+
+			Expect(len(solution)).To(Equal(5))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Support provides with versions", func() {
+			E := pkg.NewPackage("E", "1.3", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D2 := pkg.NewPackage("D", "1.9", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("D", "1.8", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D1 := pkg.NewPackage("D", "1.4", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B := pkg.NewPackage("B", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+
+			D2.SetProvides([]*pkg.DefaultPackage{{Name: "E", Version: "1.3"}})
+			A2 := pkg.NewPackage("A", "1.3", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "E", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{A, B, C, D, D1, D2, A2, E} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			s = NewSolver(dbInstalled, dbDefinitions, db)
+
+			solution, err := s.Install([]pkg.Package{A2})
+			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D1, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: C, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: D2, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
+
+			Expect(len(solution)).To(Equal(4))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Support provides with selectors", func() {
+			E := pkg.NewPackage("E", "1.3", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D2 := pkg.NewPackage("D", "1.9", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("D", "1.8", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D1 := pkg.NewPackage("D", "1.4", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B := pkg.NewPackage("B", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+
+			D2.SetProvides([]*pkg.DefaultPackage{{Name: "E", Version: ">=1.3"}})
+			A2 := pkg.NewPackage("A", "1.3", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "E", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{A, B, C, D, D1, D2, A2, E} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			s = NewSolver(dbInstalled, dbDefinitions, db)
+
+			solution, err := s.Install([]pkg.Package{A2})
+			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D1, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: C, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: D2, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
+
+			Expect(len(solution)).To(Equal(4))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
 		It("Uninstalls simple package correctly", func() {
 
 			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})

pkg/tree/builder/gentoo/builder_suite_test.go

@@ -18,11 +18,15 @@ package gentoo_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestGentooBuilder(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Gentoo Suite")
 }

pkg/tree/builder/gentoo/gentoo.go

@@ -124,7 +124,9 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.PackageDatabase, error) {
 		if info.IsDir() {
 			return nil
 		}
-		if strings.Contains(info.Name(), "ebuild") {
+		// Ensure that only file with suffix .ebuild are elaborated.
+		// and ignore .swp files or files with string ebuild on name
+		if strings.HasSuffix(info.Name(), ".ebuild") {
 			toScan <- path
 		}
 		return nil

pkg/tree/builder/gentoo/gentoo_test.go

@@ -16,6 +16,7 @@
 package gentoo_test
 
 import (
+	"fmt"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -46,4 +47,131 @@ var _ = Describe("GentooBuilder", func() {
 		}
 	})
 
+	Context("Parse ebuild1", func() {
+		parser := &SimpleEbuildParser{}
+		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/overlay/app-crypt/pinentry-gnome/pinentry-gnome-1.0.0-r2.ebuild")
+		It("parses correctly deps", func() {
+			Expect(err).ToNot(HaveOccurred())
+			fmt.Println("PKG ", pkgs[0])
+			Expect(pkgs[0].GetLicense()).To(Equal("GPL-2"))
+			Expect(pkgs[0].GetDescription()).To(Equal("GNOME 3 frontend for pinentry"))
+		})
+	})
+
+	Context("Parse ebuild2", func() {
+		parser := &SimpleEbuildParser{}
+		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/mod_dav_svn-1.12.2.ebuild")
+
+		It("Parsing ebuild2", func() {
+			Expect(err).ToNot(HaveOccurred())
+			fmt.Println("PKG ", pkgs[0])
+			Expect(pkgs[0].GetLicense()).To(Equal("Subversion"))
+			Expect(pkgs[0].GetDescription()).To(Equal("Subversion WebDAV support"))
+		})
+	})
+
+	Context("Parse ebuild3", func() {
+		parser := &SimpleEbuildParser{}
+		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/linux-sources-1.ebuild")
+
+		It("Check parsing of the ebuild3", func() {
+			Expect(err).ToNot(HaveOccurred())
+			fmt.Println("PKG ", pkgs[0])
+			Expect(len(pkgs[0].GetRequires())).To(Equal(0))
+			Expect(pkgs[0].GetLicense()).To(Equal(""))
+			Expect(pkgs[0].GetDescription()).To(Equal("Virtual for Linux kernel sources"))
+		})
+	})
+
+	Context("Parse ebuild4", func() {
+		parser := &SimpleEbuildParser{}
+		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/sabayon-mce-1.1-r5.ebuild")
+
+		It("Check parsing of the ebuild4", func() {
+			Expect(err).ToNot(HaveOccurred())
+			fmt.Println("PKG ", pkgs[0])
+			Expect(len(pkgs[0].GetRequires())).To(Equal(2))
+			Expect(pkgs[0].GetLicense()).To(Equal("GPL-2"))
+			Expect(pkgs[0].GetDescription()).To(Equal("Sabayon Linux Media Center Infrastructure"))
+		})
+	})
+
+	Context("Parse ebuild5", func() {
+		parser := &SimpleEbuildParser{}
+		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/libreoffice-l10n-meta-6.2.8.2.ebuild")
+
+		It("Check parsing of the ebuild5", func() {
+			Expect(err).ToNot(HaveOccurred())
+			fmt.Println("PKG ", pkgs[0])
+			Expect(len(pkgs[0].GetRequires())).To(Equal(146))
+			Expect(pkgs[0].GetLicense()).To(Equal("LGPL-2"))
+			Expect(pkgs[0].GetDescription()).To(Equal("LibreOffice.org localisation meta-package"))
+		})
+	})
+
+	Context("Parse ebuild6", func() {
+		parser := &SimpleEbuildParser{}
+		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/pkgs-checker-0.2.0.ebuild")
+
+		It("Check parsing of the ebuild6", func() {
+			Expect(err).ToNot(HaveOccurred())
+			fmt.Println("PKG ", pkgs[0])
+			Expect(len(pkgs[0].GetRequires())).To(Equal(0))
+			Expect(pkgs[0].GetLicense()).To(Equal("GPL-3"))
+			Expect(pkgs[0].GetDescription()).To(Equal("Sabayon Packages Checker"))
+		})
+	})
+
+	Context("Parse ebuild7", func() {
+		parser := &SimpleEbuildParser{}
+		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/calamares-sabayon-base-modules-1.15.ebuild")
+
+		It("Check parsing of the ebuild7", func() {
+			Expect(err).ToNot(HaveOccurred())
+			fmt.Println("PKG ", pkgs[0])
+			Expect(len(pkgs[0].GetRequires())).To(Equal(2))
+			Expect(pkgs[0].GetLicense()).To(Equal("CC-BY-SA-4.0"))
+			Expect(pkgs[0].GetDescription()).To(Equal("Sabayon Official Calamares base modules"))
+		})
+	})
+
+	Context("Parse ebuild8", func() {
+		parser := &SimpleEbuildParser{}
+		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/subversion-1.12.0.ebuild")
+
+		It("Check parsing of the ebuild8", func() {
+			Expect(err).ToNot(HaveOccurred())
+			fmt.Println("PKG ", pkgs[0])
+			Expect(len(pkgs[0].GetRequires())).To(Equal(25))
+			Expect(pkgs[0].GetLicense()).To(Equal("Subversion GPL-2"))
+			Expect(pkgs[0].GetDescription()).To(Equal("Advanced version control system"))
+		})
+	})
+
+	Context("Parse ebuild9", func() {
+		parser := &SimpleEbuildParser{}
+		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/kodi-raspberrypi-16.0.ebuild")
+
+		PIt("Check parsing of the ebuild9", func() {
+			Expect(err).ToNot(HaveOccurred())
+			fmt.Println("PKG ", pkgs[0])
+			Expect(len(pkgs[0].GetRequires())).To(Equal(66))
+			Expect(pkgs[0].GetLicense()).To(Equal("GPL-2"))
+			Expect(pkgs[0].GetDescription()).To(Equal("Kodi is a free and open source media-player and entertainment hub"))
+		})
+	})
+
+	Context("Parse ebuild10", func() {
+		parser := &SimpleEbuildParser{}
+		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/tango-icon-theme-0.8.90-r1.ebuild")
+
+		It("Check parsing of the ebuild10", func() {
+			Expect(err).ToNot(HaveOccurred())
+			fmt.Println("PKG ", pkgs[0])
+			Expect(len(pkgs[0].GetRequires())).To(Equal(2))
+			Expect(pkgs[0].GetLicense()).To(Equal("public-domain"))
+			Expect(pkgs[0].GetDescription()).To(Equal("SVG and PNG icon theme from the Tango project"))
+		})
+	})
+
 })

pkg/tree/builder/gentoo/simpleparser.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -31,9 +32,13 @@ import (
 
 	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	pkg "github.com/mudler/luet/pkg/package"
-	"mvdan.cc/sh/expand"
-	"mvdan.cc/sh/shell"
-	"mvdan.cc/sh/syntax"
+	"mvdan.cc/sh/v3/expand"
+	"mvdan.cc/sh/v3/shell"
+	"mvdan.cc/sh/v3/syntax"
+)
+
+const (
+	uriRegex = "(.*[.]tar[.].*|.*[.]zip|.*[.]run|.*[.]png|.*[.]rpm|.*[.]gz)"
 )
 
 // SimpleEbuildParser ignores USE flags and generates just 1-1 package
@@ -66,15 +71,15 @@ func NewGentooDependency(pkg, use string) (*GentooDependency, error) {
 
 	if pkg != "" {
 		ans.Dep, err = _gentoo.ParsePackageStr(pkg)
+		if err != nil {
+			return nil, err
+		}
 
 		// TODO: Fix this on parsing phase for handle correctly ${PV}
 		if strings.HasSuffix(ans.Dep.Name, "-") {
 			ans.Dep.Name = ans.Dep.Name[:len(ans.Dep.Name)-1]
 		}
 
-		if err != nil {
-			return nil, err
-		}
 	}
 
 	return ans, nil
@@ -143,6 +148,7 @@ func (r *GentooRDEPEND) GetDependencies() []*GentooDependency {
 func ParseRDEPEND(rdepend string) (*GentooRDEPEND, error) {
 	var lastdep []*GentooDependency = make([]*GentooDependency, 0)
 	var pendingDep = false
+	var orDep = false
 	var dep *GentooDependency
 	var err error
 
@@ -158,20 +164,34 @@ func ParseRDEPEND(rdepend string) (*GentooRDEPEND, error) {
 				continue
 			}
 
+			if strings.HasPrefix(rr, "|| (") {
+				orDep = true
+				continue
+			}
+
+			if orDep {
+				rr = strings.TrimSpace(rr)
+				if rr == ")" {
+					orDep = false
+				}
+				continue
+			}
+
 			if strings.Index(rr, "?") > 0 {
 				// use flag present
 
 				if pendingDep {
 					dep, err = lastdep[len(lastdep)-1].AddSubDependency("", rr[:strings.Index(rr, "?")])
 					if err != nil {
-						return nil, err
+						Debug("Ignoring subdependency ", rr[:strings.Index(rr, "?")])
 					}
 				} else {
 					dep, err = NewGentooDependency("", rr[:strings.Index(rr, "?")])
 					if err != nil {
-						return nil, err
+						Debug("Ignoring dep", rr)
+					} else {
+						ans.Dependencies = append(ans.Dependencies, dep)
 					}
-					ans.Dependencies = append(ans.Dependencies, dep)
 				}
 
 				if strings.Index(rr, ")") < 0 {
@@ -179,6 +199,11 @@ func ParseRDEPEND(rdepend string) (*GentooRDEPEND, error) {
 					lastdep = append(lastdep, dep)
 				}
 
+				if strings.Index(rr, "|| (") >= 0 {
+					// Ignore dep in or
+					continue
+				}
+
 				fields := strings.Split(rr[strings.Index(rr, "?")+1:], " ")
 				for _, f := range fields {
 					f = strings.TrimSpace(f)
@@ -188,7 +213,7 @@ func ParseRDEPEND(rdepend string) (*GentooRDEPEND, error) {
 
 					_, err = dep.AddSubDependency(f, "")
 					if err != nil {
-						return nil, err
+						Debug("Ignoring subdependency ", f)
 					}
 				}
 
@@ -213,11 +238,31 @@ func ParseRDEPEND(rdepend string) (*GentooRDEPEND, error) {
 				}
 
 			} else {
-				dep, err := NewGentooDependency(rr, "")
-				if err != nil {
-					return nil, err
+				rr = strings.TrimSpace(rr)
+				// Check if there multiple deps in single row
+
+				fields := strings.Split(rr, " ")
+				if len(fields) > 1 {
+					for _, rrr := range fields {
+						rrr = strings.TrimSpace(rrr)
+						if rrr == "" {
+							continue
+						}
+						dep, err := NewGentooDependency(rrr, "")
+						if err != nil {
+							Debug("Ignoring dep", rr)
+						} else {
+							ans.Dependencies = append(ans.Dependencies, dep)
+						}
+					}
+				} else {
+					dep, err := NewGentooDependency(rr, "")
+					if err != nil {
+						Debug("Ignoring dep", rr)
+					} else {
+						ans.Dependencies = append(ans.Dependencies, dep)
+					}
 				}
-				ans.Dependencies = append(ans.Dependencies, dep)
 			}
 
 		}
@@ -232,14 +277,45 @@ func SourceFile(ctx context.Context, path string, pkg *_gentoo.GentooPackage) (m
 	if err != nil {
 		return nil, fmt.Errorf("could not open: %v", err)
 	}
+	scontent := string(content)
+
 	// Add default Genoo Variables
 	ebuild := fmt.Sprintf("P=%s\n", pkg.GetP()) +
 		fmt.Sprintf("PN=%s\n", pkg.GetPN()) +
 		fmt.Sprintf("PV=%s\n", pkg.GetPV()) +
-		fmt.Sprintf("PVR=%s\n", pkg.GetPVR()) +
-		string(content)
+		fmt.Sprintf("PVR=%s\n", pkg.GetPVR())
 
-	file, err := syntax.NewParser(syntax.StopAt("src")).Parse(strings.NewReader(ebuild), path)
+	// Disable inherit
+	scontent = strings.ReplaceAll(scontent, "inherit", "#inherit")
+	// Disable function from eclass (TODO: check how fix better this)
+	scontent = strings.ReplaceAll(scontent, "need_apache", "#need_apache")
+	scontent = strings.ReplaceAll(scontent, "want_apache", "#want_apache")
+
+	regexFuncs := regexp.MustCompile(
+		"[a-zA-Z]+.*[_][a-z]+[(][)][\\s]{",
+	)
+	matches := regexFuncs.FindAllIndex([]byte(scontent), -1)
+	// Drop section after functions (src_*, *() {)
+	if len(matches) > 0 {
+		ebuild = ebuild + scontent[:matches[0][0]]
+	} else {
+		ebuild = ebuild + scontent
+	}
+
+	// [[ ${PV} == "9999" ]] is not supported. Workaround but we need a better solution.
+	regexDoubleBrakets := regexp.MustCompile(
+		//"[[][[].*",
+		"^[[][[].*",
+		//"^.*\[\[.*\]\]",
+	)
+	matchDB := regexDoubleBrakets.FindAllIndex([]byte(ebuild), -1)
+	if len(matchDB) > 0 {
+		ebuild = ebuild[:matchDB[0][0]] + "#" + ebuild[matchDB[0][0]:]
+	}
+
+	//fmt.Println("EBUILD ", ebuild)
+
+	file, err := syntax.NewParser().Parse(strings.NewReader(ebuild), path)
 	if err != nil {
 		return nil, fmt.Errorf("could not parse: %v", err)
 	}
@@ -263,17 +339,18 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) ([]pkg.Package, error) {
 		Name:     gp.Name,
 		Version:  fmt.Sprintf("%s%s", gp.Version, gp.VersionSuffix),
 		Category: gp.Category,
+		Uri:      make([]string, 0),
 	}
 
-	Debug("Prepare package ", pack)
+	Debug("Prepare package ", pack.Category+"/"+pack.Name+"-"+pack.Version)
 
 	// Adding a timeout of 60secs, as with some bash files it can hang indefinetly
 	timeout, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 	vars, err := SourceFile(timeout, path, gp)
 	if err != nil {
-		return []pkg.Package{pack}, nil
-		//	return []pkg.Package{}, err
+		Error("Error on source file ", pack.Name, ": ", err)
+		return []pkg.Package{}, err
 	}
 
 	// TODO: Handle this a bit better
@@ -285,10 +362,49 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) ([]pkg.Package, error) {
 		}
 	}
 
+	// Retrieve package description
+	descr, ok := vars["DESCRIPTION"]
+	if ok {
+		pack.SetDescription(descr.String())
+	}
+	// Retrieve package license
+	license, ok := vars["LICENSE"]
+	if ok {
+		pack.SetLicense(license.String())
+	}
+	uri, ok := vars["SRC_URI"]
+	if ok {
+		// TODO: handle mirror:
+		uris := strings.Split(uri.String(), "\n")
+		for _, u := range uris {
+			u = strings.TrimSpace(u)
+
+			if u == "" {
+				continue
+			}
+			if match, _ := regexp.Match(uriRegex, []byte(u)); match {
+				if strings.Index(u, "(") >= 0 {
+					regexUri := regexp.MustCompile("(http|ftp|mirror).*[ ]")
+					matches := regexUri.FindAllIndex([]byte(u), -1)
+					if len(matches) > 0 {
+						u = u[matches[0][0]:matches[0][1]]
+					} else {
+						continue
+					}
+				}
+				pack.AddURI(u)
+				Debug("Add uri ", u)
+			} else {
+				Debug("Skip uri ", u)
+			}
+		}
+	}
+
 	rdepend, ok := vars["RDEPEND"]
 	if ok {
 		gRDEPEND, err := ParseRDEPEND(rdepend.String())
 		if err != nil {
+			Warning("Error on parsing RDEPEND for package ", pack.Category+"/"+pack.Name, err)
 			return []pkg.Package{pack}, nil
 			//	return []pkg.Package{}, err
 		}

pkg/tree/recipes_test.go

@@ -120,7 +120,7 @@ var _ = Describe("Recipe", func() {
 				s := solver.NewSolver(pkg.NewInMemoryDatabase(false), tree, tree)
 				solution, err := s.Install([]pkg.Package{pack})
 				Expect(err).ToNot(HaveOccurred())
-				Expect(len(solution)).To(Equal(10))
+				Expect(len(solution)).To(Equal(33))
 
 				var allSol string
 				for _, sol := range solution {

pkg/tree/tree_suite_test.go

@@ -18,11 +18,15 @@ package tree_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestTree(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Tree Suite")
 }

tests/fixtures/complex/selection/0.1/build.yaml

@@ -0,0 +1,2 @@
+
+image: "alpine:3.5"

tests/fixtures/complex/selection/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "build"
+version: "0.1"

tests/fixtures/complex/selection/0.20191212/build.yaml

@@ -0,0 +1,17 @@
+requires:
+
+# This pins the portage version
+#- category: "layer"
+#  name: "portage"
+#  version: ">=0.1"
+- category: "layer"
+  name: "sabayon-build-portage"
+  version: "0.20191126"
+
+# This pins sabayon overlays (sabayon-distro, for-gentoo)
+#- category: "layer"
+#  name: "sabayon-overlay"
+#  version: ">=0.1"
+- category: "layer"
+  name: "build-sabayon-overlay"
+  version: "0.20191212"

tests/fixtures/complex/selection/0.20191212/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "build-sabayon-overlays"
+version: "0.20191212"

tests/fixtures/complex/selection/5.4.2/build.yaml

@@ -0,0 +1,9 @@
+steps:
+- echo "vhba" > /vhba
+requires:
+- category: "sys-kernel"
+  version:  "5.4.2"
+  name:     "linux-sabayon"
+- category: "sys-kernel"
+  version:  "5.4.2"
+  name:     "sabayon-sources"

tests/fixtures/complex/selection/5.4.2/definition.yaml

@@ -0,0 +1,3 @@
+category: "sys-fs-5.4.2"
+name: "vhba"
+version: "20190410"

tests/fixtures/complex/selection/kernel/build.yaml

@@ -0,0 +1,7 @@
+requires:
+- category: "layer"
+  version:  "0.1"
+  name:     "build-sabayon-overlays"
+
+includes:
+- /usr/portage/packages/.*

tests/fixtures/complex/selection/kernel/definition.yaml

@@ -0,0 +1,6 @@
+category: sys-kernel
+description: Official Sabayon Linux Standard kernel image
+license: ""
+name: linux-sabayon
+uri: []
+version: 5.4.2

tests/fixtures/complex/selection/meta/build.yaml

@@ -0,0 +1,18 @@
+
+requires:
+
+# This pins the portage version
+#- category: "layer"
+#  name: "portage"
+#  version: ">=0.1"
+- category: "layer"
+  name: "sabayon-build-portage"
+  version: "0.20191126"
+
+# This pins sabayon overlays (sabayon-distro, for-gentoo)
+#- category: "layer"
+#  name: "sabayon-overlay"
+#  version: ">=0.1"
+- category: "layer"
+  name: "build-sabayon-overlay"
+  version: ">=0.1"

tests/fixtures/complex/selection/meta/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "build-sabayon-overlays"
+version: "0.1"

tests/fixtures/complex/selection/sabayon-build-overlays/portage/0.20191126/build.yaml

@@ -0,0 +1,4 @@
+requires:
+- category: "layer"
+  name: "build"
+  version: ">=0.1"

tests/fixtures/complex/selection/sabayon-build-overlays/portage/0.20191126/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "sabayon-build-portage"
+version: "0.20191126"

tests/fixtures/complex/selection/sabayon-build-overlays/portage/0.20191212/build.yaml

@@ -0,0 +1,4 @@
+requires:
+- category: "layer"
+  version:  "0.1"
+  name:     "build"

tests/fixtures/complex/selection/sabayon-build-overlays/portage/0.20191212/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "sabayon-build-portage"
+version: "0.20191212"

tests/fixtures/complex/selection/sabayon-build-overlays/sabayon-overlay/0.20191205/build.yaml

@@ -0,0 +1,7 @@
+requires:
+- category: "layer"
+  name: "build"
+  version: ">=0.1"
+- category: "layer"
+  name: "sabayon-build-portage"
+  version: ">=0.1"

tests/fixtures/complex/selection/sabayon-build-overlays/sabayon-overlay/0.20191205/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "build-sabayon-overlay"
+version: "0.20191205"

tests/fixtures/complex/selection/sabayon-build-overlays/sabayon-overlay/0.20191212/build.yaml

@@ -0,0 +1,7 @@
+requires:
+- category: "layer"
+  name: "build"
+  version: ">=0.1"
+- category: "layer"
+  name: "sabayon-build-portage"
+  version: ">=0.1"

tests/fixtures/complex/selection/sabayon-build-overlays/sabayon-overlay/0.20191212/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "build-sabayon-overlay"
+version: "0.20191212"

tests/fixtures/complex/selection/sources5.4.2/build.yaml

@@ -0,0 +1,7 @@
+requires:
+- category: "layer"
+  version:  "0.1"
+  name:     "build-sabayon-overlays"
+
+includes:
+- /usr/portage/packages/.*

tests/fixtures/complex/selection/sources5.4.2/definition.yaml

@@ -0,0 +1,8 @@
+category: sys-kernel
+description: Official Sabayon Linux Standard kernel sources
+license: ""
+name: sabayon-sources
+uri: []
+use_flags:
+- sources_standalone
+version: 5.4.2

tests/fixtures/parser/calamares-sabayon-base-modules-1.15.ebuild

@@ -0,0 +1,25 @@
+# Copyright 1999-2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=5
+
+DESCRIPTION="Sabayon Official Calamares base modules"
+HOMEPAGE="http://www.sabayon.org/"
+SRC_URI="https://github.com/Sabayon/calamares-sabayon/archive/v${PV}.tar.gz -> ${P}.tar.gz"
+LICENSE="CC-BY-SA-4.0"
+SLOT="0"
+KEYWORDS="~amd64 ~x86"
+IUSE=""
+
+DEPEND="app-admin/calamares[networkmanager,upower]"
+RDEPEND="${DEPEND}
+	>=sys-kernel/dracut-049"
+
+S="${WORKDIR}/calamares-sabayon-${PV}"
+
+src_install() {
+	insinto "/etc/calamares/"
+	doins -r "${FILESDIR}/modules-conf/"*
+	insinto "/usr/lib/calamares/modules/"
+	doins -r "${S}/"*
+}

tests/fixtures/parser/kodi-raspberrypi-16.0.ebuild

@@ -0,0 +1,265 @@
+# Copyright 1999-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI="5"
+
+# Does not work with py3 here
+# It might work with py:2.5 but I didn't test that
+PYTHON_COMPAT=( python2_7 )
+PYTHON_REQ_USE="sqlite"
+
+inherit eutils linux-info python-single-r1 multiprocessing autotools systemd
+
+CODENAME="Jarvis"
+case ${PV} in
+9999)
+	EGIT_REPO_URI="git://github.com/xbmc/xbmc.git"
+	inherit git-r3
+	;;
+*|*_p*)
+	MY_PV=${PV/_p/_r}
+	MY_P="kodi-${MY_PV}"
+	SRC_URI="http://mirrors.kodi.tv/releases/source/${MY_PV}-${CODENAME}.tar.gz -> ${P}.tar.gz
+		https://github.com/xbmc/xbmc/archive/${PV}-${CODENAME}.tar.gz -> ${P}.tar.gz
+		!java? ( http://mirrors.kodi.tv/releases/source/${MY_P}-generated-addons.tar.xz )"
+	KEYWORDS="~arm"
+	S=${WORKDIR}/xbmc-${PV}-${CODENAME}
+	;;
+esac
+
+DESCRIPTION="Kodi is a free and open source media-player and entertainment hub"
+HOMEPAGE="http://kodi.tv/ http://kodi.wiki/"
+
+LICENSE="GPL-2"
+SLOT="0"
+IUSE="airplay +alsa avahi bluetooth bluray caps +cec css dbus debug java joystick midi mysql +nfs profile -projectm pulseaudio rtmp +samba sftp test +texturepacker udisks upnp upower +usb vaapi vdpau webserver -X"
+REQUIRED_USE="
+	udisks? ( dbus )
+	upower? ( dbus )
+"
+
+COMMON_DEPEND="${PYTHON_DEPS}
+	app-arch/bzip2
+	app-arch/unzip
+	app-arch/zip
+	app-i18n/enca
+	airplay? ( app-pda/libplist )
+	dev-libs/boost
+	dev-libs/expat
+	dev-libs/fribidi
+	dev-libs/libcdio[-minimal]
+	cec? ( >=dev-libs/libcec-3.0 )
+	dev-libs/libpcre[cxx]
+	dev-libs/libxml2
+	sys-apps/lsb-release
+	dev-libs/libxslt
+	>=dev-libs/lzo-2.04
+	dev-libs/tinyxml[stl]
+	dev-libs/yajl
+	dev-python/simplejson[${PYTHON_USEDEP}]
+	media-fonts/corefonts
+	media-fonts/roboto
+	alsa? ( media-libs/alsa-lib )
+	media-libs/flac
+	media-libs/fontconfig
+	media-libs/freetype
+	media-libs/jasper
+	x11-apps/xrefresh
+	media-libs/jbigkit
+	>=media-libs/libass-0.9.7
+	net-libs/libssh
+	bluray? ( >=media-libs/libbluray-0.7.0 )
+	css? ( media-libs/libdvdcss )
+	media-libs/libmad
+	media-libs/libmodplug
+	media-libs/libmpeg2
+	media-libs/libogg
+	media-libs/libpng:0=
+	media-libs/libsamplerate
+	joystick? ( media-libs/libsdl2 )
+	>=media-libs/taglib-1.8
+	media-libs/libvorbis
+	media-libs/tiff:0=
+	media-sound/dcadec
+	pulseaudio? ( media-sound/pulseaudio )
+	media-sound/wavpack
+	media-video/omxplayer
+	rtmp? ( media-video/rtmpdump )
+	avahi? ( net-dns/avahi )
+	nfs? ( net-fs/libnfs:= )
+	webserver? ( net-libs/libmicrohttpd[messages] )
+	sftp? ( net-libs/libssh[sftp] )
+	net-misc/curl
+	samba? ( >=net-fs/samba-3.4.6[smbclient(+)] )
+	bluetooth? ( net-wireless/bluez )
+	dbus? ( sys-apps/dbus )
+	caps? ( sys-libs/libcap )
+	sys-libs/zlib
+	virtual/jpeg:0=
+	usb? ( virtual/libusb:1 )
+	mysql? ( virtual/mysql )
+	media-libs/mesa[gles2]
+	vaapi? ( x11-libs/libva[opengl] )
+	vdpau? (
+		|| ( >=x11-libs/libvdpau-1.1 >=x11-drivers/nvidia-drivers-180.51 )
+	)
+	X? (
+		x11-apps/xdpyinfo
+		x11-apps/mesa-progs
+		x11-libs/libXinerama
+		x11-libs/libXrandr
+		x11-libs/libXrender
+	)"
+RDEPEND="${COMMON_DEPEND}
+	!media-tv/xbmc
+	!media-tv/kodi
+	udisks? ( sys-fs/udisks:0 )
+	upower? ( || ( sys-power/upower sys-power/upower-pm-utils ) )"
+DEPEND="${COMMON_DEPEND}
+	app-arch/xz-utils
+	dev-lang/swig
+	media-video/ffmpeg
+	dev-libs/crossguid
+	dev-util/gperf
+	X? ( x11-proto/xineramaproto )
+	dev-util/cmake
+	x86? ( dev-lang/nasm )
+	java? ( virtual/jre )
+	test? ( dev-cpp/gtest )
+	virtual/pkgconfig"
+# Force java for latest git version to avoid having to hand maintain the
+# generated addons package.  #488118
+[[ ${PV} == "9999" ]] && DEPEND+=" virtual/jre"
+
+CONFIG_CHECK="~IP_MULTICAST"
+ERROR_IP_MULTICAST="
+In some cases Kodi needs to access multicast addresses.
+Please consider enabling IP_MULTICAST under Networking options.
+"
+
+pkg_setup() {
+	check_extra_config
+	python-single-r1_pkg_setup
+}
+
+src_unpack() {
+	[[ ${PV} == "9999" ]] && git-r3_src_unpack || default
+}
+
+src_prepare() {
+	epatch "${FILESDIR}"/${PN}-9999-no-arm-flags.patch #400617
+	epatch "${FILESDIR}"/${PN}-9999-texturepacker.patch
+	epatch "${FILESDIR}"/${PN}-16-ffmpeg3.patch
+
+	# some dirs ship generated autotools, some dont
+	multijob_init
+	local d dirs=(
+		tools/depends/native/TexturePacker/src/configure
+		$(printf 'f:\n\t@echo $(BOOTSTRAP_TARGETS)\ninclude bootstrap.mk\n' | emake -f - f)
+	)
+	for d in "${dirs[@]}" ; do
+		[[ -e ${d} ]] && continue
+		pushd ${d/%configure/.} >/dev/null || die
+		AT_NOELIBTOOLIZE="yes" AT_TOPLEVEL_EAUTORECONF="yes" \
+		multijob_child_init eautoreconf
+		popd >/dev/null
+	done
+	multijob_finish
+	elibtoolize
+
+	[[ ${PV} == "9999" ]] && emake -f codegenerator.mk
+
+	# Disable internal func checks as our USE/DEPEND
+	# stuff handles this just fine already #408395
+	export ac_cv_lib_avcodec_ff_vdpau_vc1_decode_picture=yes
+
+	# Fix the final version string showing as "exported"
+	# instead of the SVN revision number.
+	export HAVE_GIT=no GIT_REV=${EGIT_VERSION:-exported}
+
+	# avoid long delays when powerkit isn't running #348580
+	sed -i \
+		-e '/dbus_connection_send_with_reply_and_block/s:-1:3000:' \
+		xbmc/linux/*.cpp || die
+
+	epatch_user #293109
+
+	# Tweak autotool timestamps to avoid regeneration
+	find . -type f -exec touch -r configure {} +
+}
+
+src_configure() {
+	# Disable documentation generation
+	export ac_cv_path_LATEX=no
+	# Avoid help2man
+	export HELP2MAN=$(type -P help2man || echo true)
+	# No configure flage for this #403561
+	export ac_cv_lib_bluetooth_hci_devid=$(usex bluetooth)
+	# Requiring java is asine #434662
+	[[ ${PV} != "9999" ]] && export ac_cv_path_JAVA_EXE=$(which $(usex java java true))
+
+	econf \
+		--docdir=/usr/share/doc/${PF} \
+		--disable-gl \
+		--enable-gles \
+		--with-platform=raspberry-pi \
+		--disable-sdl \
+		--enable-optimizations \
+		--disable-x11 \
+		--disable-goom \
+		--disable-xrandr \
+		--disable-mid \
+		--enable-nfs \
+		--disable-profiling \
+		--enable-rsxs \
+		--disable-debug \
+		--disable-joystick \
+		--disable-vaapi \
+		--disable-vdpau \
+		--disable-avahi \
+		--enable-libcec \
+		--disable-pulse \
+		--disable-projectm \
+		--disable-optical-drive \
+		--disable-dvdcss \
+		--disable-vtbdecoder \
+		--enable-alsa \
+		--enable-player=omxplayer
+}
+
+src_compile() {
+	emake V=1
+}
+
+src_install() {
+	default
+	rm "${ED}"/usr/share/doc/*/{LICENSE.GPL,copying.txt}* || die
+
+	domenu tools/Linux/kodi.desktop
+	newicon media/icon48x48.png kodi.png
+
+	insinto /etc/udev/rules.d
+	newins "${FILESDIR}/99-input.rules" 99-input.rules
+
+	# Remove fonconfig settings that are used only on MacOSX.
+	# Can't be patched upstream because they just find all files and install
+	# them into same structure like they have in git.
+	rm -rf "${ED}"/usr/share/kodi/system/players/dvdplayer/etc
+
+	# Replace bundled fonts with system ones
+	# teletext.ttf: unknown
+	# bold-caps.ttf: unknown
+	# roboto: roboto-bold, roboto-regular
+	# arial.ttf: font mashed from droid/roboto, not removed wrt bug#460514
+	rm -rf "${ED}"/usr/share/kodi/addons/skin.confluence/fonts/Roboto-* || die
+	dosym /usr/share/fonts/roboto/Roboto-Regular.ttf \
+		/usr/share/kodi/addons/skin.confluence/fonts/Roboto-Regular.ttf
+	dosym /usr/share/fonts/roboto/Roboto-Bold.ttf \
+		/usr/share/kodi/addons/skin.confluence/fonts/Roboto-Bold.ttf
+
+	python_domodule tools/EventClients/lib/python/xbmcclient.py
+	python_newscript "tools/EventClients/Clients/Kodi Send/kodi-send.py" kodi-send
+	dobin "${FILESDIR}"/startkodi
+	systemd_dounit "${FILESDIR}"/${PN}.service
+
+}

tests/fixtures/parser/libreoffice-l10n-meta-6.2.8.2.ebuild

@@ -0,0 +1,32 @@
+# Copyright 2004-2008 Sabayon Linux
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=4
+
+DESCRIPTION="LibreOffice.org localisation meta-package"
+HOMEPAGE="http://www.documentfoundation.org"
+LICENSE="LGPL-2"
+SLOT="0"
+KEYWORDS="~amd64 ~x86"
+SRC_URI=""
+RDEPEND=""
+DEPEND=""
+IUSE=""
+
+SPELL_DIRS="af bg ca cs cy da de el en eo es et fo fr ga gl he hr hu ia id it \
+lt lv mi mk nb nl pl pt ro ru sk sl sv sw tn uk zu"
+
+LANGS="af am ar as ast be bg bn bo br brx bs ca cs cy da de dgo dz el \
+en_GB en_US en_ZA eo es et eu fa fi fr ga gd gl gu gug he hi hr hu id is it ja ka kk km kn \
+ko kok ks lb lo lt lv mai mk ml mn mni mr my nb ne nl nn nr nso oc om or pa_IN \
+pl pt pt_BR ro ru rw sa_IN sat sd si sid sk sl sq sr ss st sv sw_TZ ta te tg \
+th tn tr ts tt ug uk uz ve vi xh zh_CN zh_TW zu"
+
+for X in ${LANGS}; do
+	IUSE+=" linguas_${X}"
+	RDEPEND+=" linguas_${X}? ( ~app-office/libreoffice-l10n-${X}-${PV} )"
+done
+for X in ${SPELL_DIRS}; do
+	IUSE+=" linguas_${X}"
+	RDEPEND+=" linguas_${X}? ( app-dicts/myspell-${X} )"
+done

tests/fixtures/parser/linux-sources-1.ebuild

@@ -0,0 +1,42 @@
+# Copyright 1999-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=2
+
+DESCRIPTION="Virtual for Linux kernel sources"
+HOMEPAGE=""
+SRC_URI=""
+
+LICENSE=""
+SLOT="0"
+KEYWORDS="~amd64 ~arm ~hppa ~ia64 ~x86"
+IUSE="firmware"
+
+SABAYON_SOURCES="sys-kernel/sabayon-sources
+		sys-kernel/server-sources
+		sys-kernel/rt-sources
+		sys-kernel/efikamx-sources
+		sys-kernel/odroid-sources
+		sys-kernel/beagle-sources
+		sys-kernel/beaglebone-sources"
+
+DEPEND="firmware? ( sys-kernel/linux-firmware )"
+RDEPEND="|| (
+		${SABAYON_SOURCES}
+		sys-kernel/gentoo-sources
+		sys-kernel/vanilla-sources
+		sys-kernel/ck-sources
+		sys-kernel/git-sources
+		sys-kernel/hardened-sources
+		sys-kernel/mips-sources
+		sys-kernel/openvz-sources
+		sys-kernel/pf-sources
+		sys-kernel/rsbac-sources
+		sys-kernel/sparc-sources
+		sys-kernel/tuxonice-sources
+		sys-kernel/usermode-sources
+		sys-kernel/vserver-sources
+		sys-kernel/xbox-sources
+		sys-kernel/xen-sources
+		sys-kernel/zen-sources
+	)"

tests/fixtures/parser/mod_dav_svn-1.12.2.ebuild

@@ -0,0 +1,298 @@
+# Copyright 1999-2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=6
+
+WANT_AUTOMAKE="none"
+MY_P="${P/_/-}"
+
+MY_SVN_PN="subversion"
+MY_SVN_P="${MY_SVN_PN}-${PV}"
+MY_SVN_PF="${MY_SVN_PN}-${PVR}"
+
+inherit autotools db-use depend.apache flag-o-matic libtool multilib xdg-utils
+
+DESCRIPTION="Subversion WebDAV support"
+HOMEPAGE="https://subversion.apache.org/"
+SRC_URI="mirror://apache/${MY_SVN_PN}/${MY_SVN_P}.tar.bz2
+	https://dev.gentoo.org/~polynomial-c/${MY_SVN_PN}-1.10.0_rc1-patches-1.tar.xz"
+S="${WORKDIR}/${MY_SVN_P}"
+
+LICENSE="Subversion"
+SLOT="0"
+KEYWORDS="~amd64 ~arm ~x86"
+IUSE="berkdb debug +dso nls sasl"
+# w/o: ctypes-python doc extras gnome-keyring java kde perl python ruby
+# test vim-syntax; implicit: apache2, http
+
+# This is an ebuild that provides mod_dav_svn and friends (it does more or
+# less the same as when USE="apache2 http" is added to dev-vcs/subversion - basically
+# provides three Apache modules and a configuration file), suitable for binary
+# packages.
+# Some flags in IUSE and their handling are only used to enforce the code be
+# compiled sufficiently the same way Subversion itself was - extra carefulness.
+# In the process of building libraries for WebDAV, a few unused libraries are
+# also built (not the whole Subversion, though, which is a win). Some build
+# time dependencies here are just for them.
+
+# If you are building it for yourself, you don't need it.
+# USE=apache2 emerge dev-vcs/subversion will do what you want.
+# However, you can use this ebuild too.
+
+# variable specific to www-apache/mod_dav_svn
+MY_CDEPS="
+	~dev-vcs/subversion-${PV}[berkdb=,debug=,dso=,nls=,sasl=,http]
+	app-arch/bzip2
+	app-arch/lz4
+	>=dev-db/sqlite-3.7.12
+	>=dev-libs/apr-1.3:1
+	>=dev-libs/apr-util-1.3:1
+	dev-libs/expat
+	dev-libs/libutf8proc:=
+	sys-apps/file
+	sys-libs/zlib
+	berkdb? ( >=sys-libs/db-4.0.14:= )
+"
+
+DEPEND="${MY_CDEPS}
+	>=net-libs/serf-1.3.4
+	sasl? ( dev-libs/cyrus-sasl )
+	virtual/pkgconfig
+	!!<sys-apps/sandbox-1.6
+	nls? ( sys-devel/gettext )
+	sys-apps/file"
+RDEPEND="${MY_CDEPS}
+	!dev-vcs/subversion[apache2]
+
+	www-servers/apache[apache2_modules_dav]
+	nls? ( virtual/libintl )"
+
+need_apache # was: want_apache
+
+pkg_setup() {
+	if use berkdb ; then
+		local apu_bdb_version="$(${EPREFIX}/usr/bin/apu-1-config --includes \
+			| grep -Eoe '-I${EPREFIX}/usr/include/db[[:digit:]]\.[[:digit:]]' \
+			| sed 's:.*b::')"
+		einfo
+		if [[ -z "${SVN_BDB_VERSION}" ]] ; then
+			if [[ -n "${apu_bdb_version}" ]] ; then
+				SVN_BDB_VERSION="${apu_bdb_version}"
+				einfo "Matching db version to apr-util"
+			else
+				SVN_BDB_VERSION="$(db_ver_to_slot "$(db_findver sys-libs/db 2>/dev/null)")"
+				einfo "SVN_BDB_VERSION variable isn't set. You can set it to enforce using of specific version of Berkeley DB."
+			fi
+		fi
+		einfo "Using: Berkeley DB ${SVN_BDB_VERSION}"
+		einfo
+
+		if [[ -n "${apu_bdb_version}" && "${SVN_BDB_VERSION}" != "${apu_bdb_version}" ]]; then
+			eerror "APR-Util is linked against Berkeley DB ${apu_bdb_version}, but you are trying"
+			eerror "to build Subversion with support for Berkeley DB ${SVN_BDB_VERSION}."
+			eerror "Rebuild dev-libs/apr-util or set SVN_BDB_VERSION=\"${apu_bdb_version}\"."
+			eerror "Aborting to avoid possible run-time crashes."
+			die "Berkeley DB version mismatch"
+		fi
+	fi
+
+	# depend.apache_pkg_setup
+
+	# https://issues.apache.org/jira/browse/SVN-4813#comment-16813739
+	append-cppflags -P
+
+	if use debug ; then
+		append-cppflags -DSVN_DEBUG -DAP_DEBUG
+	fi
+
+	# http://mail-archives.apache.org/mod_mbox/subversion-dev/201306.mbox/%3C51C42014.3060700@wandisco.com%3E
+	[[ ${CHOST} == *-solaris2* ]] && append-cppflags -D__EXTENSIONS__
+
+	# Allow for custom repository locations.
+	SVN_REPOS_LOC="${SVN_REPOS_LOC:-${EPREFIX}/var/svn}"
+}
+
+src_prepare() {
+	eapply "${WORKDIR}/patches"
+	eapply_user
+
+	chmod +x build/transform_libtool_scripts.sh || die
+
+	sed -i \
+		-e "s/\(BUILD_RULES=.*\) bdb-test\(.*\)/\1\2/g" \
+		-e "s/\(BUILD_RULES=.*\) test\(.*\)/\1\2/g" configure.ac
+
+	# this bites us in particular on Solaris
+	sed -i -e '1c\#!/usr/bin/env sh' build/transform_libtool_scripts.sh || \
+		die "/bin/sh is not POSIX shell!"
+
+	eautoconf
+	elibtoolize
+
+	#sed -e 's/\(libsvn_swig_py\)-\(1\.la\)/\1-$(EPYTHON)-\2/g' \
+	#-i build-outputs.mk || die "sed failed"
+
+	xdg_environment_reset
+}
+
+src_configure() {
+	local myconf=(
+		--libdir="${EPREFIX%/}/usr/$(get_libdir)"
+		--with-apache-libexecdir
+		--with-apxs="${EPREFIX}"/usr/bin/apxs
+		$(use_with berkdb berkeley-db "db.h:${EPREFIX%/}/usr/include/db${SVN_BDB_VERSION}::db-${SVN_BDB_VERSION}")
+		--without-ctypesgen
+		$(use_enable dso runtime-module-search)
+		--without-gnome-keyring
+		--disable-javahl
+		--without-jdk
+		--without-kwallet
+		$(use_enable nls)
+		$(use_with sasl)
+		--with-serf
+		--with-apr="${EPREFIX%/}/usr/bin/apr-1-config"
+		--with-apr-util="${EPREFIX%/}/usr/bin/apu-1-config"
+		--disable-experimental-libtool
+		--without-jikes
+		--disable-mod-activation
+		--disable-static
+		--enable-svnxx
+	)
+
+	#use python || use perl || use ruby
+	myconf+=( --without-swig )
+
+	#use java
+	myconf+=( --without-junit )
+
+	case ${CHOST} in
+		*-aix*)
+			# avoid recording immediate path to sharedlibs into executables
+			append-ldflags -Wl,-bnoipath
+		;;
+		*-cygwin*)
+			# no LD_PRELOAD support, no undefined symbols
+			myconf+=( --disable-local-library-preloading LT_LDFLAGS=-no-undefined )
+			;;
+		*-interix*)
+			# loader crashes on the LD_PRELOADs...
+			myconf+=( --disable-local-library-preloading )
+		;;
+		*-solaris*)
+			# need -lintl to link
+			use nls && append-libs intl
+			# this breaks installation, on x64 echo replacement is 32-bits
+			myconf+=( --disable-local-library-preloading )
+		;;
+		*-mint*)
+			myconf+=( --enable-all-static --disable-local-library-preloading )
+		;;
+		*)
+			# inject LD_PRELOAD entries for easy in-tree development
+			myconf+=( --enable-local-library-preloading )
+		;;
+	esac
+
+	#workaround for bug 387057
+	has_version =dev-vcs/subversion-1.6* && myconf+=( --disable-disallowing-of-undefined-references )
+
+	#version 1.7.7 again tries to link against the older installed version and fails, when trying to
+	#compile for x86 on amd64, so workaround this issue again
+	#check newer versions, if this is still/again needed
+	#myconf+=( --disable-disallowing-of-undefined-references )
+
+	# allow overriding Python include directory
+	#ac_cv_path_RUBY=$(usex ruby "${EPREFIX%/}/usr/bin/ruby${RB_VER}" "none")
+	#ac_cv_path_RDOC=(usex ruby "${EPREFIX%/}/usr/bin/rdoc${RB_VER}" "none")
+	ac_cv_python_includes='-I$(PYTHON_INCLUDEDIR)' \
+	econf "${myconf[@]}"
+}
+
+src_compile() {
+	emake apache-mod
+}
+
+src_install() {
+	emake DESTDIR="${D}" install-mods-shared
+
+	# Install Apache module configuration.
+	#use apache2
+	keepdir "${APACHE_MODULES_CONFDIR}"
+	insinto "${APACHE_MODULES_CONFDIR}"
+	doins "${FILESDIR}/47_mod_dav_svn.conf"
+}
+
+pkg_preinst() {
+	# Compare versions of Berkeley DB, bug 122877.
+	if use berkdb && [[ -f "${EROOT}/usr/bin/svn" ]] ; then
+		OLD_BDB_VERSION="$(scanelf -nq "${EROOT}/usr/$(get_libdir)/libsvn_subr-1$(get_libname 0)" | grep -Eo "libdb-[[:digit:]]+\.[[:digit:]]+" | sed -e "s/libdb-\(.*\)/\1/")"
+		NEW_BDB_VERSION="$(scanelf -nq "${ED%/}/usr/$(get_libdir)/libsvn_subr-1$(get_libname 0)" | grep -Eo "libdb-[[:digit:]]+\.[[:digit:]]+" | sed -e "s/libdb-\(.*\)/\1/")"
+		if [[ "${OLD_BDB_VERSION}" != "${NEW_BDB_VERSION}" ]] ; then
+			CHANGED_BDB_VERSION="1"
+		fi
+	fi
+}
+
+pkg_postinst() {
+	if [[ -n "${CHANGED_BDB_VERSION}" ]] ; then
+		ewarn "You upgraded from an older version of Berkeley DB and may experience"
+		ewarn "problems with your repository. Run the following commands as root to fix it:"
+		ewarn "    db4_recover -h ${SVN_REPOS_LOC}/repos"
+		ewarn "    chown -Rf apache:apache ${SVN_REPOS_LOC}/repos"
+	fi
+
+	#ewarn "If you run subversion as a daemon, you will need to restart it to avoid module mismatches."
+
+	# from src_install in Gentoo ebuild:
+	##adjust default user and group with disabled apache2 USE flag, bug 381385
+	#use apache2 || sed -e "s\USER:-apache\USER:-svn\g" \
+	#		-e "s\GROUP:-apache\GROUP:-svnusers\g" \
+	#		-i "${ED}"etc/init.d/svnserve || die
+	#use apache2 || sed -e "0,/apache/s//svn/" \
+	#		-e "s:apache:svnusers:" \
+	#		-i "${ED}"etc/xinetd.d/svnserve || die
+	# We need to address it here with a message (when Subversion ebuild is
+	# intented to be build with USE=-apache2).
+	# Also, user doesn't need to tweak init.d script - user and group can
+	# be changed in conf.d.
+	elog "svnserve users: You may want to change user and group in /etc/conf.d/svnserve"
+	elog "and /etc/xinetd.d/svnserve from current svn:svnusers to apache:apache,"
+	elog "especially if you want to make use of emerge --config	${CATEGORY}/${PN}"
+	elog "and its default ownership settings."
+}
+
+#pkg_postrm()
+
+pkg_config() {
+	# Remember: Don't use ${EROOT}${SVN_REPOS_LOC} since ${SVN_REPOS_LOC}
+	# already has EPREFIX in it
+	einfo "Initializing the database in ${SVN_REPOS_LOC}..."
+	if [[ -e "${SVN_REPOS_LOC}/repos" ]] ; then
+		echo "A Subversion repository already exists and I will not overwrite it."
+		echo "Delete \"${SVN_REPOS_LOC}/repos\" first if you're sure you want to have a clean version."
+	else
+		mkdir -p "${SVN_REPOS_LOC}/conf"
+
+		einfo "Populating repository directory..."
+		# Create initial repository.
+		"${EROOT}/usr/bin/svnadmin" create "${SVN_REPOS_LOC}/repos"
+
+		einfo "Setting repository permissions..."
+		SVNSERVE_USER="$(. "${EROOT}/etc/conf.d/svnserve"; echo "${SVNSERVE_USER}")"
+		SVNSERVE_GROUP="$(. "${EROOT}/etc/conf.d/svnserve"; echo "${SVNSERVE_GROUP}")"
+		#use apache2
+		[[ -z "${SVNSERVE_USER}" ]] && SVNSERVE_USER="apache"
+		[[ -z "${SVNSERVE_GROUP}" ]] && SVNSERVE_GROUP="apache"
+		#use !apache2
+		#[[ -z "${SVNSERVE_USER}" ]] && SVNSERVE_USER="svn"
+		#[[ -z "${SVNSERVE_GROUP}" ]] && SVNSERVE_GROUP="svnusers"
+
+		chmod -Rf go-rwx "${SVN_REPOS_LOC}/conf"
+		chmod -Rf o-rwx "${SVN_REPOS_LOC}/repos"
+		echo "Please create \"${SVNSERVE_GROUP}\" group if it does not exist yet."
+		echo "Afterwards please create \"${SVNSERVE_USER}\" user with homedir \"${SVN_REPOS_LOC}\""
+		echo "and as part of the \"${SVNSERVE_GROUP}\" group if it does not exist yet."
+		echo "Finally, execute \"chown -Rf ${SVNSERVE_USER}:${SVNSERVE_GROUP} ${SVN_REPOS_LOC}/repos\""
+		echo "to finish the configuration."
+	fi
+}