Prepare for 0.6.2 tag

Add keep-exported-images to build to instruct to remove backend artifacts (docker images)

.gitignore

@@ -1,2 +1,4 @@
 *.swp
 luet
+tests/integration/shunit2
+tests/integration/bin

.travis.yml

@@ -9,7 +9,7 @@ before_install:
   - make deps
   - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && mkdir -p $HOME/bin && export PATH=$PATH:$HOME/bin && mv container-diff-linux-amd64 $HOME/bin/container-diff
 script:
-  - make multiarch-build test
+  - make multiarch-build test test-integration
 after_success:
   - make coverage
   - bash <(curl -s https://codecov.io/bash)

Makefile

@@ -21,6 +21,10 @@ test:
 	GO111MODULE=off go get github.com/onsi/gomega/...
 	ginkgo -race -r ./...
 
+.PHONY: test-integration
+test-integration:
+	tests/integration/run.sh
+
 .PHONY: coverage
 coverage:
 	go test ./... -race -coverprofile=coverage.txt -covermode=atomic
@@ -36,6 +40,8 @@ help:
 .PHONY: clean
 clean:
 	rm -rf release/
+	rm -rf tests/integration/shunit2
+	rm -rf tests/integration/bin
 
 .PHONY: deps
 deps:

README.md

@@ -4,18 +4,60 @@
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
 [![codecov](https://codecov.io/gh/mudler/luet/branch/master/graph/badge.svg)](https://codecov.io/gh/mudler/luet)
 
-Luet is a Package Manager based off from containers - it uses Docker (and other tech) to sandbox your builds and generate packages from them. It has no dependencies and it is well suitable for "from scratch" environments.
+Luet is a multi-platform Package Manager based off from containers - it uses Docker (and other tech) to sandbox your builds and generate packages from them. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
+
+It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/specfile/) in YAML notation to define both packages and rootfs. As it is based on containers, it can be used to build seed stages for Linux From Scratch installations and it can build and track updates for those systems.
+
+It is written entirely in Golang and where used as package manager, it can run in from scratch environment, with zero dependencies.
 
 ## In a glance
 
 - Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.
 - It builds, installs, uninstalls and perform upgrades on machines
-- Installer doesn't depend on anything
+- Installer doesn't depend on anything ( 0 dep installer !), statically built
 - Support for packages as "layers"
 - It uses SAT solving techniques to solve the deptree ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
 
+## Install
+
+To install luet, you can grab a release on the [Release page](https://github.com/mudler/luet/releases) or compile it in your machine (requires Golang installed):
+
+    $ git clone https://github.com/mudler/luet.git
+    $ cd luet
+    $ make build
+
 ## Status
 
 Luet is not feature-complete yet, it can build, install/uninstall/upgrade packages - but it doesn't support yet all the features you would normally expect from a Package Manager nowadays.
 
-Check out the [Wiki](https://github.com/mudler/luet/wiki) for more informations.
+# Dependency solving
+
+Luet uses SAT and Reinforcement learning engine for dependency solving.
+It encodes the package requirements into a SAT problem, using gophersat to solve the dependency tree and give a concrete model as result.
+
+## SAT encoding
+
+Each package and its constraints are encoded and built around [OPIUM](https://ranjitjhala.github.io/static/opium.pdf). Additionally, Luet treats
+also selectors seamlessly while building the model, adding *ALO* ( *At least one* ) and *AMO* ( *At most one* ) rules to guarantee coherence within the installed system.
+
+## Reinforcement learning
+
+Luet also implements a small and portable qlearning agent that will try to solve conflict on your behalf
+when they arises while trying to validate your queries against the system model.
+
+To leverage it, simply pass ```--solver-type qlearning``` to the subcommands that supports it ( you can check out by invoking ```--help``` ).
+
+## Documentation
+
+[Documentation](https://luet-lab.github.io/docs) is available, or
+run `luet --help`,  any subcommand is documented as well, try e.g.: `luet build --help`.
+
+## Authors
+
+Luet is here thanks to our amazing [contributors](https://github.com/mudler/luet/graphs/contributors)!.
+
+Luet was originally created by Ettore Di Giacinto, mudler@sabayon.org, mudler@gentoo.org.
+
+## License
+
+Luet is distributed under the terms of GPLv3, check out the LICENSE file.

cmd/build.go

@@ -17,11 +17,11 @@ package cmd
 import (
 	"io/ioutil"
 	"os"
-	"regexp"
-	"runtime"
 
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/config"
+	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	tree "github.com/mudler/luet/pkg/tree"
@@ -35,25 +35,49 @@ var buildCmd = &cobra.Command{
 	Short: "build a package or a tree",
 	Long:  `build packages or trees from luet tree definitions. Packages are in [category]/[name]-[version] form`,
 	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("clean", cmd.Flags().Lookup("clean"))
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
 		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
 		viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged"))
 		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
 		viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
 		viper.BindPFlag("all", cmd.Flags().Lookup("all"))
+		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+
+		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
+		viper.BindPFlag("push", cmd.Flags().Lookup("push"))
+		viper.BindPFlag("pull", cmd.Flags().Lookup("pull"))
+		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
+
+		LuetCfg.Viper.BindPFlag("keep-exported-images", cmd.Flags().Lookup("keep-exported-images"))
+
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
+		clean := viper.GetBool("clean")
 		src := viper.GetString("tree")
 		dst := viper.GetString("destination")
-		concurrency := viper.GetInt("concurrency")
+		concurrency := LuetCfg.GetGeneral().Concurrency
 		backendType := viper.GetString("backend")
 		privileged := viper.GetBool("privileged")
 		revdeps := viper.GetBool("revdeps")
 		all := viper.GetBool("all")
 		databaseType := viper.GetString("database")
+		compressionType := viper.GetString("compression")
+		imageRepository := viper.GetString("image-repository")
+		push := viper.GetBool("push")
+		pull := viper.GetBool("pull")
+		keepImages := viper.GetBool("keep-images")
+		nodeps := viper.GetBool("nodeps")
+		onlydeps := viper.GetBool("onlydeps")
+		keepExportedImages := viper.GetBool("keep-exported-images")
 
 		compilerSpecs := compiler.NewLuetCompilationspecs()
 		var compilerBackend compiler.CompilerBackend
@@ -88,19 +112,42 @@ var buildCmd = &cobra.Command{
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase())
 
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		opts := compiler.NewDefaultCompilerOptions()
+		opts.SolverOptions = *LuetCfg.GetSolverOptions()
+		opts.ImageRepository = imageRepository
+		opts.Clean = clean
+		opts.PullFirst = pull
+		opts.KeepImg = keepImages
+		opts.Push = push
+		opts.OnlyDeps = onlydeps
+		opts.NoDeps = nodeps
+		opts.KeepImageExport = keepExportedImages
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts)
+		luetCompiler.SetConcurrency(concurrency)
+		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
 		if !all {
 			for _, a := range args {
-				decodepackage, err := regexp.Compile(`^([<>]?\~?=?)((([^\/]+)\/)?(?U)(\S+))(-(\d+(\.\d+)*[a-z]?(_(alpha|beta|pre|rc|p)\d*)*(-r\d+)?))?$`)
+
+				pack, err := helpers.ParsePackageStr(a)
 				if err != nil {
-					Fatal("Error: " + err.Error())
+					Fatal("Invalid package string ", a, ": ", err.Error())
 				}
-				packageInfo := decodepackage.FindAllStringSubmatch(a, -1)
-				category := packageInfo[0][4]
-				name := packageInfo[0][5]
-				version := packageInfo[0][1] + packageInfo[0][7]
-				spec, err := luetCompiler.FromPackage(&pkg.DefaultPackage{Name: name, Category: category, Version: version})
+
+				spec, err := luetCompiler.FromPackage(pack)
 				if err != nil {
 					Fatal("Error: " + err.Error())
 				}
@@ -125,10 +172,10 @@ var buildCmd = &cobra.Command{
 		var artifact []compiler.Artifact
 		var errs []error
 		if revdeps {
-			artifact, errs = luetCompiler.CompileWithReverseDeps(concurrency, privileged, compilerSpecs)
+			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
 		} else {
-			artifact, errs = luetCompiler.CompileParallel(concurrency, privileged, compilerSpecs)
+			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
 
 		}
 		if len(errs) != 0 {
@@ -148,14 +195,27 @@ func init() {
 	if err != nil {
 		Fatal(err)
 	}
+	buildCmd.Flags().Bool("clean", true, "Build all packages without considering the packages present in the build directory")
 	buildCmd.Flags().String("tree", path, "Source luet tree")
 	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
-	buildCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
 	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
 	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all packages in the tree")
 	buildCmd.Flags().String("destination", path, "Destination folder")
+	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
+	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+	buildCmd.Flags().Bool("push", false, "Push images to a hub")
+	buildCmd.Flags().Bool("pull", false, "Pull images from a hub")
+	buildCmd.Flags().Bool("keep-images", true, "Keep built docker images in the host")
+	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
+	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
+	buildCmd.Flags().Bool("keep-exported-images", false, "Keep exported images used during building")
+
+	buildCmd.Flags().String("solver-type", "", "Solver strategy")
+	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 
 	RootCmd.AddCommand(buildCmd)
 }

cmd/cleanup.go

@@ -0,0 +1,70 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	config "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+var cleanupCmd = &cobra.Command{
+	Use:   "cleanup",
+	Short: "Clean packages cache.",
+	Long:  `remove downloaded packages tarballs and clean cache directory`,
+	Run: func(cmd *cobra.Command, args []string) {
+		var cleaned int = 0
+
+		// Check if cache dir exists
+		if helpers.Exists(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
+
+			files, err := ioutil.ReadDir(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
+			if err != nil {
+				Fatal("Error on read cachedir ", err.Error())
+			}
+
+			for _, file := range files {
+				if file.IsDir() {
+					continue
+				}
+
+				if config.LuetCfg.GetGeneral().Debug {
+					Info("Removing ", file.Name())
+				}
+
+				err := os.RemoveAll(
+					filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
+				if err != nil {
+					Fatal("Error on removing", file.Name())
+				}
+				cleaned++
+			}
+		}
+
+		Info("Cleaned: ", cleaned, "packages.")
+
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(cleanupCmd)
+}

cmd/config.go

@@ -0,0 +1,59 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+
+	config "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+var configCmd = &cobra.Command{
+	Use:   "config",
+	Short: "Print config",
+	Long:  `Show luet configuration`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println(config.LuetCfg.GetLogging())
+		fmt.Println(config.LuetCfg.GetGeneral())
+		fmt.Println(config.LuetCfg.GetSystem())
+		if len(config.LuetCfg.CacheRepositories) > 0 {
+			fmt.Println("repetitors:")
+			for _, r := range config.LuetCfg.CacheRepositories {
+				fmt.Println("  - ", r.String())
+			}
+		}
+		if len(config.LuetCfg.SystemRepositories) > 0 {
+			fmt.Println("repositories:")
+			for _, r := range config.LuetCfg.SystemRepositories {
+				fmt.Println("  - ", r.String())
+			}
+		}
+
+		if len(config.LuetCfg.RepositoriesConfDir) > 0 {
+			fmt.Println("repos_confdir:")
+			for _, dir := range config.LuetCfg.RepositoriesConfDir {
+				fmt.Println("  - ", dir)
+			}
+		}
+
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(configCmd)
+}

cmd/convert.go

@@ -16,8 +16,8 @@ package cmd
 
 import (
 	"io/ioutil"
-	"runtime"
 
+	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	tree "github.com/mudler/luet/pkg/tree"
@@ -33,13 +33,11 @@ var convertCmd = &cobra.Command{
 	Long:  `Parses external PM and produces a luet parsable tree`,
 	PreRun: func(cmd *cobra.Command, args []string) {
 		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
 		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
 		t := viper.GetString("type")
-		c := viper.GetInt("concurrency")
 		databaseType := viper.GetString("database")
 		var db pkg.PackageDatabase
 
@@ -54,9 +52,15 @@ var convertCmd = &cobra.Command{
 		var builder tree.Parser
 		switch t {
 		case "gentoo":
-			builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c, gentoo.InMemory)
+			builder = gentoo.NewGentooBuilder(
+				&gentoo.SimpleEbuildParser{},
+				LuetCfg.GetGeneral().Concurrency,
+				gentoo.InMemory)
 		default: // dup
-			builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c, gentoo.InMemory)
+			builder = gentoo.NewGentooBuilder(
+				&gentoo.SimpleEbuildParser{},
+				LuetCfg.GetGeneral().Concurrency,
+				gentoo.InMemory)
 		}
 
 		switch databaseType {
@@ -91,7 +95,6 @@ var convertCmd = &cobra.Command{
 
 func init() {
 	convertCmd.Flags().String("type", "gentoo", "source type")
-	convertCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
 	convertCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
 
 	RootCmd.AddCommand(convertCmd)

cmd/create-repo.go

@@ -17,8 +17,9 @@ package cmd
 import (
 	"os"
 
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
-
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -35,23 +36,71 @@ var createrepoCmd = &cobra.Command{
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("output", cmd.Flags().Lookup("output"))
 		viper.BindPFlag("name", cmd.Flags().Lookup("name"))
-		viper.BindPFlag("uri", cmd.Flags().Lookup("uri"))
+		viper.BindPFlag("descr", cmd.Flags().Lookup("descr"))
+		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
 		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
+		viper.BindPFlag("tree-compression", cmd.Flags().Lookup("tree-compression"))
+		viper.BindPFlag("tree-path", cmd.Flags().Lookup("tree-path"))
+		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
+		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
+		var err error
+		var repo installer.Repository
 
 		tree := viper.GetString("tree")
 		dst := viper.GetString("output")
 		packages := viper.GetString("packages")
 		name := viper.GetString("name")
-		uri := viper.GetString("uri")
+		descr := viper.GetString("descr")
+		urls := viper.GetStringSlice("urls")
 		t := viper.GetString("type")
+		reset := viper.GetBool("reset-revision")
+		treetype := viper.GetString("tree-compression")
+		treepath := viper.GetString("tree-path")
+		source_repo := viper.GetString("repo")
+
+		if source_repo != "" {
+			// Search for system repository
+			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
+			if tree == "" {
+				tree = lrepo.TreePath
+			}
+
+			if t == "" {
+				t = lrepo.Type
+			}
+
+			repo, err = installer.GenerateRepository(lrepo.Name,
+				lrepo.Description, t,
+				lrepo.Urls,
+				lrepo.Priority,
+				packages,
+				tree,
+				pkg.NewInMemoryDatabase(false))
+
+		} else {
+			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
+				tree, pkg.NewInMemoryDatabase(false))
+		}
 
-		repo, err := installer.GenerateRepository(name, uri, t, 1, packages, tree, pkg.NewInMemoryDatabase(false))
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
-		err = repo.Write(dst)
+
+		if treetype != "" {
+			repo.SetTreeCompressionType(compiler.CompressionImplementation(treetype))
+		}
+
+		if treepath != "" {
+			repo.SetTreePath(treepath)
+		}
+
+		err = repo.Write(dst, reset)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
@@ -67,8 +116,14 @@ func init() {
 	createrepoCmd.Flags().String("tree", path, "Source luet tree")
 	createrepoCmd.Flags().String("output", path, "Destination folder")
 	createrepoCmd.Flags().String("name", "luet", "Repository name")
-	createrepoCmd.Flags().String("uri", path, "Repository uri")
-	createrepoCmd.Flags().String("type", "local", "Repository type (local)")
+	createrepoCmd.Flags().String("descr", "luet", "Repository description")
+	createrepoCmd.Flags().StringSlice("urls", []string{}, "Repository URLs")
+	createrepoCmd.Flags().String("type", "disk", "Repository type (disk)")
+	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
+	createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
+
+	createrepoCmd.Flags().String("tree-compression", "none", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("tree-path", installer.TREE_TARBALL, "Repository tree filename")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

cmd/install.go

@@ -17,68 +17,87 @@ package cmd
 import (
 	"os"
 	"path/filepath"
-	"regexp"
-	"runtime"
 
 	installer "github.com/mudler/luet/pkg/installer"
 
+	. "github.com/mudler/luet/pkg/config"
+	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 var installCmd = &cobra.Command{
 	Use:   "install <pkg1> <pkg2> ...",
 	Short: "Install a package",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("system-dbpath", cmd.Flags().Lookup("system-dbpath"))
-		viper.BindPFlag("system-target", cmd.Flags().Lookup("system-target"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+
 	},
 	Long: `Install packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
-		c := []*installer.LuetRepository{}
-		err := viper.UnmarshalKey("system-repositories", &c)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-
 		var toInstall []pkg.Package
+		var systemDB pkg.PackageDatabase
 
 		for _, a := range args {
-			decodepackage, err := regexp.Compile(`^([<>]?\~?=?)((([^\/]+)\/)?(?U)(\S+))(-(\d+(\.\d+)*[a-z]?(_(alpha|beta|pre|rc|p)\d*)*(-r\d+)?))?$`)
+			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
-				Fatal("Error: " + err.Error())
+				Fatal("Invalid package string ", a, ": ", err.Error())
 			}
-			packageInfo := decodepackage.FindAllStringSubmatch(a, -1)
-
-			category := packageInfo[0][4]
-			name := packageInfo[0][5]
-			version := packageInfo[0][1] + packageInfo[0][7]
-			toInstall = append(toInstall, &pkg.DefaultPackage{Name: name, Category: category, Version: version})
-
+			toInstall = append(toInstall, pack)
 		}
 
 		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
-		synced := installer.Repositories{}
-		for _, toSync := range c {
-			s, err := toSync.Sync()
-			if err != nil {
-				Fatal("Error: " + err.Error())
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
 			}
-			synced = append(synced, s)
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
 		}
 
-		inst := installer.NewLuetInstaller(viper.GetInt("concurrency"))
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps := LuetCfg.Viper.GetBool("nodeps")
+		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
 
-		inst.Repositories(synced)
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
 
-		os.MkdirAll(viper.GetString("system-dbpath"), os.ModePerm)
-		systemDB := pkg.NewBoltDatabase(filepath.Join(viper.GetString("system-dbpath"), "luet.db"))
-		system := &installer.System{Database: systemDB, Target: viper.GetString("system-target")}
-		err = inst.Install(toInstall, system)
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:   LuetCfg.GetGeneral().Concurrency,
+			SolverOptions: *LuetCfg.GetSolverOptions(),
+			NoDeps:        nodeps,
+			Force:         force,
+			OnlyDeps:      onlydeps,
+		})
+		inst.Repositories(repos)
+
+		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+			systemDB = pkg.NewBoltDatabase(
+				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+		} else {
+			systemDB = pkg.NewInMemoryDatabase(true)
+		}
+		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Install(toInstall, system)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
@@ -92,7 +111,13 @@ func init() {
 	}
 	installCmd.Flags().String("system-dbpath", path, "System db path")
 	installCmd.Flags().String("system-target", path, "System rootpath")
-	installCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
+	installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	installCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	installCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/repo.go

@@ -0,0 +1,37 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/repo"
+
+	"github.com/spf13/cobra"
+)
+
+var repoGroupCmd = &cobra.Command{
+	Use:   "repo [command] [OPTIONS]",
+	Short: "Manage repositories",
+}
+
+func init() {
+	RootCmd.AddCommand(repoGroupCmd)
+
+	repoGroupCmd.AddCommand(
+		NewRepoListCommand(),
+		NewRepoUpdateCommand(),
+	)
+}

cmd/repo/list.go

@@ -0,0 +1,101 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_repo
+
+import (
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	. "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+
+	. "github.com/logrusorgru/aurora"
+	"github.com/spf13/cobra"
+)
+
+func NewRepoListCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "list [OPTIONS]",
+		Short: "List of the configured repositories.",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var repoColor, repoText, repoRevision string
+
+			enable, _ := cmd.Flags().GetBool("enabled")
+			quiet, _ := cmd.Flags().GetBool("quiet")
+			repoType, _ := cmd.Flags().GetString("type")
+
+			for _, repo := range LuetCfg.SystemRepositories {
+				if enable && !repo.Enable {
+					continue
+				}
+
+				if repoType != "" && repo.Type != repoType {
+					continue
+				}
+
+				repoRevision = ""
+
+				if quiet {
+					fmt.Println(repo.Name)
+				} else {
+					if repo.Enable {
+						repoColor = Bold(BrightGreen(repo.Name)).String()
+					} else {
+						repoColor = Bold(BrightRed(repo.Name)).String()
+					}
+					if repo.Description != "" {
+						repoText = Yellow(repo.Description).String()
+					} else {
+						repoText = Yellow(repo.Urls[0]).String()
+					}
+
+					repobasedir := LuetCfg.GetSystem().GetRepoDatabaseDirPath(repo.Name)
+					if repo.Cached {
+
+						r := installer.NewSystemRepository(repo)
+						localRepo, _ := r.(*installer.LuetSystemRepository).ReadSpecFile(filepath.Join(repobasedir,
+							installer.REPOSITORY_SPECFILE), false)
+						if localRepo != nil {
+							tsec, _ := strconv.ParseInt(localRepo.GetLastUpdate(), 10, 64)
+							repoRevision = Bold(Red(localRepo.GetRevision())).String() +
+								" - " + Bold(Green(time.Unix(tsec, 0).String())).String()
+						}
+					}
+
+					if repoRevision != "" {
+						fmt.Println(
+							fmt.Sprintf("%s\n  %s\n  Revision %s", repoColor, repoText, repoRevision))
+					} else {
+						fmt.Println(
+							fmt.Sprintf("%s\n  %s", repoColor, repoText))
+					}
+				}
+			}
+		},
+	}
+
+	ans.Flags().Bool("enabled", false, "Show only enable repositories.")
+	ans.Flags().BoolP("quiet", "q", false, "Show only name of the repositories.")
+	ans.Flags().StringP("type", "t", "", "Filter repositories of a specific type")
+
+	return ans
+}

cmd/repo/update.go

@@ -0,0 +1,83 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_repo
+
+import (
+	. "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+func NewRepoUpdateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "update [repo1] [repo2] [OPTIONS]",
+		Short: "Update a specific cached repository or all cached repositories.",
+		Example: `
+# Update all cached repositories:
+$> luet repo update
+
+# Update only repo1 and repo2
+$> luet repo update repo1 repo2
+`,
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			ignore, _ := cmd.Flags().GetBool("ignore-errors")
+			force, _ := cmd.Flags().GetBool("force")
+
+			if len(args) > 0 {
+				for _, rname := range args {
+					repo, err := LuetCfg.GetSystemRepository(rname)
+					if err != nil && !ignore {
+						Fatal(err.Error())
+					} else if err != nil {
+						continue
+					}
+
+					r := installer.NewSystemRepository(*repo)
+					Spinner(32)
+					_, err = r.Sync(force)
+					if err != nil && !ignore {
+						Fatal("Error on sync repository " + rname + ": " + err.Error())
+					}
+					SpinnerStop()
+				}
+
+			} else {
+				for _, repo := range LuetCfg.SystemRepositories {
+					if repo.Cached {
+						r := installer.NewSystemRepository(repo)
+						Spinner(32)
+						_, err := r.Sync(force)
+						if err != nil && !ignore {
+							Fatal("Error on sync repository " + r.GetName() + ": " + err.Error())
+						}
+						SpinnerStop()
+					}
+				}
+			}
+		},
+	}
+
+	ans.Flags().BoolP("ignore-errors", "i", false, "Ignore errors on sync repositories.")
+	ans.Flags().BoolP("force", "f", false, "Force resync.")
+
+	return ans
+}

cmd/root.go

@@ -16,12 +16,18 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
-	"path"
+	"os/user"
 	"path/filepath"
+	"runtime"
+	"strings"
 
 	"github.com/marcsauter/single"
+	config "github.com/mudler/luet/pkg/config"
+	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
+	repo "github.com/mudler/luet/pkg/repository"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -30,7 +36,10 @@ import (
 var cfgFile string
 var Verbose bool
 
-const LuetCLIVersion = "0.3"
+const (
+	LuetCLIVersion = "0.6.2"
+	LuetEnvPrefix  = "LUET"
+)
 
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
@@ -38,6 +47,45 @@ var RootCmd = &cobra.Command{
 	Short:   "Package manager for the XXth century!",
 	Long:    `Package manager which uses containers to build packages`,
 	Version: LuetCLIVersion,
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		err := LoadConfig(config.LuetCfg)
+		if err != nil {
+			Fatal("failed to load configuration:", err.Error())
+		}
+	},
+	SilenceErrors: true,
+}
+
+func LoadConfig(c *config.LuetConfig) error {
+	// If a config file is found, read it in.
+	if err := c.Viper.ReadInConfig(); err != nil {
+		Debug(err)
+	}
+
+	err := c.Viper.Unmarshal(&config.LuetCfg)
+	if err != nil {
+		return err
+	}
+
+	Debug("Using config file:", c.Viper.ConfigFileUsed())
+
+	NewSpinner()
+
+	if c.GetLogging().Path != "" {
+		// Init zap logger
+		err = ZapLogger()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Load repositories
+	err = repo.LoadRepositories(c)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // Execute adds all child commands to the root command sets flags appropriately.
@@ -54,16 +102,44 @@ func Execute() {
 		}
 		defer s.TryUnlock()
 	}
+
 	if err := RootCmd.Execute(); err != nil {
-		Error(err)
+		if len(os.Args) > 0 {
+			for _, c := range RootCmd.Commands() {
+				if c.Name() == os.Args[1] {
+					os.Exit(-1) // Something failed
+				}
+			}
+			// Try to load a bin from path.
+			helpers.Exec("luet-"+os.Args[1], os.Args[1:], os.Environ())
+		}
+		fmt.Println(err)
 		os.Exit(-1)
 	}
 }
 
 func init() {
 	cobra.OnInitialize(initConfig)
-	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
-	RootCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
+	pflags := RootCmd.PersistentFlags()
+	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
+	pflags.BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
+	pflags.Bool("fatal", false, "Enables Warnings to exit")
+
+	u, err := user.Current()
+	if err != nil {
+		Fatal("failed to retrieve user identity:", err.Error())
+	}
+	sameOwner := false
+	if u.Uid == "0" {
+		sameOwner = true
+	}
+	pflags.Bool("same-owner", sameOwner, "Maintain same owner on uncompress.")
+	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
+
+	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
+	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("verbose"))
+	config.LuetCfg.Viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
+	config.LuetCfg.Viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
 }
 
 // initConfig reads in config file and ENV variables if set.
@@ -74,29 +150,23 @@ func initConfig() {
 		Error(err)
 		os.Exit(1)
 	}
+	viper.SetEnvPrefix(LuetEnvPrefix)
 	viper.SetConfigType("yaml")
 	viper.SetConfigName(".luet") // name of config file (without extension)
 	if cfgFile != "" {           // enable ability to specify config file via flag
 		Info(">>> cfgFile: ", cfgFile)
 		viper.SetConfigFile(cfgFile)
-		configDir := path.Dir(cfgFile)
-		if configDir != "." && configDir != dir {
-			viper.AddConfigPath(configDir)
-		}
+	} else {
+		viper.AddConfigPath(dir)
+		viper.AddConfigPath(".")
+		viper.AddConfigPath("$HOME")
+		viper.AddConfigPath("/etc/luet")
 	}
 
-	viper.AddConfigPath(dir)
-	viper.AddConfigPath(".")
-	viper.AddConfigPath("$HOME")
-	viper.AddConfigPath("/etc/luet")
-
 	viper.AutomaticEnv() // read in environment variables that match
 
-	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err == nil {
-		Info("Using config file:", viper.ConfigFileUsed())
-	} else {
-		Warning(err)
-	}
-
+	// Create EnvKey Replacer for handle complex structure
+	replacer := strings.NewReplacer(".", "__")
+	viper.SetEnvKeyReplacer(replacer)
+	viper.SetTypeByDefaultValue(true)
 }

cmd/search.go

@@ -18,15 +18,13 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-	"runtime"
 
+	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
-
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 var searchCmd = &cobra.Command{
@@ -34,43 +32,68 @@ var searchCmd = &cobra.Command{
 	Short: "Search packages",
 	Long:  `Search for installed and available packages`,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("system-dbpath", cmd.Flags().Lookup("system-dbpath"))
-		viper.BindPFlag("system-target", cmd.Flags().Lookup("system-target"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
-		viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		c := []*installer.LuetRepository{}
-		err := viper.UnmarshalKey("system-repositories", &c)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
+		var systemDB pkg.PackageDatabase
 
 		if len(args) != 1 {
 			Fatal("Wrong number of arguments (expected 1)")
 		}
-		installed := viper.GetBool("installed")
+		installed := LuetCfg.Viper.GetBool("installed")
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
 		if !installed {
-			synced := installer.Repositories{}
 
-			for _, toSync := range c {
-				s, err := toSync.Sync()
-				if err != nil {
-					Fatal("Error: " + err.Error())
+			repos := installer.Repositories{}
+			for _, repo := range LuetCfg.SystemRepositories {
+				if !repo.Enable {
+					continue
 				}
-				synced = append(synced, s)
+				r := installer.NewSystemRepository(repo)
+				repos = append(repos, r)
 			}
+
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
+
+			inst.Repositories(repos)
+			synced, err := inst.SyncRepositories(false)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
 			Info("--- Search results: ---")
 
 			matches := synced.Search(args[0])
 			for _, m := range matches {
-				Info(":package:", m.Package.GetCategory(), m.Package.GetName(), m.Package.GetVersion(), "repository:", m.Repo.GetName())
+				Info(":package:", m.Package.GetCategory(), m.Package.GetName(),
+					m.Package.GetVersion(), "repository:", m.Repo.GetName())
 			}
 		} else {
-			os.MkdirAll(viper.GetString("system-dbpath"), os.ModePerm)
-			systemDB := pkg.NewBoltDatabase(filepath.Join(viper.GetString("system-dbpath"), "luet.db"))
-			system := &installer.System{Database: systemDB, Target: viper.GetString("system-target")}
+
+			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+				systemDB = pkg.NewBoltDatabase(
+					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+			} else {
+				systemDB = pkg.NewInMemoryDatabase(true)
+			}
+			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
 			var term = regexp.MustCompile(args[0])
 
 			for _, k := range system.Database.GetPackages() {
@@ -91,7 +114,10 @@ func init() {
 	}
 	searchCmd.Flags().String("system-dbpath", path, "System db path")
 	searchCmd.Flags().String("system-target", path, "System rootpath")
-	searchCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
 	searchCmd.Flags().Bool("installed", false, "Search between system packages")
+	searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	searchCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	searchCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	searchCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	RootCmd.AddCommand(searchCmd)
 }

cmd/serve-repo.go

@@ -0,0 +1,59 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"net/http"
+	"os"
+
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var serverepoCmd = &cobra.Command{
+	Use:   "serve-repo",
+	Short: "Embedded micro-http server",
+	Long:  `Embedded mini http server for serving local repositories`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("dir", cmd.Flags().Lookup("dir"))
+		viper.BindPFlag("address", cmd.Flags().Lookup("address"))
+		viper.BindPFlag("port", cmd.Flags().Lookup("port"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+
+		dir := viper.GetString("dir")
+		port := viper.GetString("port")
+		address := viper.GetString("address")
+
+		http.Handle("/", http.FileServer(http.Dir(dir)))
+
+		Info("Serving ", dir, " on HTTP port: ", port)
+		Fatal(http.ListenAndServe(address+":"+port, nil))
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	serverepoCmd.Flags().String("dir", path, "Packages folder (output from build)")
+	serverepoCmd.Flags().String("port", "9090", "Listening port")
+	serverepoCmd.Flags().String("address", "0.0.0.0", "Listening address")
+
+	RootCmd.AddCommand(serverepoCmd)
+}

cmd/uninstall.go

@@ -17,50 +17,72 @@ package cmd
 import (
 	"os"
 	"path/filepath"
-	"regexp"
-	"runtime"
 
+	. "github.com/mudler/luet/pkg/config"
+	helpers "github.com/mudler/luet/pkg/helpers"
 	installer "github.com/mudler/luet/pkg/installer"
-
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 var uninstallCmd = &cobra.Command{
-	Use:   "uninstall <pkg>",
-	Short: "Uninstall a package",
+	Use:   "uninstall <pkg> <pkg2> ...",
+	Short: "Uninstall a package or a list of packages",
 	Long:  `Uninstall packages`,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("system-dbpath", cmd.Flags().Lookup("system-dbpath"))
-		viper.BindPFlag("system-target", cmd.Flags().Lookup("system-target"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		if len(args) != 1 {
-			Fatal("Wrong number of args")
-		}
+		var systemDB pkg.PackageDatabase
 
-		a := args[0]
-		decodepackage, err := regexp.Compile(`^([<>]?\~?=?)((([^\/]+)\/)?(?U)(\S+))(-(\d+(\.\d+)*[a-z]?(_(alpha|beta|pre|rc|p)\d*)*(-r\d+)?))?$`)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-		packageInfo := decodepackage.FindAllStringSubmatch(a, -1)
+		for _, a := range args {
 
-		category := packageInfo[0][4]
-		name := packageInfo[0][5]
-		version := packageInfo[0][7]
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
 
-		inst := installer.NewLuetInstaller(viper.GetInt("concurrency"))
-		os.MkdirAll(viper.GetString("system-dbpath"), os.ModePerm)
-		systemDB := pkg.NewBoltDatabase(filepath.Join(viper.GetString("system-dbpath"), "luet.db"))
-		system := &installer.System{Database: systemDB, Target: viper.GetString("system-target")}
-		err = inst.Uninstall(&pkg.DefaultPackage{Name: name, Category: category, Version: version}, system)
-		if err != nil {
-			Fatal("Error: " + err.Error())
+			stype := LuetCfg.Viper.GetString("solver.type")
+			discount := LuetCfg.Viper.GetFloat64("solver.discount")
+			rate := LuetCfg.Viper.GetFloat64("solver.rate")
+			attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+			force := LuetCfg.Viper.GetBool("force")
+			nodeps := LuetCfg.Viper.GetBool("nodeps")
+
+			LuetCfg.GetSolverOptions().Type = stype
+			LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+			LuetCfg.GetSolverOptions().Discount = float32(discount)
+			LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+				Concurrency:   LuetCfg.GetGeneral().Concurrency,
+				SolverOptions: *LuetCfg.GetSolverOptions(),
+				NoDeps:        nodeps,
+				Force:         force,
+			})
+
+			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+				systemDB = pkg.NewBoltDatabase(
+					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+			} else {
+				systemDB = pkg.NewInMemoryDatabase(true)
+			}
+			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+			err = inst.Uninstall(pack, system)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
 		}
 	},
 }
@@ -72,6 +94,12 @@ func init() {
 	}
 	uninstallCmd.Flags().String("system-dbpath", path, "System db path")
 	uninstallCmd.Flags().String("system-target", path, "System rootpath")
-	uninstallCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
+	uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	uninstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	uninstallCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	uninstallCmd.Flags().Bool("force", false, "Force uninstall")
+
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -17,50 +17,72 @@ package cmd
 import (
 	"os"
 	"path/filepath"
-	"runtime"
 
+	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
-
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 var upgradeCmd = &cobra.Command{
 	Use:   "upgrade",
 	Short: "Upgrades the system",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("system-dbpath", cmd.Flags().Lookup("system-dbpath"))
-		viper.BindPFlag("system-target", cmd.Flags().Lookup("system-target"))
-		viper.BindPFlag("concurrency", cmd.Flags().Lookup("concurrency"))
+		LuetCfg.Viper.BindPFlag("system.database_path", installCmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", installCmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Long: `Upgrades packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
-		c := []*installer.LuetRepository{}
-		err := viper.UnmarshalKey("system-repositories", &c)
+		var systemDB pkg.PackageDatabase
+
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		Debug("Solver", LuetCfg.GetSolverOptions().String())
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:   LuetCfg.GetGeneral().Concurrency,
+			SolverOptions: *LuetCfg.GetSolverOptions(),
+			Force:         force,
+		})
+		inst.Repositories(repos)
+		_, err := inst.SyncRepositories(false)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
 
-		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
-		synced := installer.Repositories{}
-		for _, toSync := range c {
-			s, err := toSync.Sync()
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-			synced = append(synced, s)
+		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+			systemDB = pkg.NewBoltDatabase(
+				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+		} else {
+			systemDB = pkg.NewInMemoryDatabase(true)
 		}
-
-		inst := installer.NewLuetInstaller(viper.GetInt("concurrency"))
-
-		inst.Repositories(synced)
-
-		os.MkdirAll(viper.GetString("system-dbpath"), os.ModePerm)
-		systemDB := pkg.NewBoltDatabase(filepath.Join(viper.GetString("system-dbpath"), "luet.db"))
-		system := &installer.System{Database: systemDB, Target: viper.GetString("system-target")}
+		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
 		err = inst.Upgrade(system)
 		if err != nil {
 			Fatal("Error: " + err.Error())
@@ -75,7 +97,11 @@ func init() {
 	}
 	upgradeCmd.Flags().String("system-dbpath", path, "System db path")
 	upgradeCmd.Flags().String("system-target", path, "System rootpath")
-	upgradeCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
+	upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

contrib/config/luet.yaml

@@ -0,0 +1,128 @@
+# Luet Configuration File
+#
+# ---------------------------------------------
+# Logging configuration section:
+# ---------------------------------------------
+# logging:
+#   Leave empty to skip logging to file.
+#   path: ""
+#
+#   Set logging level: error|warning|info|debug
+#   level: "info"
+#
+#   Enable JSON log format instead of console mode.
+#   json_format: false.
+#
+# ---------------------------------------------
+# General configuration section:
+# ---------------------------------------------
+# general:
+#   Define max concurrency processes. Default is based of arch: runtime.NumCPU()
+#   concurrency: 1
+#
+#   Enable Debug. If debug is active spinner is disabled.
+#   debug: false
+#
+#   Show output of build execution (docker, img, etc.)
+#   show_build_output: false
+#
+#   Define spinner ms
+#   spinner_ms: 200
+#
+#   Define spinner charset. See https://github.com/briandowns/spinner
+#   spinner_charset: 22
+#
+#   Enable warnings to exit
+#   fatal_warnings: false
+#
+#   Try extracting tree/packages with the same ownership as exists in the archive (default for superuser).
+#   same_owner: false
+#
+# ---------------------------------------------
+# System configuration section:
+# ---------------------------------------------
+# system:
+#
+#   Rootfs path of the luet system. Default is /.
+#   A specific path could be used for test installation to
+#   a chroot environment.
+#   rootfs: "/"
+#
+#   Choice database engine used for luet database.
+#   Supported values: boltdb|memory
+#   database_engine: boltdb
+#
+#   Database path directory where store luet database.
+#   The path is append to rootfs option path.
+#   database_path: "/var/cache/luet"
+#
+# ---------------------------------------------
+# Repositories configurations directories.
+# ---------------------------------------------
+# Define the list of directories where luet
+# try for files with .yml extension that define
+# luet repository.
+# repos_confdir:
+#   - /etc/luet/repos.conf.d
+#
+#
+# ---------------------------------------------
+# System repositories
+# ---------------------------------------------
+# In alternative to define repositories files
+# through repos_confdir option is possible
+# define directly the list of the repositories.
+#
+# repositories:
+#
+#     Name of the repository. It's better that this name is unique. Mandatory.
+#   - name: "repo1"
+#
+#     A user-friendly description of the repository
+#     description: "My luet repo"
+#
+#     Type of the repository. Supported types are: dir|http. Mandatory.
+#     type: "dir"
+#
+#     Define the priority of the repository on research packages. Default is 9999.
+#     priority: 9999
+#
+#     Enable/Disable of the repository.
+#     enable: false
+#
+#     Cached repository. If true a local cache of the remote repository tree is maintained
+#     locally in the $tree_path else it is used a temporary directory that is removed when
+#     installation of a package is completed. A cached repository reduce time on search/install
+#     packages. By default caching is disable.
+#     cached: false
+#
+#     Path where store tree of the specifications. Default path is $database_path/repos/$repo_name
+#     tree_path: "/var/cache/luet/repos/local"
+#
+#     Define the list of the URL where retrieve tree and packages.
+#     urls:
+#        - https://mydomain.local/luet/repo1
+#
+#     auth:
+#        Define Basic authentication header
+#        basic: "mybasicauth"
+#        Define token authentication header
+#        token: "mytoken"
+# ---------------------------------------------
+# Solver parameter configuration:
+# ---------------------------------------------
+# solver:
+#
+#   Solver strategy to solve possible conflicts during depedency
+#   solving. Defaults to empty (none). Available: qlearning
+#   type: ""
+#
+#   Solver agent learning rate. 0.1 to 1.0
+#   rate: 0.7
+#
+#   Learning discount factor.
+#   discount: 1.0
+#
+#   Number of overall attempts that the solver has available before bailing out.
+#   max_attempts: 9000
+#

go.mod

@@ -5,16 +5,19 @@ go 1.12
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
 	github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56 // indirect
-	github.com/Sabayon/pkgs-checker v0.4.1
+	github.com/Sabayon/pkgs-checker v0.5.1-0.20200221202320-073693f2c657
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/briandowns/spinner v1.7.0
 	github.com/cavaliercoder/grab v2.0.0+incompatible
-	github.com/crillab/gophersat v1.1.7
+	github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3
 	github.com/docker/docker v0.7.3-0.20180827131323-0c5f8d2b9b23
+	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-yaml/yaml v2.1.0+incompatible // indirect
 	github.com/hashicorp/go-version v1.2.0
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
+	github.com/klauspost/pgzip v1.2.1
+	github.com/kr/pretty v0.2.0 // indirect
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
@@ -32,9 +35,15 @@ require (
 	github.com/spf13/viper v1.5.0
 	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
 	go.etcd.io/bbolt v1.3.3
-	golang.org/x/crypto v0.0.0-20191028145041-f83a4685e152 // indirect
-	golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4 // indirect
-	gopkg.in/yaml.v2 v2.2.5
-	mvdan.cc/sh v2.6.4+incompatible // indirect
+	go.uber.org/atomic v1.5.1 // indirect
+	go.uber.org/multierr v1.4.0 // indirect
+	go.uber.org/zap v1.13.0
+	golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect
+	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
+	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
+	golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c // indirect
+	golang.org/x/tools v0.0.0-20200102200121-6de373a2766c // indirect
+	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
+	gopkg.in/yaml.v2 v2.2.7
 	mvdan.cc/sh/v3 v3.0.0-beta1
 )

go.sum

@@ -14,8 +14,11 @@ github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56/
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/Sabayon/pkgs-checker v0.4.1 h1:NImZhA5Z9souyr9Ff3nDzP0Bs9SGuDLxRzduVqci3dQ=
-github.com/Sabayon/pkgs-checker v0.4.1/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
+github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7 h1:Vf80sSLu1ZWjjMmUKhw0FqM43lEOvT8O5B22NaHB6AQ=
+github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
+github.com/Sabayon/pkgs-checker v0.5.0 h1:VRyyAxo6ox41Dytyl+K+QC+Tps0IxvqYbidu+AH+HUQ=
+github.com/Sabayon/pkgs-checker v0.5.1-0.20200221202320-073693f2c657 h1:VK5S2Gh9kPUxX81zCFUgKVQn+hFy6VgyZMD3QLm76u8=
+github.com/Sabayon/pkgs-checker v0.5.1-0.20200221202320-073693f2c657/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
 github.com/Sereal/Sereal v0.0.0-20181211220259-509a78ddbda3 h1:Xu7z47ZiE/J+sKXHZMGxEor/oY2q6dq51fkO0JqdSwY=
 github.com/Sereal/Sereal v0.0.0-20181211220259-509a78ddbda3/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -57,6 +60,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/crillab/gophersat v1.1.7 h1:f2Phe0W9jGyN1OefygKdcTdNM99q/goSjbWrFRjZGWc=
 github.com/crillab/gophersat v1.1.7/go.mod h1:S91tHga1PCZzYhCkStwZAhvp1rCc+zqtSi55I+vDWGc=
+github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3 h1:HO63LCf9kTXQgUnlvFeS2qSDQhZ/cLP8DAJO89CythY=
+github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3/go.mod h1:S91tHga1PCZzYhCkStwZAhvp1rCc+zqtSi55I+vDWGc=
 github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -77,6 +82,8 @@ github.com/docker/libnetwork v0.8.0-dev.2.0.20180608203834-19279f049241 h1:+ebE/
 github.com/docker/libnetwork v0.8.0-dev.2.0.20180608203834-19279f049241/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd h1:JWEotl+g5uCCn37eVAYLF3UjBqO5HJ0ezZ5Zgnsdoqc=
+github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd/go.mod h1:y0+kb0ORo7mC8lQbUzC4oa7ufu565J6SyUgWd39Z1Ic=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@@ -107,6 +114,7 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
@@ -146,6 +154,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
@@ -197,7 +207,9 @@ github.com/opencontainers/runtime-spec v1.0.1 h1:wY4pOY8fBdSIvs9+IDHC55thBuEulhz
 github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc=
 github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
@@ -222,6 +234,7 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.5.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rootless-containers/proto v0.1.0 h1:gS1JOMEtk1YDYHCzBAf/url+olMJbac7MTrgSeP6zh4=
 github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZPu0UNHLXIQxhyqAFh8=
@@ -257,8 +270,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
-github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.5.0 h1:GpsTwfsQ27oS/Aha/6d1oD7tpKIqWnOA6tgOX9HHkt4=
 github.com/spf13/viper v1.5.0/go.mod h1:AkYRkVJF8TkSG/xet6PzXX+l39KhhXa2pdqVSxnTcn4=
 github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b h1:wJSBFlabo96ySlmSX0a02WAPyGxagzTo9c5sk3sHP3E=
@@ -296,19 +307,36 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
+go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E=
+go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU=
+go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191028145041-f83a4685e152 h1:ZC1Xn5A1nlpSmQCIva4bZ3ob3lmhYIefc+GU+DLg1Ow=
-golang.org/x/crypto v0.0.0-20191028145041-f83a4685e152/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc=
+golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -319,6 +347,8 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 h1:4dVFTC832rPn4pomLSz1vA+are2+dU19w1H8OngV7nc=
 golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -341,8 +371,8 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4 h1:Hynbrlo6LbYI3H1IqXpkVDOcX/3HiPdhVEuyj5a59RM=
-golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c h1:OYFUffxXPezb7BVTx9AaD4Vl0qtxmklBIkwCKH1YwDY=
+golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -351,10 +381,18 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190913181337-0240832f5c3d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200102200121-6de373a2766c h1:PBxLbymhzlh6kZuAXmeh8JK2tAJR0GM5Q/W71G2QJ40=
+golang.org/x/tools v0.0.0-20200102200121-6de373a2766c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -378,13 +416,13 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.1.0+incompatible h1:5USw7CrJBYKqjg9R7QlA6jzqZKEAtvW82aNmsxxGPxw=
 gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 mvdan.cc/editorconfig v0.1.1-0.20191109213504-890940e3f00e/go.mod h1:Ge4atmRUYqueGppvJ7JNrtqpqokoJEFxYbP0Z+WeKS8=
-mvdan.cc/sh v2.6.4+incompatible h1:eD6tDeh0pw+/TOTI1BBEryZ02rD2nMcFsgcvde7jffM=
-mvdan.cc/sh v2.6.4+incompatible/go.mod h1:IeeQbZq+x2SUGBensq/jge5lLQbS3XT2ktyp3wrt4x8=
 mvdan.cc/sh/v3 v3.0.0-beta1 h1:UqiwBEXEPzelaGxuvixaOtzc7WzKtrElePJ8HqvW7K8=
 mvdan.cc/sh/v3 v3.0.0-beta1/go.mod h1:rBIndNJFYPp8xSppiZcGIk6B5d1g3OEARxEaXjPxwVI=

pkg/compiler/artifact.go

@@ -16,22 +16,34 @@
 package compiler
 
 import (
+	"archive/tar"
+	"bufio"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 	"regexp"
 
+	gzip "github.com/klauspost/pgzip"
+
 	//"strconv"
 	"strings"
 	"sync"
 
+	. "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	"github.com/mudler/luet/pkg/solver"
-	yaml "gopkg.in/yaml.v2"
-
-	"github.com/mudler/luet/pkg/helpers"
 	"github.com/pkg/errors"
+	yaml "gopkg.in/yaml.v2"
+)
+
+type CompressionImplementation string
+
+const (
+	None CompressionImplementation = "none" // e.g. tar for standard packages
+	GZip CompressionImplementation = "gzip"
 )
 
 type ArtifactIndex []Artifact
@@ -40,7 +52,15 @@ func (i ArtifactIndex) CleanPath() ArtifactIndex {
 	newIndex := ArtifactIndex{}
 	for _, n := range i {
 		art := n.(*PackageArtifact)
-		newIndex = append(newIndex, &PackageArtifact{Path: path.Base(n.GetPath()), SourceAssertion: art.SourceAssertion, CompileSpec: art.CompileSpec, Dependencies: art.Dependencies})
+		// FIXME: This is a dup and makes difficult to add attributes to artifacts
+		newIndex = append(newIndex, &PackageArtifact{
+			Path:            path.Base(n.GetPath()),
+			SourceAssertion: art.SourceAssertion,
+			CompileSpec:     art.CompileSpec,
+			Dependencies:    art.Dependencies,
+			CompressionType: art.CompressionType,
+			Checksums:       art.Checksums,
+		})
 	}
 	return newIndex
 	//Update if exists, otherwise just create
@@ -50,19 +70,21 @@ func (i ArtifactIndex) CleanPath() ArtifactIndex {
 // which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact.
 // In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages.
 type PackageArtifact struct {
-	Path         string               `json:"path"`
-	Dependencies []*PackageArtifact   `json:"dependencies"`
-	CompileSpec  *LuetCompilationSpec `json:"compilationspec"`
+	Path string `json:"path"`
 
+	Dependencies    []*PackageArtifact        `json:"dependencies"`
+	CompileSpec     *LuetCompilationSpec      `json:"compilationspec"`
+	Checksums       Checksums                 `json:"checksums"`
 	SourceAssertion solver.PackagesAssertions `json:"-"`
+	CompressionType CompressionImplementation `json:"compressiontype"`
 }
 
 func NewPackageArtifact(path string) Artifact {
-	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}}
+	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: None}
 }
 
 func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
-	p := &PackageArtifact{}
+	p := &PackageArtifact{Checksums: Checksums{}}
 	err := yaml.Unmarshal(data, &p)
 	if err != nil {
 		return p, err
@@ -71,7 +93,56 @@ func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
 	return p, err
 }
 
+func LoadArtifactFromYaml(spec CompilationSpec) (Artifact, error) {
+
+	metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml"
+	dat, err := ioutil.ReadFile(spec.Rel(metaFile))
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading file "+metaFile)
+	}
+	art, err := NewPackageArtifactFromYaml(dat)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error writing file "+metaFile)
+	}
+	// It is relative, set it back to abs
+	art.SetPath(spec.Rel(art.GetPath()))
+	return art, nil
+}
+
+func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) {
+	a.CompressionType = t
+}
+
+func (a *PackageArtifact) GetChecksums() Checksums {
+	return a.Checksums
+}
+func (a *PackageArtifact) SetChecksums(c Checksums) {
+	a.Checksums = c
+}
+func (a *PackageArtifact) Hash() error {
+	return a.Checksums.Generate(a)
+}
+
+func (a *PackageArtifact) Verify() error {
+	sum := Checksums{}
+	err := sum.Generate(a)
+	if err != nil {
+		return err
+	}
+	err = sum.Compare(a.Checksums)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (a *PackageArtifact) WriteYaml(dst string) error {
+	// First compute checksum of artifact. When we write the yaml we want to write up-to-date informations.
+	err := a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for artifact")
+	}
+
 	//p := a.CompileSpec.GetPackage().GetPath()
 
 	//a.CompileSpec.GetPackage().SetPath("")
@@ -148,6 +219,155 @@ func (a *PackageArtifact) SetPath(p string) {
 	a.Path = p
 }
 
+// Compress Archives and compress (TODO) to the artifact path
+func (a *PackageArtifact) Compress(src string, concurrency int) error {
+	switch a.CompressionType {
+	case GZip:
+		err := helpers.Tar(src, a.Path)
+		if err != nil {
+			return err
+		}
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return err
+		}
+		defer original.Close()
+
+		gzipfile := a.Path + ".gz"
+		bufferedReader := bufio.NewReader(original)
+
+		// Open a file for writing.
+		dst, err := os.Create(gzipfile)
+		if err != nil {
+			return err
+		}
+		// Create gzip writer.
+		w := gzip.NewWriter(dst)
+		w.SetConcurrency(concurrency, 10)
+		defer w.Close()
+		defer dst.Close()
+		_, err = io.Copy(w, bufferedReader)
+		if err != nil {
+			return err
+		}
+		w.Close()
+		os.RemoveAll(a.Path) // Remove original
+		//	a.CompressedPath = gzipfile
+		a.Path = gzipfile
+		return nil
+		//a.Path = gzipfile
+
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		return helpers.Tar(src, a.Path)
+
+	}
+	return errors.New("Compression type must be supplied")
+}
+
+// Unpack Untar and decompress (TODO) to the given path
+func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+	switch a.CompressionType {
+	case GZip:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := gzip.NewReader(bufferedReader)
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+
+		_, err = io.Copy(archive, r)
+		if err != nil {
+			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
+		}
+
+		err = helpers.Untar(a.GetPath()+".uncompressed", dst,
+			LuetCfg.GetGeneral().SameOwner)
+		if err != nil {
+			return err
+		}
+		return nil
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		return helpers.Untar(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner)
+	}
+	return errors.New("Compression type must be supplied")
+}
+
+func (a *PackageArtifact) FileList() ([]string, error) {
+	var tr *tar.Reader
+	switch a.CompressionType {
+	case GZip:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		if err != nil {
+			return []string{}, err
+		}
+		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := gzip.NewReader(bufferedReader)
+		if err != nil {
+			return []string{}, err
+		}
+		defer r.Close()
+		tr = tar.NewReader(r)
+
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		tarFile, err := os.Open(a.GetPath())
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Could not open package archive")
+		}
+		defer tarFile.Close()
+		tr = tar.NewReader(tarFile)
+
+	}
+
+	var files []string
+	// untar each segment
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return []string{}, err
+		}
+		// determine proper file path info
+		finfo := hdr.FileInfo()
+		fileName := hdr.Name
+		if finfo.Mode().IsDir() {
+			continue
+		}
+		files = append(files, fileName)
+
+		// if a dir, create it, then go to next segment
+	}
+	return files, nil
+}
+
 type CopyJob struct {
 	Src, Dst string
 	Artifact string
@@ -175,7 +395,7 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 }
 
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string) (Artifact, error) {
+func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, t CompressionImplementation) (Artifact, error) {
 
 	archive, err := ioutil.TempDir(os.TempDir(), "archive")
 	if err != nil {
@@ -239,10 +459,46 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 
 	close(toCopy)
 	wg.Wait()
-
-	err = helpers.Tar(archive, dst)
+	a := NewPackageArtifact(dst)
+	a.SetCompressionType(t)
+	err = a.Compress(archive, concurrency)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}
-	return NewPackageArtifact(dst), nil
+	return a, nil
+}
+
+func ComputeArtifactLayerSummary(diffs []ArtifactLayer) ArtifactLayersSummary {
+
+	ans := ArtifactLayersSummary{
+		Layers: make([]ArtifactLayerSummary, 0),
+	}
+
+	for _, layer := range diffs {
+		sum := ArtifactLayerSummary{
+			FromImage:   layer.FromImage,
+			ToImage:     layer.ToImage,
+			AddFiles:    0,
+			AddSizes:    0,
+			DelFiles:    0,
+			DelSizes:    0,
+			ChangeFiles: 0,
+			ChangeSizes: 0,
+		}
+		for _, a := range layer.Diffs.Additions {
+			sum.AddFiles++
+			sum.AddSizes += int64(a.Size)
+		}
+		for _, d := range layer.Diffs.Deletions {
+			sum.DelFiles++
+			sum.DelSizes += int64(d.Size)
+		}
+		for _, c := range layer.Diffs.Changes {
+			sum.ChangeFiles++
+			sum.ChangeSizes += int64(c.Size)
+		}
+		ans.Layers = append(ans.Layers, sum)
+	}
+
+	return ans
 }

pkg/compiler/artifact_test.go

@@ -32,7 +32,7 @@ import (
 
 var _ = Describe("Artifact", func() {
 	Context("Simple package build definition", func() {
-		It("Generates a delta", func() {
+		It("Generates a verified delta", func() {
 
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
@@ -41,7 +41,7 @@ var _ = Describe("Artifact", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -125,7 +125,7 @@ RUN echo bar > /test2`))
 			err = b.ExtractRootfs(CompilerBackendOptions{SourcePath: filepath.Join(tmpdir, "output2.tar"), Destination: rootfs}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{})
+			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, None)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
 			err = helpers.Untar(artifact.GetPath(), unpacked, false)
@@ -138,6 +138,15 @@ RUN echo bar > /test2`))
 			content2, err := helpers.Read(filepath.Join(unpacked, "test2"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content2).To(Equal("bar\n"))
+
+			err = artifact.Hash()
+			Expect(err).ToNot(HaveOccurred())
+			err = artifact.Verify()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
+
+			err = artifact.Verify()
+			Expect(err).To(HaveOccurred())
 		})
 
 	})

pkg/compiler/backend/backend_suite_test.go

@@ -18,11 +18,15 @@ package backend_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Backend Suite")
 }

pkg/compiler/backend/simpledocker.go

@@ -17,6 +17,7 @@ package backend
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -26,6 +27,7 @@ import (
 	capi "github.com/mudler/docker-companion/api"
 
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -54,7 +56,12 @@ func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
 	}
 	Info(":whale: Building image " + name + " done")
 
-	//Info(string(out))
+	if config.LuetCfg.GetGeneral().ShowBuildOutput {
+		Info(string(out))
+	} else {
+		Debug(string(out))
+	}
+
 	return nil
 }
 
@@ -94,6 +101,18 @@ func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
+func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
+	name := opts.ImageName
+	pusharg := []string{"push", name}
+	out, err := exec.Command("docker", pusharg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed pushing image: "+string(out))
+	}
+	Info(":whale: Pushed image:", name)
+	//Info(string(out))
+	return nil
+}
+
 func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
@@ -229,11 +248,27 @@ func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLaye
 		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
 	}
 
+	if config.LuetCfg.GetGeneral().ShowBuildOutput {
+		Info(string(out))
+	}
+
 	var diffs []compiler.ArtifactLayer
 
 	err = json.Unmarshal(out, &diffs)
 	if err != nil {
 		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
 	}
+
+	if config.LuetCfg.GetLogging().Level == "debug" {
+		summary := compiler.ComputeArtifactLayerSummary(diffs)
+		for _, l := range summary.Layers {
+			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
+				l.FromImage, l.ToImage,
+				l.AddFiles, l.AddSizes,
+				l.DelFiles, l.DelSizes,
+				l.ChangeFiles, l.ChangeSizes))
+		}
+	}
+
 	return diffs, nil
 }

pkg/compiler/backend/simpledocker_test.go

@@ -40,7 +40,7 @@ var _ = Describe("Docker backend", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 

pkg/compiler/backend/simpleimg.go

@@ -145,3 +145,15 @@ func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms
 func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
 	return NewSimpleDockerBackend().Changes(fromImage, toImage)
 }
+
+func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {
+	name := opts.ImageName
+	pusharg := []string{"push", name}
+	out, err := exec.Command("img", pusharg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed pushing image: "+string(out))
+	}
+	Info(":tea: Pushed image:", name)
+	//Info(string(out))
+	return nil
+}

pkg/compiler/checksum.go

@@ -0,0 +1,78 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+
+	//"strconv"
+
+	"crypto/sha256"
+	"fmt"
+	"hash"
+	"io"
+	"os"
+
+	//	. "github.com/mudler/luet/pkg/logger"
+	"github.com/pkg/errors"
+)
+
+type HashImplementation string
+
+const (
+	SHA256 HashImplementation = "sha256"
+)
+
+type Checksums map[string]string
+
+type HashOptions struct {
+	Hasher hash.Hash
+	Type   HashImplementation
+}
+
+// Generate generates all Checksums supported for the artifact
+func (c *Checksums) Generate(a Artifact) error {
+	return c.generateSHA256(a)
+}
+
+func (c Checksums) Compare(d Checksums) error {
+	for t, sum := range d {
+		if v, ok := c[t]; ok && v != sum {
+			return errors.New("Checksum mismsatch")
+		}
+	}
+	return nil
+}
+
+func (c *Checksums) generateSHA256(a Artifact) error {
+	return c.generateSum(a, HashOptions{Hasher: sha256.New(), Type: SHA256})
+}
+
+func (c *Checksums) generateSum(a Artifact, opts HashOptions) error {
+
+	f, err := os.Open(a.GetPath())
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := io.Copy(opts.Hasher, f); err != nil {
+		return err
+	}
+
+	sum := fmt.Sprintf("%x", opts.Hasher.Sum(nil))
+
+	(*c)[string(opts.Type)] = sum
+	return nil
+}

pkg/compiler/checksum_test.go

@@ -0,0 +1,61 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler_test
+
+import (
+	"io/ioutil"
+	"os"
+
+	. "github.com/mudler/luet/pkg/compiler"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Checksum", func() {
+	Context("Generation", func() {
+		It("Compares successfully", func() {
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			buildsum := Checksums{}
+			definitionsum := Checksums{}
+			definitionsum2 := Checksums{}
+
+			Expect(len(buildsum)).To(Equal(0))
+			Expect(len(definitionsum)).To(Equal(0))
+			Expect(len(definitionsum2)).To(Equal(0))
+
+			err = buildsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/build.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = definitionsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = definitionsum2.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(buildsum)).To(Equal(1))
+			Expect(len(definitionsum)).To(Equal(1))
+			Expect(len(definitionsum2)).To(Equal(1))
+
+			Expect(definitionsum.Compare(buildsum)).To(HaveOccurred())
+			Expect(definitionsum.Compare(definitionsum2)).ToNot(HaveOccurred())
+		})
+	})
+
+})

pkg/compiler/compiler.go

@@ -36,13 +36,16 @@ const BuildFile = "build.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
-	Backend            CompilerBackend
-	Database           pkg.PackageDatabase
-	ImageRepository    string
-	PullFirst, KeepImg bool
+	Backend                   CompilerBackend
+	Database                  pkg.PackageDatabase
+	ImageRepository           string
+	PullFirst, KeepImg, Clean bool
+	Concurrency               int
+	CompressionType           CompressionImplementation
+	Options                   CompilerOptions
 }
 
-func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase) Compiler {
+func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions) Compiler {
 	// The CompilerRecipe will gives us a tree with only build deps listed.
 	return &LuetCompiler{
 		Backend: backend,
@@ -50,12 +53,24 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase) Compiler {
 			tree.Recipe{Database: db},
 		},
 		Database:        db,
-		ImageRepository: "luet/cache",
-		PullFirst:       true,
-		KeepImg:         true,
+		ImageRepository: opt.ImageRepository,
+		PullFirst:       opt.PullFirst,
+		CompressionType: opt.CompressionType,
+		KeepImg:         opt.KeepImg,
+		Concurrency:     opt.Concurrency,
+		Clean:           opt.Clean,
+		Options:         *opt,
 	}
 }
 
+func (cs *LuetCompiler) SetConcurrency(i int) {
+	cs.Concurrency = i
+}
+
+func (cs *LuetCompiler) SetCompressionType(t CompressionImplementation) {
+	cs.CompressionType = t
+}
+
 func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan CompilationSpec, a *[]Artifact, m *sync.Mutex, concurrency int, keepPermissions bool, errors chan error) {
 	defer wg.Done()
 
@@ -70,8 +85,9 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan Co
 		m.Unlock()
 	}
 }
-func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
-	artifacts, err := cs.CompileParallel(concurrency, keepPermissions, ps)
+
+func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
+	artifacts, err := cs.CompileParallel(keepPermissions, ps)
 	if len(err) != 0 {
 		return artifacts, err
 	}
@@ -119,11 +135,11 @@ func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions
 		Info(" :arrow_right_hook:", u.GetPackage().GetName(), ":leaves:", u.GetPackage().GetVersion(), "(", u.GetPackage().GetCategory(), ")")
 	}
 
-	artifacts2, err := cs.CompileParallel(concurrency, keepPermissions, uniques)
+	artifacts2, err := cs.CompileParallel(keepPermissions, uniques)
 	return append(artifacts, artifacts2...), err
 }
 
-func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
+func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
 	Spinner(22)
 	defer SpinnerStop()
 	all := make(chan CompilationSpec)
@@ -131,9 +147,9 @@ func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, p
 	mutex := &sync.Mutex{}
 	errors := make(chan error, ps.Len())
 	var wg = new(sync.WaitGroup)
-	for i := 0; i < concurrency; i++ {
+	for i := 0; i < cs.Concurrency; i++ {
 		wg.Add(1)
-		go cs.compilerWorker(i, wg, all, &artifacts, mutex, concurrency, keepPermissions, errors)
+		go cs.compilerWorker(i, wg, all, &artifacts, mutex, cs.Concurrency, keepPermissions, errors)
 	}
 
 	for _, p := range ps.All() {
@@ -213,6 +229,12 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 }
 
 func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec) (Artifact, error) {
+	if !cs.Clean {
+		if art, err := LoadArtifactFromYaml(p); err == nil {
+			Debug("Artifact reloaded. Skipping build")
+			return art, err
+		}
+	}
 	pkgTag := ":package:  " + p.GetPackage().GetName()
 
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
@@ -234,17 +256,21 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		return nil, errors.Wrap(err, "Could not copy package sources")
 
 	}
-	if buildertaggedImage == "" {
-		buildertaggedImage = cs.ImageRepository + "-" + p.GetPackage().GetFingerPrint() + "-builder"
-	}
-	if packageImage == "" {
-		packageImage = cs.ImageRepository + "-" + p.GetPackage().GetFingerPrint()
+
+	// Copy file into the build context, the compilespec might have requested to do so.
+	if len(p.GetRetrieve()) > 0 {
+		err := p.CopyRetrieves(buildDir)
+		if err != nil {
+			Warning("Failed copying retrieves", err.Error())
+		}
 	}
 
-	if cs.PullFirst {
-		//Best effort pull
-		cs.Backend.DownloadImage(CompilerBackendOptions{ImageName: buildertaggedImage})
-		cs.Backend.DownloadImage(CompilerBackendOptions{ImageName: packageImage})
+	fp := p.GetPackage().HashFingerprint()
+	if buildertaggedImage == "" {
+		buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder"
+	}
+	if packageImage == "" {
+		packageImage = cs.ImageRepository + "-" + fp
 	}
 
 	Info(pkgTag, "Generating :whale: definition for builder image from", image)
@@ -258,16 +284,32 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 	}
 
-	err = cs.Backend.BuildImage(builderOpts)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
+	buildBuilderImage := true
+	if cs.Options.PullFirst {
+		if err := cs.Backend.DownloadImage(builderOpts); err == nil {
+			buildBuilderImage = false
+		}
 	}
 
-	err = cs.Backend.ExportImage(builderOpts)
-	if err != nil {
+	if buildBuilderImage {
+		if err = cs.Backend.BuildImage(builderOpts); err != nil {
+			return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
+
+	if err = cs.Backend.ExportImage(builderOpts); err != nil {
 		return nil, errors.Wrap(err, "Could not export image")
 	}
 
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(builderOpts.Destination)
+	}
+
+	if cs.Options.Push && buildBuilderImage {
+		if err = cs.Backend.Push(builderOpts); err != nil {
+			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
 	// Then we write the step image, which uses the builder one
 	p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
 	runnerOpts := CompilerBackendOptions{
@@ -283,12 +325,34 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	// 		return nil, errors.Wrap(err, "Could not export image to tar")
 	// 	}
 	// } else {
-	if err := cs.Backend.BuildImage(runnerOpts); err != nil {
-		return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
+	buildPackageImage := true
+	if cs.Options.PullFirst {
+		//Best effort pull
+		if err := cs.Backend.DownloadImage(runnerOpts); err == nil {
+			buildPackageImage = false
+		}
 	}
+
+	if buildPackageImage {
+		if err := cs.Backend.BuildImage(runnerOpts); err != nil {
+			return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
+		}
+	}
+
 	if err := cs.Backend.ExportImage(runnerOpts); err != nil {
 		return nil, errors.Wrap(err, "Failed exporting image")
 	}
+
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(runnerOpts.Destination)
+	}
+
+	if cs.Options.Push && buildPackageImage {
+		err = cs.Backend.Push(runnerOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
 	//	}
 
 	var diffs []ArtifactLayer
@@ -321,13 +385,11 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		// TODO: Handle caching and optionally do not remove things
 		err = cs.Backend.RemoveImage(builderOpts)
 		if err != nil {
-			// TODO: Have a --fatal flag which enables Warnings to exit.
 			Warning("Could not remove image ", builderOpts.ImageName)
 			//	return nil, errors.Wrap(err, "Could not remove image")
 		}
 		err = cs.Backend.RemoveImage(runnerOpts)
 		if err != nil {
-			// TODO: Have a --fatal flag which enables Warnings to exit.
 			Warning("Could not remove image ", builderOpts.ImageName)
 			//	return nil, errors.Wrap(err, "Could not remove image")
 		}
@@ -339,21 +401,22 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 			// strip from includes
 			cs.stripIncludesFromRootfs(p.GetIncludes(), rootfs)
 		}
-
-		err = helpers.Tar(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"))
+		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
+		artifact.SetCompressionType(cs.CompressionType)
+		err = artifact.Compress(rootfs, concurrency)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating package archive")
 		}
 
-		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 		artifact.SetCompileSpec(p)
 	} else {
 		Info(pkgTag, "Generating delta")
 
-		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes())
+		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not generate deltas")
 		}
+
 		artifact.SetCompileSpec(p)
 	}
 
@@ -366,8 +429,14 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	return artifact, nil
 }
 
-func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions, keepImg bool) (Artifact, error) {
-	pkgTag := ":package:  " + p.GetPackage().GetName()
+func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions, keepImg bool, concurrency int) (Artifact, error) {
+	if !cs.Clean {
+		if art, err := LoadArtifactFromYaml(p); err == nil {
+			Debug("Artifact reloaded. Skipping build")
+			return art, err
+		}
+	}
+	pkgTag := ":package:  " + p.GetPackage().HumanReadableString()
 
 	Info(pkgTag, "   🍩 Build starts 🔨 🔨 🔨 ")
 
@@ -391,6 +460,10 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 		return nil, errors.Wrap(err, "Could not export image")
 	}
 
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(builderOpts.Destination)
+	}
+
 	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
@@ -404,8 +477,11 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not extract rootfs")
 	}
+	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
+	artifact.SetCompileSpec(p)
+	artifact.SetCompressionType(cs.CompressionType)
 
-	err = helpers.Tar(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"))
+	err = artifact.Compress(rootfs, concurrency)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}
@@ -415,15 +491,13 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 		// TODO: Handle caching and optionally do not remove things
 		err = cs.Backend.RemoveImage(builderOpts)
 		if err != nil {
-			// TODO: Have a --fatal flag which enables Warnings to exit.
 			Warning("Could not remove image ", builderOpts.ImageName)
 			//	return nil, errors.Wrap(err, "Could not remove image")
 		}
 	}
 
-	Info(pkgTag, "   :white_check_mark: Done")
-	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
-	artifact.SetCompileSpec(p)
+	Info(pkgTag, " :white_check_mark: Done")
+
 	err = artifact.WriteYaml(p.GetOutputPath())
 	if err != nil {
 		return artifact, err
@@ -433,11 +507,11 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 
 func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
 
-	s := solver.NewSolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false))
+	s := solver.NewResolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
 
 	solution, err := s.Install([]pkg.Package{p.GetPackage()})
 	if err != nil {
-		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
 	}
 
 	dependencies := solution.Order(cs.Database, p.GetPackage().GetFingerPrint())
@@ -459,17 +533,17 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
 }
 
 // Compile is non-parallel
-func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
+func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifact, error) {
 	asserts, err := cs.ComputeDepTree(p)
 	if err != nil {
 		panic(err)
 	}
 	p.SetSourceAssertion(asserts)
-	return cs.compile(concurrency, keepPermissions, p)
+	return cs.compile(cs.Concurrency, keepPermissions, p)
 }
 
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
-	Info(":package: Compiling", p.GetPackage().GetName(), "version", p.GetPackage().GetVersion(), ".... :coffee:")
+	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
 	if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" {
 		Error("Package with no deps and no seed image supplied, bailing out")
@@ -480,7 +554,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
 		if p.ImageUnpack() { // If it is just an entire image, create a package from it
-			return cs.packageFromImage(p, "", keepPermissions, cs.KeepImg)
+			return cs.packageFromImage(p, "", keepPermissions, cs.KeepImg, concurrency)
 		}
 
 		return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, cs.KeepImg, p)
@@ -497,74 +571,82 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	depsN := 0
 	currentN := 0
 
-	Info(":deciduous_tree: Build dependencies for " + p.GetPackage().GetName())
-	for _, assertion := range dependencies { //highly dependent on the order
-		depsN++
-		Info(" :arrow_right_hook:", assertion.Package.GetName(), ":leaves:", assertion.Package.GetVersion(), "(", assertion.Package.GetCategory(), ")")
-
-	}
-
-	for _, assertion := range dependencies { //highly dependent on the order
-		currentN++
-		pkgTag := fmt.Sprintf(":package:  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().GetName(), assertion.Package.GetName())
-		Info(pkgTag, "   :zap:  Building dependency")
-		compileSpec, err := cs.FromPackage(assertion.Package)
-		if err != nil {
-			return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+	if !cs.Options.NoDeps {
+		Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
+		for _, assertion := range dependencies { //highly dependent on the order
+			depsN++
+			Info(" :arrow_right_hook:", assertion.Package.HumanReadableString(), ":leaves:")
 		}
-		compileSpec.SetOutputPath(p.GetOutputPath())
 
-		buildImageHash := cs.ImageRepository + ":" + assertion.Hash.BuildHash
-		currentPackageImageHash := cs.ImageRepository + ":" + assertion.Hash.PackageHash
-		Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from", buildImageHash)
-		Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
+		for _, assertion := range dependencies { //highly dependent on the order
+			currentN++
+			pkgTag := fmt.Sprintf(":package:  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
+			Info(pkgTag, "   :zap:  Building dependency")
+			compileSpec, err := cs.FromPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+			}
+			compileSpec.SetOutputPath(p.GetOutputPath())
 
-		lastHash = currentPackageImageHash
-		if compileSpec.GetImage() != "" {
-			// TODO: Refactor this
-			if compileSpec.ImageUnpack() { // If it is just an entire image, create a package from it
-				if compileSpec.GetImage() == "" {
-					return nil, errors.New("No image defined for package: " + assertion.Package.GetName())
+			buildImageHash := cs.ImageRepository + ":" + assertion.Hash.BuildHash
+			currentPackageImageHash := cs.ImageRepository + ":" + assertion.Hash.PackageHash
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from", buildImageHash)
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
+
+			lastHash = currentPackageImageHash
+			if compileSpec.GetImage() != "" {
+				// TODO: Refactor this
+				if compileSpec.ImageUnpack() { // If it is just an entire image, create a package from it
+					if compileSpec.GetImage() == "" {
+						return nil, errors.New("No image defined for package: " + assertion.Package.HumanReadableString())
+					}
+					Info(pkgTag, ":whale: Sourcing package from image", compileSpec.GetImage())
+					artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions, cs.KeepImg, concurrency)
+					if err != nil {
+						return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
+					}
+					departifacts = append(departifacts, artifact)
+					continue
 				}
-				Info(pkgTag, ":whale: Sourcing package from image", compileSpec.GetImage())
-				artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions, cs.KeepImg)
+
+				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
+				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
 				if err != nil {
-					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
+					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				}
 				departifacts = append(departifacts, artifact)
+				Info(pkgTag, ":white_check_mark: Done")
 				continue
 			}
 
-			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().GetFingerPrint()+" from image")
-			artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
+			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
 			if err != nil {
-				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
+				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
+				//	deperrs = append(deperrs, err)
+				//		break // stop at first error
 			}
 			departifacts = append(departifacts, artifact)
-			Info(pkgTag, ":white_check_mark: Done")
-			continue
+			Info(pkgTag, ":collision: Done")
 		}
 
-		Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().GetFingerPrint()+" from tree")
-		artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+	} else if len(dependencies) > 0 {
+		lastHash = dependencies[len(dependencies)-1].Hash.PackageHash
+	}
+
+	if !cs.Options.OnlyDeps {
+		Info(":package:", p.GetPackage().HumanReadableString(), ":cyclone:  Building package target from:", lastHash)
+		artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, cs.KeepImg, p)
 		if err != nil {
-			return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
-			//	deperrs = append(deperrs, err)
-			//		break // stop at first error
+			return artifact, err
 		}
-		departifacts = append(departifacts, artifact)
-		Info(pkgTag, ":collision: Done")
-	}
+		artifact.SetDependencies(departifacts)
+		artifact.SetSourceAssertion(p.GetSourceAssertion())
 
-	Info(":package:", p.GetPackage().GetName(), ":cyclone:  Building package target from:", lastHash)
-	artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, cs.KeepImg, p)
-	if err != nil {
 		return artifact, err
+	} else {
+		return departifacts[len(departifacts)-1], nil
 	}
-	artifact.SetDependencies(departifacts)
-	artifact.SetSourceAssertion(p.GetSourceAssertion())
-
-	return artifact, err
 }
 
 func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {

pkg/compiler/compiler_suite_test.go

@@ -18,11 +18,15 @@ package compiler_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Compiler Suite")
 }

pkg/compiler/compiler_test.go

@@ -38,7 +38,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -53,7 +53,9 @@ var _ = Describe("Compiler", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			artifact, err := compiler.Compile(2, false, spec)
+			compiler.SetConcurrency(2)
+
+			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -80,7 +82,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -95,7 +97,8 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec, spec2))
+			compiler.SetConcurrency(2)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
 				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
@@ -117,7 +120,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -134,8 +137,9 @@ var _ = Describe("Compiler", func() {
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(2)
 
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec, spec2, spec3))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2, spec3))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(3))
 
@@ -173,7 +177,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -181,12 +185,12 @@ var _ = Describe("Compiler", func() {
 			Expect(err).ToNot(HaveOccurred())
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
-			artifacts2, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec2))
+			artifacts2, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts2)).To(Equal(1))
 
@@ -215,7 +219,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -224,8 +228,9 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
@@ -249,7 +254,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -258,8 +263,8 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
@@ -287,7 +292,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -296,8 +301,9 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
@@ -327,7 +333,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -336,8 +342,9 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
@@ -370,7 +377,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -379,8 +386,9 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
@@ -411,7 +419,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -421,7 +429,7 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(2))
 
@@ -449,7 +457,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
 			Expect(err).ToNot(HaveOccurred())
@@ -459,7 +467,7 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].GetDependencies())).To(Equal(6))
@@ -488,13 +496,13 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(4))
 
@@ -540,7 +548,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -552,7 +560,9 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(2)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
 				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
@@ -581,7 +591,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -593,7 +603,9 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
@@ -602,4 +614,40 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
 	})
+
+	Context("Compression", func() {
+		It("Builds packages in gzip", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/packagelayers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			compiler.SetCompressionType(GZip)
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
+			Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred())
+			//	Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
+		})
+	})
 })

pkg/compiler/interface.go

@@ -16,20 +16,24 @@
 package compiler
 
 import (
+	"runtime"
+
+	"github.com/mudler/luet/pkg/config"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 )
 
 type Compiler interface {
-	Compile(int, bool, CompilationSpec) (Artifact, error)
-	CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
+	Compile(bool, CompilationSpec) (Artifact, error)
+	CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
+	CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
-
+	SetConcurrency(i int)
 	FromPackage(pkg.Package) (CompilationSpec, error)
 
 	SetBackend(CompilerBackend)
 	GetBackend() CompilerBackend
+	SetCompressionType(t CompressionImplementation)
 }
 
 type CompilerBackendOptions struct {
@@ -39,6 +43,33 @@ type CompilerBackendOptions struct {
 	Destination    string
 }
 
+type CompilerOptions struct {
+	ImageRepository          string
+	PullFirst, KeepImg, Push bool
+	Concurrency              int
+	CompressionType          CompressionImplementation
+	Clean                    bool
+	KeepImageExport          bool
+
+	OnlyDeps      bool
+	NoDeps        bool
+	SolverOptions config.LuetSolverOptions
+}
+
+func NewDefaultCompilerOptions() *CompilerOptions {
+	return &CompilerOptions{
+		ImageRepository: "luet/cache",
+		PullFirst:       false,
+		Push:            false,
+		CompressionType: None,
+		KeepImg:         true,
+		Concurrency:     runtime.NumCPU(),
+		Clean:           true,
+		OnlyDeps:        false,
+		NoDeps:          false,
+	}
+}
+
 type CompilerBackend interface {
 	BuildImage(CompilerBackendOptions) error
 	ExportImage(CompilerBackendOptions) error
@@ -49,6 +80,8 @@ type CompilerBackend interface {
 
 	CopyImage(string, string) error
 	DownloadImage(opts CompilerBackendOptions) error
+
+	Push(opts CompilerBackendOptions) error
 }
 
 type Artifact interface {
@@ -62,6 +95,15 @@ type Artifact interface {
 	SetCompileSpec(as CompilationSpec)
 	GetCompileSpec() CompilationSpec
 	WriteYaml(dst string) error
+	Unpack(dst string, keepPerms bool) error
+	Compress(src string, concurrency int) error
+	SetCompressionType(t CompressionImplementation)
+	FileList() ([]string, error)
+	Hash() error
+	Verify() error
+
+	GetChecksums() Checksums
+	SetChecksums(c Checksums)
 }
 
 type ArtifactNode struct {
@@ -78,6 +120,19 @@ type ArtifactLayer struct {
 	ToImage   string        `json:"Image2"`
 	Diffs     ArtifactDiffs `json:"Diff"`
 }
+type ArtifactLayerSummary struct {
+	FromImage   string `json:"image1"`
+	ToImage     string `json:"image2"`
+	AddFiles    int    `json:"add_files"`
+	AddSizes    int64  `json:"add_sizes"`
+	DelFiles    int    `json:"del_files"`
+	DelSizes    int64  `json:"del_sizes"`
+	ChangeFiles int    `json:"change_files"`
+	ChangeSizes int64  `json:"change_sizes"`
+}
+type ArtifactLayersSummary struct {
+	Layers []ArtifactLayerSummary `json:"summary"`
+}
 
 // CompilationSpec represent a compilation specification derived from a package
 type CompilationSpec interface {
@@ -107,6 +162,9 @@ type CompilationSpec interface {
 
 	GetSourceAssertion() solver.PackagesAssertions
 	SetSourceAssertion(as solver.PackagesAssertions)
+
+	GetRetrieve() []string
+	CopyRetrieves(dest string) error
 }
 
 type CompilationSpecs interface {

pkg/compiler/spec.go

@@ -21,6 +21,7 @@ import (
 
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
+	"github.com/otiai10/copy"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -95,6 +96,8 @@ type LuetCompilationSpec struct {
 	Package         *pkg.DefaultPackage       `json:"package"`
 	SourceAssertion solver.PackagesAssertions `json:"-"`
 
+	Retrieve []string `json:"retrieve"`
+
 	OutputPath string   `json:"-"` // Where the build processfiles go
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
@@ -136,6 +139,10 @@ func (cs *LuetCompilationSpec) GetIncludes() []string {
 	return cs.Includes
 }
 
+func (cs *LuetCompilationSpec) GetRetrieve() []string {
+	return cs.Retrieve
+}
+
 func (cs *LuetCompilationSpec) GetSeedImage() string {
 	return cs.Seed
 }
@@ -164,6 +171,24 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 	cs.Seed = s
 }
 
+func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
+	var err error
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			matches, err := filepath.Glob(cs.Rel(s))
+
+			if err != nil {
+				continue
+			}
+
+			for _, m := range matches {
+				err = copy.Copy(m, filepath.Join(dest, filepath.Base(m)))
+			}
+		}
+	}
+	return err
+}
+
 // TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
 func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
 	spec := `
@@ -174,6 +199,19 @@ ENV PACKAGE_NAME=` + cs.Package.GetName() + `
 ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
 ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
 
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			//var file string
+			// if helpers.IsValidUrl(s) {
+			// 	file = s
+			// } else {
+			// 	file = cs.Rel(s)
+			// }
+			spec = spec + `
+ADD ` + s + ` /luetbuild/`
+		}
+	}
+
 	for _, s := range cs.Env {
 		spec = spec + `
 ENV ` + s

pkg/compiler/spec_test.go

@@ -61,7 +61,7 @@ var _ = Describe("Spec", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -105,4 +105,74 @@ RUN echo bar > /test2`))
 		})
 
 	})
+
+	It("Renders retrieve and env fields", func() {
+		generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
+
+		err := generalRecipe.Load("../../tests/fixtures/retrieve")
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+		spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
+		Expect(err).ToNot(HaveOccurred())
+
+		lspec, ok := spec.(*LuetCompilationSpec)
+		Expect(ok).To(BeTrue())
+
+		Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
+		Expect(lspec.Image).To(Equal("luet/base"))
+		Expect(lspec.Seed).To(Equal("alpine"))
+		tmpdir, err := ioutil.TempDir("", "tree")
+		Expect(err).ToNot(HaveOccurred())
+		defer os.RemoveAll(tmpdir) // clean up
+
+		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1`))
+
+		lspec.SetOutputPath("/foo/bar")
+
+		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1`))
+
+		err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(dockerfile).To(Equal(`
+FROM luet/base
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ENV test=1
+RUN echo foo > /test
+RUN echo bar > /test2`))
+
+	})
 })

pkg/config/config.go

@@ -0,0 +1,326 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	solver "github.com/mudler/luet/pkg/solver"
+	v "github.com/spf13/viper"
+)
+
+var LuetCfg = NewLuetConfig(v.GetViper())
+var AvailableResolvers = strings.Join([]string{solver.QLearningResolverType}, " ")
+
+type LuetLoggingConfig struct {
+	Path       string `mapstructure:"path"`
+	JsonFormat bool   `mapstructure:"json_format"`
+	Level      string `mapstructure:"level"`
+}
+
+type LuetGeneralConfig struct {
+	SameOwner       bool `mapstructure:"same_owner"`
+	Concurrency     int  `mapstructure:"concurrency"`
+	Debug           bool `mapstructure:"debug"`
+	ShowBuildOutput bool `mapstructure:"show_build_output"`
+	SpinnerMs       int  `mapstructure:"spinner_ms"`
+	SpinnerCharset  int  `mapstructure:"spinner_charset"`
+	FatalWarns      bool `mapstructure:"fatal_warnings"`
+}
+
+type LuetSolverOptions struct {
+	Type        string  `mapstructure:"type"`
+	LearnRate   float32 `mapstructure:"rate"`
+	Discount    float32 `mapstructure:"discount"`
+	MaxAttempts int     `mapstructure:"max_attempts"`
+}
+
+func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		if opts.LearnRate != 0.0 {
+			return solver.NewQLearningResolver(opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+
+		}
+		return solver.SimpleQLearningSolver()
+	}
+
+	return &solver.DummyPackageResolver{}
+}
+
+func (opts *LuetSolverOptions) CompactString() string {
+	return fmt.Sprintf("type: %s rate: %f, discount: %f, attempts: %d, initialobserved: %d",
+		opts.Type, opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+}
+
+type LuetSystemConfig struct {
+	DatabaseEngine string `yaml:"database_engine" mapstructure:"database_engine"`
+	DatabasePath   string `yaml:"database_path" mapstructure:"database_path"`
+	Rootfs         string `yaml:"rootfs" mapstructure:"rootfs"`
+	PkgsCachePath  string `yaml:"pkgs_cache_path" mapstructure:"pkgs_cache_path"`
+}
+
+func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
+	dbpath = filepath.Join(dbpath, "repos/"+name)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+	dbpath := filepath.Join(sc.Rootfs,
+		sc.DatabasePath)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+	var cachepath string
+	if sc.PkgsCachePath != "" {
+		cachepath = sc.PkgsCachePath
+	} else {
+		// Create dynamic cache for test suites
+		cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
+	}
+
+	if filepath.IsAbs(cachepath) {
+		ans = cachepath
+	} else {
+		ans = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
+	}
+
+	return
+}
+
+type LuetRepository struct {
+	Name           string            `json:"name" yaml:"name" mapstructure:"name"`
+	Description    string            `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
+	Urls           []string          `json:"urls" yaml:"urls" mapstructure:"urls"`
+	Type           string            `json:"type" yaml:"type" mapstructure:"type"`
+	Mode           string            `json:"mode,omitempty" yaml:"mode,omitempty" mapstructure:"mode,omitempty"`
+	Priority       int               `json:"priority,omitempty" yaml:"priority,omitempty" mapstructure:"priority"`
+	Enable         bool              `json:"enable" yaml:"enable" mapstructure:"enable"`
+	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
+	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
+	TreePath       string            `json:"tree_path,omitempty" yaml:"tree_path,omitempty" mapstructure:"tree_path"`
+
+	// Serialized options not used in repository configuration
+
+	// Incremented value that identify revision of the repository in a user-friendly way.
+	Revision int `json:"revision,omitempty" yaml:"-,omitempty" mapstructure:"-,omitempty"`
+	// Epoch time in seconds
+	LastUpdate string `json:"last_update,omitempty" yaml:"-,omitempty" mapstructure:"-,omitempty"`
+}
+
+func NewLuetRepository(name, t, descr string, urls []string, priority int, enable, cached bool) *LuetRepository {
+	return &LuetRepository{
+		Name:        name,
+		Description: descr,
+		Urls:        urls,
+		Type:        t,
+		// Used in cached repositories
+		Mode:           "",
+		Priority:       priority,
+		Enable:         enable,
+		Cached:         cached,
+		Authentication: make(map[string]string, 0),
+		TreePath:       "",
+	}
+}
+
+func NewEmptyLuetRepository() *LuetRepository {
+	return &LuetRepository{
+		Name:           "",
+		Description:    "",
+		Urls:           []string{},
+		Type:           "",
+		Priority:       9999,
+		TreePath:       "",
+		Enable:         false,
+		Cached:         false,
+		Authentication: make(map[string]string, 0),
+	}
+}
+
+func (r *LuetRepository) String() string {
+	return fmt.Sprintf("[%s] prio: %d, type: %s, enable: %t, cached: %t",
+		r.Name, r.Priority, r.Type, r.Enable, r.Cached)
+}
+
+type LuetConfig struct {
+	Viper *v.Viper
+
+	Logging LuetLoggingConfig `mapstructure:"logging"`
+	General LuetGeneralConfig `mapstructure:"general"`
+	System  LuetSystemConfig  `mapstructure:"system"`
+	Solver  LuetSolverOptions `mapstructure:"solver"`
+
+	RepositoriesConfDir []string         `mapstructure:"repos_confdir"`
+	CacheRepositories   []LuetRepository `mapstructure:"repetitors"`
+	SystemRepositories  []LuetRepository `mapstructure:"repositories"`
+}
+
+func NewLuetConfig(viper *v.Viper) *LuetConfig {
+	if viper == nil {
+		viper = v.New()
+	}
+
+	GenDefault(viper)
+	return &LuetConfig{Viper: viper}
+}
+
+func GenDefault(viper *v.Viper) {
+	viper.SetDefault("logging.level", "info")
+	viper.SetDefault("logging.path", "")
+	viper.SetDefault("logging.json_format", false)
+
+	viper.SetDefault("general.concurrency", runtime.NumCPU())
+	viper.SetDefault("general.debug", false)
+	viper.SetDefault("general.show_build_output", false)
+	viper.SetDefault("general.spinner_ms", 100)
+	viper.SetDefault("general.spinner_charset", 22)
+	viper.SetDefault("general.fatal_warnings", false)
+
+	u, _ := user.Current()
+	if u.Uid == "0" {
+		viper.SetDefault("general.same_owner", true)
+	} else {
+		viper.SetDefault("general.same_owner", false)
+	}
+
+	viper.SetDefault("system.database_engine", "boltdb")
+	viper.SetDefault("system.database_path", "/var/cache/luet")
+	viper.SetDefault("system.rootfs", "/")
+	viper.SetDefault("system.pkgs_cache_path", "packages")
+
+	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
+	viper.SetDefault("cache_repositories", []string{})
+	viper.SetDefault("system_repositories", []string{})
+
+	viper.SetDefault("solver.type", "")
+	viper.SetDefault("solver.rate", 0.7)
+	viper.SetDefault("solver.discount", 1.0)
+	viper.SetDefault("solver.max_attempts", 9000)
+}
+
+func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
+	c.SystemRepositories = append(c.SystemRepositories, r)
+}
+
+func (c *LuetConfig) GetLogging() *LuetLoggingConfig {
+	return &c.Logging
+}
+
+func (c *LuetConfig) GetGeneral() *LuetGeneralConfig {
+	return &c.General
+}
+
+func (c *LuetConfig) GetSystem() *LuetSystemConfig {
+	return &c.System
+}
+
+func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
+	return &c.Solver
+}
+
+func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
+	var ans *LuetRepository = nil
+
+	for idx, repo := range c.SystemRepositories {
+		if repo.Name == name {
+			ans = &c.SystemRepositories[idx]
+			break
+		}
+	}
+	if ans == nil {
+		return nil, errors.New("Repository " + name + " not found")
+	}
+
+	return ans, nil
+}
+
+func (c *LuetSolverOptions) String() string {
+	ans := fmt.Sprintf(`
+solver:
+  type: %s
+  rate: %f
+  discount: %f
+  max_attempts: %d`, c.Type, c.LearnRate, c.Discount,
+		c.MaxAttempts)
+
+	return ans
+}
+
+func (c *LuetGeneralConfig) String() string {
+	ans := fmt.Sprintf(`
+general:
+  concurrency: %d
+  same_owner: %t
+  debug: %t
+  fatal_warnings: %t
+  show_build_output: %t
+  spinner_ms: %d
+  spinner_charset: %d`, c.Concurrency, c.SameOwner, c.Debug,
+		c.FatalWarns, c.ShowBuildOutput,
+		c.SpinnerMs, c.SpinnerCharset)
+
+	return ans
+}
+
+func (c *LuetGeneralConfig) GetSpinnerMs() time.Duration {
+	duration, err := time.ParseDuration(fmt.Sprintf("%dms", c.SpinnerMs))
+	if err != nil {
+		return 100 * time.Millisecond
+	}
+	return duration
+}
+
+func (c *LuetLoggingConfig) String() string {
+	ans := fmt.Sprintf(`
+logging:
+  path: %s
+  json_format: %t
+  level: %s`, c.Path, c.JsonFormat, c.Level)
+
+	return ans
+}
+
+func (c *LuetSystemConfig) String() string {
+	ans := fmt.Sprintf(`
+system:
+  database_engine: %s
+  database_path: %s
+  pkgs_cache_path: %s
+  rootfs: %s`,
+		c.DatabaseEngine, c.DatabasePath, c.PkgsCachePath, c.Rootfs)
+
+	return ans
+}

pkg/helpers/archive.go

@@ -16,8 +16,10 @@
 package helpers
 
 import (
+	"archive/tar"
 	"io"
 	"os"
+	"path/filepath"
 
 	"github.com/docker/docker/pkg/archive"
 )
@@ -49,14 +51,82 @@ func Tar(src, dest string) error {
 
 // Untar just a wrapper around the docker functions
 func Untar(src, dest string, sameOwner bool) error {
+	var ans error
+
 	in, err := os.Open(src)
 	if err != nil {
 		return err
 	}
 	defer in.Close()
 
-	return archive.Untar(in, dest, &archive.TarOptions{
-		NoLchown:        !sameOwner,
-		ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
-	})
+	if sameOwner {
+		// PRE: i have root privileged.
+
+		opts := &archive.TarOptions{
+			// NOTE: NoLchown boolean is used for chmod of the symlink
+			// Probably it's needed set this always to true.
+			NoLchown:        true,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+		}
+
+		ans = archive.Untar(in, dest, opts)
+	} else {
+
+		var fileReader io.ReadCloser = in
+
+		tr := tar.NewReader(fileReader)
+		for {
+			header, err := tr.Next()
+
+			switch {
+			case err == io.EOF:
+				goto tarEof
+			case err != nil:
+				return err
+			case header == nil:
+				continue
+			}
+
+			// the target location where the dir/file should be created
+			target := filepath.Join(dest, header.Name)
+
+			// Check the file type
+			switch header.Typeflag {
+
+			// if its a dir and it doesn't exist create it
+			case tar.TypeDir:
+				if _, err := os.Stat(target); err != nil {
+					if err := os.MkdirAll(target, 0755); err != nil {
+						return err
+					}
+				}
+
+				// handle creation of file
+			case tar.TypeReg:
+				f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+				if err != nil {
+					return err
+				}
+
+				// copy over contents
+				if _, err := io.Copy(f, tr); err != nil {
+					return err
+				}
+
+				// manually close here after each file operation; defering would cause each
+				// file close to wait until all operations have completed.
+				f.Close()
+
+			case tar.TypeSymlink:
+				source := header.Linkname
+				err := os.Symlink(source, target)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	tarEof:
+	}
+
+	return ans
 }

pkg/helpers/cli.go

@@ -0,0 +1,60 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"fmt"
+
+	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
+	pkg "github.com/mudler/luet/pkg/package"
+)
+
+func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
+	gp, err := _gentoo.ParsePackageStr(p)
+	if err != nil {
+		return nil, err
+	}
+	if gp.Version == "" {
+		gp.Version = "0"
+		gp.Condition = _gentoo.PkgCondGreaterEqual
+	}
+
+	pkgVersion := ""
+	if gp.VersionBuild != "" {
+		pkgVersion = fmt.Sprintf("%s%s%s+%s",
+			pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+			gp.Version,
+			gp.VersionSuffix,
+			gp.VersionBuild,
+		)
+	} else {
+		pkgVersion = fmt.Sprintf("%s%s%s",
+			pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+			gp.Version,
+			gp.VersionSuffix,
+		)
+	}
+
+	pack := &pkg.DefaultPackage{
+		Name:     gp.Name,
+		Category: gp.Category,
+		Version:  pkgVersion,
+		Uri:      make([]string, 0),
+	}
+
+	return pack, nil
+}

pkg/helpers/helpers_suite_test.go

@@ -18,11 +18,15 @@ package helpers_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Helpers Suite")
 }

pkg/helpers/math.go

@@ -0,0 +1,24 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+func Factorial(n uint64) (result uint64) {
+	if n > 0 {
+		result = n * Factorial(n-1)
+		return result
+	}
+	return 1
+}

pkg/helpers/sys.go

@@ -0,0 +1,32 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"os/exec"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+// This allows a multi-platform switch in the future
+func Exec(cmd string, args []string, env []string) error {
+	path, err := exec.LookPath(cmd)
+	if err != nil {
+		return errors.Wrap(err, "Could not find binary in path: "+cmd)
+	}
+	return syscall.Exec(path, args, env)
+}

pkg/installer/client/client_suite_test.go

@@ -18,11 +18,19 @@ package client_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestClient(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	// Set temporary directory for rootfs
+	config.LuetCfg.GetSystem().Rootfs = "/tmp/luet-root"
+	// Force dynamic path for packages cache
+	config.LuetCfg.GetSystem().PkgsCachePath = ""
 	RunSpecs(t, "Client Suite")
 }

pkg/installer/client/http.go

@@ -16,7 +16,9 @@
 package client
 
 import (
+	"fmt"
 	"io/ioutil"
+	"math"
 	"net/url"
 	"os"
 	"path"
@@ -25,6 +27,7 @@ import (
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 
 	"github.com/cavaliercoder/grab"
@@ -38,60 +41,151 @@ func NewHttpClient(r RepoData) *HttpClient {
 	return &HttpClient{RepoData: r}
 }
 
+func (c *HttpClient) PrepareReq(dst, url string) (*grab.Request, error) {
+
+	req, err := grab.NewRequest(dst, url)
+	if err != nil {
+		return nil, err
+	}
+
+	if val, ok := c.RepoData.Authentication["token"]; ok {
+		req.HTTPRequest.Header.Set("Authorization", "token "+val)
+	} else if val, ok := c.RepoData.Authentication["basic"]; ok {
+		req.HTTPRequest.Header.Set("Authorization", "Basic "+val)
+	}
+
+	return req, err
+}
+
+func Round(input float64) float64 {
+	if input < 0 {
+		return math.Ceil(input - 0.5)
+	}
+	return math.Floor(input + 0.5)
+}
+
 func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+	var u *url.URL = nil
+	var err error
+	var req *grab.Request
+	var temp string
+
 	artifactName := path.Base(artifact.GetPath())
-	Info("Downloading artifact", artifactName, "from", c.RepoData.Uri)
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
+	ok := false
 
-	temp, err := ioutil.TempDir(os.TempDir(), "tree")
-	if err != nil {
-		return nil, err
+	// Check if file is already in cache
+	if helpers.Exists(cacheFile) {
+		Info("Use artifact", artifactName, "from cache.")
+	} else {
+
+		temp, err = ioutil.TempDir(os.TempDir(), "tree")
+		if err != nil {
+			return nil, err
+		}
+		defer os.RemoveAll(temp)
+
+		client := grab.NewClient()
+
+		for _, uri := range c.RepoData.Urls {
+			Info("Downloading artifact", artifactName, "from", uri)
+
+			u, err = url.Parse(uri)
+			if err != nil {
+				continue
+			}
+			u.Path = path.Join(u.Path, artifactName)
+
+			req, err = c.PrepareReq(temp, u.String())
+			if err != nil {
+				continue
+			}
+
+			resp := client.Do(req)
+			if err = resp.Err(); err != nil {
+				continue
+			}
+
+			Info("Downloaded", artifactName, "of",
+				fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
+				fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
+
+			Debug("Copying file ", filepath.Join(temp, artifactName), "to", cacheFile)
+			err = helpers.CopyFile(filepath.Join(temp, artifactName), cacheFile)
+			if err != nil {
+				continue
+			}
+
+			ok = true
+			break
+		}
+
+		if !ok {
+			return nil, err
+		}
 	}
 
-	file, err := ioutil.TempFile(temp, "HttpClient")
-	if err != nil {
-		return nil, err
-	}
-
-	u, err := url.Parse(c.RepoData.Uri)
-	if err != nil {
-		return nil, err
-	}
-	u.Path = path.Join(u.Path, artifactName)
-
-	_, err = grab.Get(temp, u.String())
-	if err != nil {
-		return nil, err
-	}
-
-	err = helpers.CopyFile(filepath.Join(temp, artifactName), file.Name())
-
-	return compiler.NewPackageArtifact(file.Name()), nil
+	newart := artifact
+	newart.SetPath(cacheFile)
+	return newart, nil
 }
 
 func (c *HttpClient) DownloadFile(name string) (string, error) {
-	temp, err := ioutil.TempDir(os.TempDir(), "tree")
-	if err != nil {
-		return "", err
-	}
-	file, err := ioutil.TempFile(os.TempDir(), "HttpClient")
-	if err != nil {
-		return "", err
-	}
-	//defer os.Remove(file.Name())
-	u, err := url.Parse(c.RepoData.Uri)
-	if err != nil {
-		return "", err
-	}
-	u.Path = path.Join(u.Path, name)
+	var file *os.File = nil
+	var u *url.URL = nil
+	var err error
+	var req *grab.Request
+	var temp string
 
-	Info("Downloading", u.String())
+	ok := false
 
-	_, err = grab.Get(temp, u.String())
+	temp, err = ioutil.TempDir(os.TempDir(), "tree")
 	if err != nil {
 		return "", err
 	}
 
-	err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+	client := grab.NewClient()
+
+	for _, uri := range c.RepoData.Urls {
+
+		file, err = ioutil.TempFile(os.TempDir(), "HttpClient")
+		if err != nil {
+			continue
+		}
+
+		u, err = url.Parse(uri)
+		if err != nil {
+			continue
+		}
+		u.Path = path.Join(u.Path, name)
+
+		Info("Downloading", u.String())
+
+		req, err = c.PrepareReq(temp, u.String())
+		if err != nil {
+			continue
+		}
+
+		resp := client.Do(req)
+		if err = resp.Err(); err != nil {
+			continue
+		}
+
+		Info("Downloaded", filepath.Base(resp.Filename), "of",
+			fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
+			fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
+
+		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
+
+	if !ok {
+		return "", err
+	}
 
 	return file.Name(), err
 }

pkg/installer/client/http_test.go

@@ -44,7 +44,7 @@ var _ = Describe("Http client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewHttpClient(RepoData{Uri: ts.URL})
+			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
 			path, err := c.DownloadFile("test.txt")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path)).To(Equal("test"))
@@ -62,7 +62,7 @@ var _ = Describe("Http client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewHttpClient(RepoData{Uri: ts.URL})
+			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
 			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path.GetPath())).To(Equal("test"))

pkg/installer/client/interface.go

@@ -16,5 +16,6 @@
 package client
 
 type RepoData struct {
-	Uri string
-}
+	Urls           []string
+	Authentication map[string]string
+}

pkg/installer/client/local.go

@@ -21,6 +21,7 @@ import (
 	"path"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/mudler/luet/pkg/compiler"
@@ -36,28 +37,62 @@ func NewLocalClient(r RepoData) *LocalClient {
 }
 
 func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+	var err error
+
 	artifactName := path.Base(artifact.GetPath())
-	Info("Downloading artifact", artifactName, "from", c.RepoData.Uri)
-	file, err := ioutil.TempFile(os.TempDir(), "localclient")
-	if err != nil {
-		return nil, err
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
+
+	// Check if file is already in cache
+	if helpers.Exists(cacheFile) {
+		Info("Use artifact", artifactName, "from cache.")
+	} else {
+		ok := false
+		for _, uri := range c.RepoData.Urls {
+			Info("Downloading artifact", artifactName, "from", uri)
+
+			//defer os.Remove(file.Name())
+			err = helpers.CopyFile(filepath.Join(uri, artifactName), cacheFile)
+			if err != nil {
+				continue
+			}
+			ok = true
+			break
+		}
+
+		if !ok {
+			return nil, err
+		}
 	}
-	//defer os.Remove(file.Name())
 
-	err = helpers.CopyFile(filepath.Join(c.RepoData.Uri, artifactName), file.Name())
-
-	return compiler.NewPackageArtifact(file.Name()), nil
+	newart := artifact
+	newart.SetPath(cacheFile)
+	return newart, nil
 }
+
 func (c *LocalClient) DownloadFile(name string) (string, error) {
-	Info("Downloading file", name, "from", c.RepoData.Uri)
+	var err error
+	var file *os.File = nil
 
-	file, err := ioutil.TempFile(os.TempDir(), "localclient")
-	if err != nil {
-		return "", err
+	ok := false
+	for _, uri := range c.RepoData.Urls {
+		Info("Downloading file", name, "from", uri)
+		file, err = ioutil.TempFile(os.TempDir(), "localclient")
+		if err != nil {
+			continue
+		}
+		//defer os.Remove(file.Name())
+
+		err = helpers.CopyFile(filepath.Join(uri, name), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
 	}
-	//defer os.Remove(file.Name())
 
-	err = helpers.CopyFile(filepath.Join(c.RepoData.Uri, name), file.Name())
+	if ok {
+		return file.Name(), nil
+	}
 
-	return file.Name(), err
+	return "", err
 }

pkg/installer/client/local_test.go

@@ -39,7 +39,7 @@ var _ = Describe("Local client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewLocalClient(RepoData{Uri: tmpdir})
+			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
 			path, err := c.DownloadFile("test.txt")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path)).To(Equal("test"))
@@ -55,7 +55,7 @@ var _ = Describe("Local client", func() {
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			c := NewLocalClient(RepoData{Uri: tmpdir})
+			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
 			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Read(path.GetPath())).To(Equal("test"))

pkg/installer/installer.go

@@ -16,8 +16,6 @@
 package installer
 
 import (
-	"archive/tar"
-	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -27,6 +25,7 @@ import (
 
 	"github.com/ghodss/yaml"
 	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -36,9 +35,18 @@ import (
 	"github.com/pkg/errors"
 )
 
+type LuetInstallerOptions struct {
+	SolverOptions config.LuetSolverOptions
+	Concurrency   int
+	NoDeps        bool
+	OnlyDeps      bool
+	Force         bool
+}
+
 type LuetInstaller struct {
 	PackageRepositories Repositories
-	Concurrency         int
+
+	Options LuetInstallerOptions
 }
 
 type ArtifactMatch struct {
@@ -88,32 +96,21 @@ func NewLuetFinalizerFromYaml(data []byte) (*LuetFinalizer, error) {
 	return &p, err
 }
 
-func NewLuetInstaller(concurrency int) Installer {
-	return &LuetInstaller{Concurrency: concurrency}
+func NewLuetInstaller(opts LuetInstallerOptions) Installer {
+	return &LuetInstaller{Options: opts}
 }
 
 func (l *LuetInstaller) Upgrade(s *System) error {
-	Spinner(32)
-	defer SpinnerStop()
-	syncedRepos := Repositories{}
-	for _, r := range l.PackageRepositories {
-		repo, err := r.Sync()
-		if err != nil {
-			return errors.Wrap(err, "Failed syncing repository: "+r.GetName())
-		}
-		syncedRepos = append(syncedRepos, repo)
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
 	}
-
-	// compute what to install and from where
-	sort.Sort(syncedRepos)
-
 	// First match packages against repositories by priority
 	//	matches := syncedRepos.PackageMatches(p)
-
-	// compute a "big" world
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
-	solv := solver.NewSolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false))
+	// compute a "big" world
+	solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 	uninstall, solution, err := solv.Upgrade()
 	if err != nil {
 		return errors.Wrap(err, "Failed solving solution for upgrade")
@@ -121,8 +118,9 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 
 	for _, u := range uninstall {
 		err := l.Uninstall(u, s)
-		if err != nil {
-			Warning("Failed uninstall for ", u.GetFingerPrint())
+		if err != nil && !l.Options.Force {
+			Error("Failed uninstall for ", u.HumanReadableString())
+			return errors.Wrap(err, "uninstalling "+u.HumanReadableString())
 		}
 	}
 
@@ -136,16 +134,14 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 	return l.Install(toInstall, s)
 }
 
-func (l *LuetInstaller) Install(p []pkg.Package, s *System) error {
-	// First get metas from all repos (and decodes trees)
-
+func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
 	Spinner(32)
 	defer SpinnerStop()
 	syncedRepos := Repositories{}
 	for _, r := range l.PackageRepositories {
-		repo, err := r.Sync()
+		repo, err := r.Sync(false)
 		if err != nil {
-			return errors.Wrap(err, "Failed syncing repository: "+r.GetName())
+			return nil, errors.Wrap(err, "Failed syncing repository: "+r.GetName())
 		}
 		syncedRepos = append(syncedRepos, repo)
 	}
@@ -153,48 +149,96 @@ func (l *LuetInstaller) Install(p []pkg.Package, s *System) error {
 	// compute what to install and from where
 	sort.Sort(syncedRepos)
 
+	if !inMemory {
+		l.PackageRepositories = syncedRepos
+	}
+
+	return syncedRepos, nil
+}
+
+func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
+	var p []pkg.Package
+
+	// Check if the package is installed first
+	for _, pi := range cp {
+
+		vers, _ := s.Database.FindPackageVersions(pi)
+
+		if len(vers) >= 1 {
+			Warning("Filtering out package " + pi.HumanReadableString() + ", it has other versions already installed. Uninstall one of them first ")
+			continue
+			//return errors.New("Package " + pi.GetFingerPrint() + " has other versions already installed. Uninstall one of them first: " + strings.Join(vers, " "))
+
+		}
+		p = append(p, pi)
+	}
+
+	if len(p) == 0 {
+		Warning("No package to install, bailing out with no errors")
+		return nil
+	}
+	// First get metas from all repos (and decodes trees)
+
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
 	// First match packages against repositories by priority
 	//	matches := syncedRepos.PackageMatches(p)
 
 	// compute a "big" world
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
-
-	solv := solver.NewSolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false))
-	solution, err := solv.Install(p)
-	if err != nil {
-		return errors.Wrap(err, "Failed solving solution for package")
-	}
-	// Gathers things to install
+	p = syncedRepos.ResolveSelectors(p)
 	toInstall := map[string]ArtifactMatch{}
-	for _, assertion := range solution {
-		if assertion.Value {
-			matches := syncedRepos.PackageMatches([]pkg.Package{assertion.Package})
-			if len(matches) != 1 {
-				return errors.New("Failed matching solutions against repository - where are definitions coming from?!")
-			}
-		A:
-			for _, artefact := range matches[0].Repo.GetIndex() {
-				if artefact.GetCompileSpec().GetPackage() == nil {
-					return errors.New("Package in compilespec empty")
+	var packagesToInstall []pkg.Package
 
-				}
-				if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
-					// Filter out already installed
-					if _, err := s.Database.FindPackage(assertion.Package); err != nil {
-						toInstall[assertion.Package.GetFingerPrint()] = ArtifactMatch{Package: assertion.Package, Artifact: artefact, Repository: matches[0].Repo}
-					}
-					break A
-				}
+	var solution solver.PackagesAssertions
+
+	if !l.Options.NoDeps {
+		solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		solution, err = solv.Install(p)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed solving solution for package")
+		}
+		// Gathers things to install
+		for _, assertion := range solution {
+			if assertion.Value {
+				packagesToInstall = append(packagesToInstall, assertion.Package)
 			}
 		}
+	} else if !l.Options.OnlyDeps {
+		for _, currentPack := range p {
+			packagesToInstall = append(packagesToInstall, currentPack)
+		}
 	}
 
+	// Gathers things to install
+	for _, currentPack := range packagesToInstall {
+		matches := syncedRepos.PackageMatches([]pkg.Package{currentPack})
+		if len(matches) == 0 {
+			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
+		}
+	A:
+		for _, artefact := range matches[0].Repo.GetIndex() {
+			if artefact.GetCompileSpec().GetPackage() == nil {
+				return errors.New("Package in compilespec empty")
+
+			}
+			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+				// Filter out already installed
+				if _, err := s.Database.FindPackage(currentPack); err != nil {
+					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
+				}
+				break A
+			}
+		}
+	}
 	// Install packages into rootfs in parallel.
 	all := make(chan ArtifactMatch)
 
 	var wg = new(sync.WaitGroup)
-	for i := 0; i < l.Concurrency; i++ {
+	for i := 0; i < l.Options.Concurrency; i++ {
 		wg.Add(1)
 		go l.installerWorker(i, wg, all, s)
 	}
@@ -214,40 +258,39 @@ func (l *LuetInstaller) Install(p []pkg.Package, s *System) error {
 		for _, ass := range ordered {
 			if ass.Value {
 				// Annotate to the system that the package was installed
-				// TODO: Annotate also files that belong to the package, somewhere to uninstall
 				if _, err := s.Database.FindPackage(ass.Package); err == nil {
 					err := s.Database.UpdatePackage(ass.Package)
-					if err != nil {
+					if err != nil && !l.Options.Force {
 						return errors.Wrap(err, "Failed updating package")
 					}
 				} else {
 					_, err := s.Database.CreatePackage(ass.Package)
-					if err != nil {
+					if err != nil && !l.Options.Force {
 						return errors.Wrap(err, "Failed creating package")
 					}
 				}
 				installed, ok := toInstall[ass.Package.GetFingerPrint()]
 				if !ok {
-					return errors.New("Couldn't find ArtifactMatch for " + ass.Package.GetFingerPrint())
+					return errors.New("Couldn't find ArtifactMatch for " + ass.Package.HumanReadableString())
 				}
 
 				treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
 				if err != nil {
-					return errors.Wrap(err, "Error getting package "+ass.Package.GetFingerPrint())
+					return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
 				}
 				if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
-					Info("Executing finalizer for " + ass.Package.GetName())
+					Info("Executing finalizer for " + ass.Package.HumanReadableString())
 					finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
-					if err != nil {
+					if err != nil && !l.Options.Force {
 						return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
 					}
 					if _, exists := executedFinalizer[ass.Package.GetFingerPrint()]; !exists {
 						finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
-						if err != nil {
+						if err != nil && !l.Options.Force {
 							return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
 						}
 						err = finalizer.RunInstall()
-						if err != nil {
+						if err != nil && !l.Options.Force {
 							return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
 						}
 						executedFinalizer[ass.Package.GetFingerPrint()] = true
@@ -257,47 +300,29 @@ func (l *LuetInstaller) Install(p []pkg.Package, s *System) error {
 		}
 
 	}
-
 	return nil
 
 }
 
 func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
 
-	// FIXME: Implement
 	artifact, err := a.Repository.Client().DownloadArtifact(a.Artifact)
-	defer os.Remove(artifact.GetPath())
-
-	tarFile, err := os.Open(artifact.GetPath())
 	if err != nil {
+		return errors.Wrap(err, "Error on download artifact")
+	}
+
+	err = artifact.Verify()
+	if err != nil && !l.Options.Force {
+		return errors.Wrap(err, "Artifact integrity check failure")
+	}
+
+	files, err := artifact.FileList()
+	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Could not open package archive")
 	}
-	defer tarFile.Close()
-	tr := tar.NewReader(tarFile)
 
-	var files []string
-	// untar each segment
-	for {
-		hdr, err := tr.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return err
-		}
-		// determine proper file path info
-		finfo := hdr.FileInfo()
-		fileName := hdr.Name
-		if finfo.Mode().IsDir() {
-			continue
-		}
-		files = append(files, fileName)
-
-		// if a dir, create it, then go to next segment
-	}
-
-	err = helpers.Untar(artifact.GetPath(), s.Target, true)
-	if err != nil {
+	err = artifact.Unpack(s.Target, true)
+	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Error met while unpacking rootfs")
 	}
 
@@ -312,11 +337,16 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
 		err := l.installPackage(p, s)
-		if err != nil {
+		if err != nil && !l.Options.Force {
 			//TODO: Uninstall, rollback.
 			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
 			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
 		}
+		if err == nil {
+			Info(":package: ", p.Package.HumanReadableString(), "installed")
+		} else if err != nil && l.Options.Force {
+			Info(":package: ", p.Package.HumanReadableString(), "installed with failures (force install)")
+		}
 	}
 
 	return nil
@@ -354,17 +384,26 @@ func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
 	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) - mark the uninstallation in db
 	// Get installed definition
 
-	solv := solver.NewSolver(s.Database, s.Database, pkg.NewInMemoryDatabase(false))
-	solution, err := solv.Uninstall(p)
-	if err != nil {
-		return errors.Wrap(err, "Uninstall failed")
-	}
-	for _, p := range solution {
-		Info("Uninstalling", p.GetFingerPrint())
+	if !l.Options.NoDeps {
+		solv := solver.NewResolver(s.Database, s.Database, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		solution, err := solv.Uninstall(p)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+		}
+		for _, p := range solution {
+			Info("Uninstalling", p.HumanReadableString())
+			err := l.uninstall(p, s)
+			if err != nil && !l.Options.Force {
+				return errors.Wrap(err, "Uninstall failed")
+			}
+		}
+	} else {
+		Info("Uninstalling", p.HumanReadableString(), "without deps")
 		err := l.uninstall(p, s)
-		if err != nil {
+		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Uninstall failed")
 		}
+		Info(":package: ", p.HumanReadableString(), "uninstalled")
 	}
 	return nil
 

pkg/installer/installer_suite_test.go

@@ -18,11 +18,19 @@ package installer_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestInstaller(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	// Set temporary directory for rootfs
+	config.LuetCfg.GetSystem().Rootfs = "/tmp/luet-root"
+	// Force dynamic path for packages cache
+	config.LuetCfg.GetSystem().PkgsCachePath = ""
 	RunSpecs(t, "Installer Suite")
 }

pkg/installer/installer_test.go

@@ -34,7 +34,7 @@ import (
 var _ = Describe("Installer", func() {
 	Context("Writes a repository definition", func() {
 		It("Writes a repo and can install packages from it", func() {
-			//repo:=NewLuetRepository()
+			//repo:=NewLuetSystemRepository()
 
 			tmpdir, err := ioutil.TempDir("", "tree")
 			Expect(err).ToNot(HaveOccurred())
@@ -47,7 +47,7 @@ var _ = Describe("Installer", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -62,7 +62,9 @@ var _ = Describe("Installer", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			artifact, err := compiler.Compile(2, false, spec)
+			compiler.SetConcurrency(2)
+
+			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -80,34 +82,35 @@ var _ = Describe("Installer", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", tmpdir, "local", 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir)
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(fakeroot) // clean up
 
-			inst := NewLuetInstaller(1)
-			repo2, err := NewLuetRepositoryFromYaml([]byte(`
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
 name: "test"
-type: "local"
-uri: "`+tmpdir+`"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
 `), pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 
 			inst.Repositories(Repositories{repo2})
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 			systemDB := pkg.NewInMemoryDatabase(false)
 			system := &System{Database: systemDB, Target: fakeroot}
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
@@ -144,7 +147,7 @@ uri: "`+tmpdir+`"
 
 	Context("Installation", func() {
 		It("Installs in a system with a persistent db", func() {
-			//repo:=NewLuetRepository()
+			//repo:=NewLuetSystemRepository()
 
 			tmpdir, err := ioutil.TempDir("", "tree")
 			Expect(err).ToNot(HaveOccurred())
@@ -157,7 +160,7 @@ uri: "`+tmpdir+`"
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -172,7 +175,9 @@ uri: "`+tmpdir+`"
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			artifact, err := compiler.Compile(2, false, spec)
+			compiler.SetConcurrency(2)
+
+			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -190,34 +195,35 @@ uri: "`+tmpdir+`"
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", tmpdir, "local", 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir)
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(fakeroot) // clean up
 
-			inst := NewLuetInstaller(1)
-			repo2, err := NewLuetRepositoryFromYaml([]byte(`
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
 name: "test"
-type: "local"
-uri: "`+tmpdir+`"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
 `), pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 
 			inst.Repositories(Repositories{repo2})
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 
 			bolt, err := ioutil.TempDir("", "db")
 			Expect(err).ToNot(HaveOccurred())
@@ -260,7 +266,7 @@ uri: "`+tmpdir+`"
 
 	Context("Simple upgrades", func() {
 		It("Installs packages and Upgrades a system with a persistent db", func() {
-			//repo:=NewLuetRepository()
+			//repo:=NewLuetSystemRepository()
 
 			tmpdir, err := ioutil.TempDir("", "tree")
 			Expect(err).ToNot(HaveOccurred())
@@ -273,7 +279,7 @@ uri: "`+tmpdir+`"
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -291,38 +297,156 @@ uri: "`+tmpdir+`"
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			_, errs := c.CompileParallel(2, false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+			c.SetConcurrency(2)
+
+			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", tmpdir, "local", 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir)
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(fakeroot) // clean up
 
-			inst := NewLuetInstaller(1)
-			repo2, err := NewLuetRepositoryFromYaml([]byte(`
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
 name: "test"
-type: "local"
-uri: "`+tmpdir+`"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
 `), pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 
 			inst.Repositories(Repositories{repo2})
-			Expect(repo.GetUri()).To(Equal(tmpdir))
-			Expect(repo.GetType()).To(Equal("local"))
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.GetPackages())).To(Equal(1))
+			p, err := system.Database.GetPackage(system.Database.GetPackages()[0])
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetName()).To(Equal("b"))
+
+			files, err := systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(files).To(Equal([]string{"artifact42", "test5", "test6"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = inst.Upgrade(system)
+			Expect(err).ToNot(HaveOccurred())
+
+			// Nothing should be there anymore (files, packagedb entry)
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+
+			// New version - new files
+			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+
+			// New package should be there
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+		})
+
+	})
+
+	Context("Compressed packages", func() {
+		It("Installs", func() {
+			//repo:=NewLuetSystemRepository()
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/upgrade")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec3, err := c.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+			spec3.SetOutputPath(tmpdir)
+			c.SetConcurrency(2)
+			c.SetCompressionType(compiler.GZip)
+			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+
+			Expect(errs).To(BeEmpty())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+
+			inst.Repositories(Repositories{repo2})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
 
 			bolt, err := ioutil.TempDir("", "db")
 			Expect(err).ToNot(HaveOccurred())

pkg/installer/interface.go

@@ -27,6 +27,7 @@ type Installer interface {
 	Uninstall(pkg.Package, *System) error
 	Upgrade(s *System) error
 	Repositories([]Repository)
+	SyncRepositories(bool) (Repositories, error)
 }
 
 type Client interface {
@@ -38,17 +39,30 @@ type Repositories []Repository
 
 type Repository interface {
 	GetName() string
-	GetUri() string
-	SetUri(string)
+	GetDescription() string
+	GetUrls() []string
+	SetUrls([]string)
+	AddUrl(string)
 	GetPriority() int
 	GetIndex() compiler.ArtifactIndex
 	GetTree() tree.Builder
 	SetTree(tree.Builder)
-	Write(path string) error
-	Sync() (Repository, error)
+	Write(path string, resetRevision bool) error
+	Sync(bool) (Repository, error)
 	GetTreePath() string
 	SetTreePath(string)
 	GetType() string
 	SetType(string)
+	SetAuthentication(map[string]string)
+	GetAuthentication() map[string]string
+	GetRevision() int
+	IncrementRevision()
+	GetLastUpdate() string
+	SetLastUpdate(string)
 	Client() Client
+
+	GetTreeChecksums() compiler.Checksums
+	GetTreeCompressionType() compiler.CompressionImplementation
+	SetTreeCompressionType(c compiler.CompressionImplementation)
+	SetTreeChecksums(c compiler.Checksums)
 }

pkg/installer/repository.go

@@ -16,42 +16,60 @@
 package installer
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
+	"time"
 
-	"github.com/mudler/luet/pkg/installer/client"
-
-	"github.com/ghodss/yaml"
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/installer/client"
+	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/ghodss/yaml"
+	. "github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 )
 
-type LuetRepository struct {
-	Name     string                 `json:"name"`
-	Uri      string                 `json:"uri"`
-	Priority int                    `json:"priority"`
-	Index    compiler.ArtifactIndex `json:"index"`
-	Tree     tree.Builder           `json:"-"`
-	TreePath string                 `json:"-"`
-	Type     string                 `json:"type"`
+const (
+	REPOSITORY_SPECFILE = "repository.yaml"
+	TREE_TARBALL        = "tree.tar"
+)
+
+type LuetSystemRepository struct {
+	*config.LuetRepository
+
+	Index               compiler.ArtifactIndex             `json:"index"`
+	Tree                tree.Builder                       `json:"-"`
+	TreePath            string                             `json:"treepath"`
+	TreeCompressionType compiler.CompressionImplementation `json:"treecompressiontype"`
+	TreeChecksums       compiler.Checksums                 `json:"treechecksums"`
 }
 
-type LuetRepositorySerialized struct {
-	Name     string                      `json:"name"`
-	Uri      string                      `json:"uri"`
-	Priority int                         `json:"priority"`
-	Index    []*compiler.PackageArtifact `json:"index"`
-	Type     string                      `json:"type"`
+type LuetSystemRepositorySerialized struct {
+	Name                string                             `json:"name"`
+	Description         string                             `json:"description,omitempty"`
+	Urls                []string                           `json:"urls"`
+	Priority            int                                `json:"priority"`
+	Index               []*compiler.PackageArtifact        `json:"index"`
+	Type                string                             `json:"type"`
+	Revision            int                                `json:"revision,omitempty"`
+	LastUpdate          string                             `json:"last_update,omitempty"`
+	TreePath            string                             `json:"treepath"`
+	TreeCompressionType compiler.CompressionImplementation `json:"treecompressiontype"`
+	TreeChecksums       compiler.Checksums                 `json:"treechecksums"`
 }
 
-func GenerateRepository(name, uri, t string, priority int, src, treeDir string, db pkg.PackageDatabase) (Repository, error) {
+func GenerateRepository(name, descr, t string, urls []string, priority int, src, treeDir string, db pkg.PackageDatabase) (Repository, error) {
 
 	art, err := buildPackageIndex(src)
 	if err != nil {
@@ -63,24 +81,51 @@ func GenerateRepository(name, uri, t string, priority int, src, treeDir string,
 		return nil, err
 	}
 
-	return NewLuetRepository(name, uri, t, priority, art, tr), nil
+	return NewLuetSystemRepository(
+		config.NewLuetRepository(name, t, descr, urls, priority, true, false),
+		art, tr), nil
 }
 
-func NewLuetRepository(name, uri, t string, priority int, art []compiler.Artifact, builder tree.Builder) Repository {
-	return &LuetRepository{Index: art, Type: t, Tree: builder, Name: name, Uri: uri, Priority: priority}
+func NewSystemRepository(repo config.LuetRepository) Repository {
+	return &LuetSystemRepository{
+		LuetRepository: &repo,
+	}
 }
 
-func NewLuetRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, error) {
-	var p *LuetRepositorySerialized
-	r := &LuetRepository{}
+func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder) Repository {
+	return &LuetSystemRepository{
+		LuetRepository: repo,
+		Index:          art,
+		Tree:           builder,
+	}
+}
+
+func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, error) {
+	var p *LuetSystemRepositorySerialized
 	err := yaml.Unmarshal(data, &p)
 	if err != nil {
 		return nil, err
 	}
-	r.Name = p.Name
-	r.Uri = p.Uri
-	r.Priority = p.Priority
-	r.Type = p.Type
+	r := &LuetSystemRepository{
+		LuetRepository: config.NewLuetRepository(
+			p.Name,
+			p.Type,
+			p.Description,
+			p.Urls,
+			p.Priority,
+			true,
+			false,
+		),
+		TreeCompressionType: p.TreeCompressionType,
+		TreeChecksums:       p.TreeChecksums,
+		TreePath:            p.TreePath,
+	}
+	if p.Revision > 0 {
+		r.Revision = p.Revision
+	}
+	if p.LastUpdate != "" {
+		r.LastUpdate = p.LastUpdate
+	}
 	i := compiler.ArtifactIndex{}
 	for _, ii := range p.Index {
 		i = append(i, ii)
@@ -122,56 +167,129 @@ func buildPackageIndex(path string) ([]compiler.Artifact, error) {
 	return art, nil
 }
 
-func (r *LuetRepository) GetName() string {
-	return r.Name
+func (r *LuetSystemRepository) GetName() string {
+	return r.LuetRepository.Name
 }
-func (r *LuetRepository) GetTreePath() string {
+func (r *LuetSystemRepository) GetDescription() string {
+	return r.LuetRepository.Description
+}
+
+func (r *LuetSystemRepository) GetAuthentication() map[string]string {
+	return r.LuetRepository.Authentication
+}
+
+func (r *LuetSystemRepository) GetTreeCompressionType() compiler.CompressionImplementation {
+	return r.TreeCompressionType
+}
+
+func (r *LuetSystemRepository) GetTreeChecksums() compiler.Checksums {
+	return r.TreeChecksums
+}
+
+func (r *LuetSystemRepository) SetTreeCompressionType(c compiler.CompressionImplementation) {
+	r.TreeCompressionType = c
+}
+
+func (r *LuetSystemRepository) SetTreeChecksums(c compiler.Checksums) {
+	r.TreeChecksums = c
+}
+
+func (r *LuetSystemRepository) GetType() string {
+	return r.LuetRepository.Type
+}
+func (r *LuetSystemRepository) SetType(p string) {
+	r.LuetRepository.Type = p
+}
+func (r *LuetSystemRepository) AddUrl(p string) {
+	r.LuetRepository.Urls = append(r.LuetRepository.Urls, p)
+}
+func (r *LuetSystemRepository) GetUrls() []string {
+	return r.LuetRepository.Urls
+}
+func (r *LuetSystemRepository) SetUrls(urls []string) {
+	r.LuetRepository.Urls = urls
+}
+func (r *LuetSystemRepository) GetPriority() int {
+	return r.LuetRepository.Priority
+}
+func (r *LuetSystemRepository) GetTreePath() string {
 	return r.TreePath
 }
-func (r *LuetRepository) SetTreePath(p string) {
+func (r *LuetSystemRepository) SetTreePath(p string) {
 	r.TreePath = p
 }
-
-func (r *LuetRepository) SetTree(b tree.Builder) {
+func (r *LuetSystemRepository) SetTree(b tree.Builder) {
 	r.Tree = b
 }
-
-func (r *LuetRepository) GetType() string {
-	return r.Type
-}
-func (r *LuetRepository) SetType(p string) {
-	r.Type = p
-}
-
-func (r *LuetRepository) SetUri(p string) {
-	r.Uri = p
-}
-func (r *LuetRepository) GetUri() string {
-	return r.Uri
-}
-func (r *LuetRepository) GetPriority() int {
-	return r.Priority
-}
-func (r *LuetRepository) GetIndex() compiler.ArtifactIndex {
+func (r *LuetSystemRepository) GetIndex() compiler.ArtifactIndex {
 	return r.Index
 }
-func (r *LuetRepository) GetTree() tree.Builder {
+func (r *LuetSystemRepository) GetTree() tree.Builder {
 	return r.Tree
 }
+func (r *LuetSystemRepository) GetRevision() int {
+	return r.LuetRepository.Revision
+}
+func (r *LuetSystemRepository) GetLastUpdate() string {
+	return r.LuetRepository.LastUpdate
+}
+func (r *LuetSystemRepository) SetLastUpdate(u string) {
+	r.LuetRepository.LastUpdate = u
+}
+func (r *LuetSystemRepository) IncrementRevision() {
+	r.LuetRepository.Revision++
+}
 
-func (r *LuetRepository) Write(dst string) error {
+func (r *LuetSystemRepository) SetAuthentication(auth map[string]string) {
+	r.LuetRepository.Authentication = auth
+}
 
-	os.MkdirAll(dst, os.ModePerm)
+func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repository, error) {
+	dat, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading file "+file)
+	}
+	if removeFile {
+		defer os.Remove(file)
+	}
+
+	var repo Repository
+	repo, err = NewLuetSystemRepositoryFromYaml(dat, pkg.NewInMemoryDatabase(false))
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading repository from file "+file)
+	}
+
+	return repo, err
+}
+
+func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
+	err := os.MkdirAll(dst, os.ModePerm)
+	if err != nil {
+		return err
+	}
 	r.Index = r.Index.CleanPath()
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
 
-	data, err := yaml.Marshal(r)
-	if err != nil {
-		return err
-	}
-	err = ioutil.WriteFile(filepath.Join(dst, "repository.yaml"), data, os.ModePerm)
-	if err != nil {
-		return err
+	repospec := filepath.Join(dst, REPOSITORY_SPECFILE)
+	if resetRevision {
+		r.Revision = 0
+	} else {
+		if _, err := os.Stat(repospec); !os.IsNotExist(err) {
+			// Read existing file for retrieve revision
+			spec, err := r.ReadSpecFile(repospec, false)
+			if err != nil {
+				return err
+			}
+			r.Revision = spec.GetRevision()
+		}
 	}
+	r.Revision++
+
+	Info(fmt.Sprintf(
+		"For repository %s creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
 	archive, err := ioutil.TempDir(os.TempDir(), "archive")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for archive")
@@ -181,67 +299,148 @@ func (r *LuetRepository) Write(dst string) error {
 	if err != nil {
 		return errors.Wrap(err, "Error met while saving the tree")
 	}
-	err = helpers.Tar(archive, filepath.Join(dst, "tree.tar"))
+	tpath := r.GetTreePath()
+	if tpath == "" {
+		tpath = TREE_TARBALL
+	}
+
+	a := compiler.NewPackageArtifact(filepath.Join(dst, tpath))
+	a.SetCompressionType(r.TreeCompressionType)
+	err = a.Compress(archive, 1)
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating package archive")
 	}
-	return nil
-}
 
-func (r *LuetRepository) Client() Client {
-	switch r.GetType() {
-	case "local":
-		return client.NewLocalClient(client.RepoData{Uri: r.GetUri()})
-	case "http":
-		return client.NewHttpClient(client.RepoData{Uri: r.GetUri()})
+	r.TreePath = path.Base(a.GetPath())
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	r.TreeChecksums = a.GetChecksums()
+
+	data, err := yaml.Marshal(r)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(repospec, data, os.ModePerm)
+	if err != nil {
+		return err
 	}
 
 	return nil
 }
-func (r *LuetRepository) Sync() (Repository, error) {
+
+func (r *LuetSystemRepository) Client() Client {
+	switch r.GetType() {
+	case "disk":
+		return client.NewLocalClient(client.RepoData{Urls: r.GetUrls()})
+	case "http":
+		return client.NewHttpClient(
+			client.RepoData{
+				Urls:           r.GetUrls(),
+				Authentication: r.GetAuthentication(),
+			})
+	}
+
+	return nil
+}
+func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
+	var repoUpdated bool = false
+	var treefs string
+
+	Debug("Sync of the repository", r.Name, "in progress...")
 	c := r.Client()
 	if c == nil {
 		return nil, errors.New("No client could be generated from repository.")
 	}
-	file, err := c.DownloadFile("repository.yaml")
-	if err != nil {
-		return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri())
-	}
-	dat, err := ioutil.ReadFile(file)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error reading file "+file)
-	}
-	defer os.Remove(file)
 
-	// TODO: make it swappable
-	repo, err := NewLuetRepositoryFromYaml(dat, pkg.NewInMemoryDatabase(false))
+	// Retrieve remote repository.yaml for retrieve revision and date
+	file, err := c.DownloadFile(REPOSITORY_SPECFILE)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error reading repository from file "+file)
-
+		return nil, errors.Wrap(err, "While downloading "+REPOSITORY_SPECFILE)
 	}
 
-	archivetree, err := c.DownloadFile("tree.tar")
+	repobasedir := config.LuetCfg.GetSystem().GetRepoDatabaseDirPath(r.GetName())
+	repo, err := r.ReadSpecFile(file, false)
 	if err != nil {
-		return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri())
+		return nil, err
 	}
-	defer os.RemoveAll(archivetree) // clean up
+	// Remove temporary file that contains repository.html.
+	// Example: /tmp/HttpClient236052003
+	defer os.RemoveAll(file)
 
-	treefs, err := ioutil.TempDir(os.TempDir(), "treefs")
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	if r.Cached {
+		if !force {
+			localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir, REPOSITORY_SPECFILE), false)
+			if localRepo != nil {
+				if localRepo.GetRevision() == repo.GetRevision() &&
+					localRepo.GetLastUpdate() == repo.GetLastUpdate() {
+					repoUpdated = true
+				}
+			}
+		}
+		if r.GetTreePath() == "" {
+			treefs = filepath.Join(repobasedir, "treefs")
+		} else {
+			treefs = r.GetTreePath()
+		}
+
+	} else {
+		treefs, err = ioutil.TempDir(os.TempDir(), "treefs")
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
 	}
-	//defer os.RemoveAll(treefs) // clean up
 
-	// TODO: Following as option if archive as output?
-	// archive, err := ioutil.TempDir(os.TempDir(), "archive")
-	// if err != nil {
-	// 	return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	// }
-	// defer os.RemoveAll(archive) // clean up
+	if !repoUpdated {
+		tpath := repo.GetTreePath()
+		if tpath == "" {
+			tpath = TREE_TARBALL
+		}
+		a := compiler.NewPackageArtifact(tpath)
 
-	err = helpers.Untar(archivetree, treefs, false)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while unpacking rootfs")
+		artifact, err := c.DownloadArtifact(a)
+		if err != nil {
+			return nil, errors.Wrap(err, "While downloading "+tpath)
+		}
+		defer os.Remove(artifact.GetPath())
+
+		artifact.SetChecksums(repo.GetTreeChecksums())
+		artifact.SetCompressionType(repo.GetTreeCompressionType())
+
+		err = artifact.Verify()
+		if err != nil {
+			return nil, errors.Wrap(err, "Tree integrity check failure")
+		}
+
+		Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
+
+		if r.Cached {
+			// Copy updated repository.yaml file to repo dir now that the tree is synced.
+			err = helpers.CopyFile(file, filepath.Join(repobasedir, REPOSITORY_SPECFILE))
+			if err != nil {
+				return nil, errors.Wrap(err, "Error on update "+REPOSITORY_SPECFILE)
+			}
+			// Remove previous tree
+			os.RemoveAll(treefs)
+		}
+		Debug("Decompress tree of the repository " + r.Name + "...")
+
+		err = artifact.Unpack(treefs, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while unpacking tree")
+		}
+
+		tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64)
+
+		InfoC(
+			Bold(Red(":house: Repository "+r.GetName()+" revision: ")).String() +
+				Bold(Green(repo.GetRevision())).String() + " - " +
+				Bold(Green(time.Unix(tsec, 0).String())).String(),
+		)
+
+	} else {
+		Info("Repository", r.GetName(), "is already up to date.")
 	}
 
 	reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
@@ -249,9 +448,11 @@ func (r *LuetRepository) Sync() (Repository, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while unpacking rootfs")
 	}
+
 	repo.SetTree(reciper)
 	repo.SetTreePath(treefs)
-	repo.SetUri(r.GetUri())
+	repo.SetUrls(r.GetUrls())
+	repo.SetAuthentication(r.GetAuthentication())
 
 	return repo, nil
 }
@@ -323,6 +524,36 @@ PACKAGE:
 
 }
 
+func (re Repositories) ResolveSelectors(p []pkg.Package) []pkg.Package {
+	// If a selector is given, get the best from each repo
+	sort.Sort(re) // respect prio
+	var matches []pkg.Package
+PACKAGE:
+	for _, pack := range p {
+	REPOSITORY:
+		for _, r := range re {
+			if pack.IsSelector() {
+				c, err := r.GetTree().GetDatabase().FindPackageCandidate(pack)
+				// If FindPackageCandidate returns the same package, it means it couldn't find one.
+				// Skip this repository and keep looking.
+				if c.String() == pack.String() {
+					continue REPOSITORY
+				}
+				if err == nil {
+					matches = append(matches, c)
+					continue PACKAGE
+				}
+			} else {
+				// If it's not a selector, just append it
+				matches = append(matches, pack)
+			}
+		}
+	}
+
+	return matches
+
+}
+
 func (re Repositories) Search(s string) []PackageMatch {
 	sort.Sort(re)
 	var term = regexp.MustCompile(s)

pkg/installer/repository_test.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
+	config "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -46,7 +47,7 @@ var _ = Describe("Repository", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -61,7 +62,9 @@ var _ = Describe("Repository", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			artifact, err := compiler.Compile(2, false, spec)
+			compiler.SetConcurrency(1)
+
+			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -79,16 +82,16 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", tmpdir, "local", 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir)
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("tree.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
 		})
 	})
 	Context("Matching packages", func() {
@@ -103,8 +106,8 @@ var _ = Describe("Repository", func() {
 
 			_, err = builder2.GetDatabase().CreatePackage(package2)
 			Expect(err).ToNot(HaveOccurred())
-			repo1 := &LuetRepository{Name: "test1", Tree: builder1}
-			repo2 := &LuetRepository{Name: "test2", Tree: builder2}
+			repo1 := &LuetSystemRepository{LuetRepository: &config.LuetRepository{Name: "test1"}, Tree: builder1}
+			repo2 := &LuetSystemRepository{LuetRepository: &config.LuetRepository{Name: "test2"}, Tree: builder2}
 			repositories := Repositories{repo1, repo2}
 			matches := repositories.PackageMatches([]pkg.Package{package1})
 			Expect(matches).To(Equal([]PackageMatch{{Repo: repo1, Package: package1}}))

pkg/logger/logger.go

@@ -3,17 +3,54 @@ package logger
 import (
 	"fmt"
 	"os"
-	"time"
+
+	. "github.com/mudler/luet/pkg/config"
 
 	"github.com/briandowns/spinner"
 	"github.com/kyokomi/emoji"
 	. "github.com/logrusorgru/aurora"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
-var s *spinner.Spinner = spinner.New(spinner.CharSets[22], 100*time.Millisecond)
+var s *spinner.Spinner = nil
+var z *zap.Logger = nil
 
-// TODO: handle this from configuration
-var debug = false
+func NewSpinner() {
+	if s == nil {
+		s = spinner.New(
+			spinner.CharSets[LuetCfg.GetGeneral().SpinnerCharset],
+			LuetCfg.GetGeneral().GetSpinnerMs())
+	}
+}
+
+func ZapLogger() error {
+	var err error
+	if z == nil {
+		// TODO: test permission for open logfile.
+		cfg := zap.NewProductionConfig()
+		cfg.OutputPaths = []string{LuetCfg.GetLogging().Path}
+		cfg.Level = level2AtomicLevel(LuetCfg.GetLogging().Level)
+		cfg.ErrorOutputPaths = []string{}
+		if LuetCfg.GetLogging().JsonFormat {
+			cfg.Encoding = "json"
+		} else {
+			cfg.Encoding = "console"
+		}
+		cfg.DisableCaller = true
+		cfg.DisableStacktrace = true
+		cfg.EncoderConfig.TimeKey = "time"
+		cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+
+		z, err = cfg.Build()
+		if err != nil {
+			fmt.Fprint(os.Stderr, "Error on initialize file logger: "+err.Error()+"\n")
+			return err
+		}
+	}
+
+	return nil
+}
 
 func Spinner(i int) {
 
@@ -21,7 +58,7 @@ func Spinner(i int) {
 		i = 43
 	}
 
-	if !debug && !s.Active() {
+	if !LuetCfg.GetGeneral().Debug && !s.Active() {
 		//	s.UpdateCharSet(spinner.CharSets[i])
 		s.Start() // Start the spinner
 	}
@@ -30,7 +67,7 @@ func Spinner(i int) {
 func SpinnerText(suffix, prefix string) {
 	s.Lock()
 	defer s.Unlock()
-	if debug {
+	if LuetCfg.GetGeneral().Debug {
 		fmt.Println(fmt.Sprintf("%s %s",
 			Bold(Cyan(prefix)).String(),
 			Bold(Magenta(suffix)).BgBlack().String(),
@@ -42,59 +79,119 @@ func SpinnerText(suffix, prefix string) {
 }
 
 func SpinnerStop() {
-	if !debug {
+	if !LuetCfg.GetGeneral().Debug {
 		s.Stop()
 	}
 }
 
-func msg(level string, msg ...interface{}) {
+func level2Number(level string) int {
+	switch level {
+	case "error":
+		return 0
+	case "warning":
+		return 1
+	case "info":
+		return 2
+	default:
+		return 3
+	}
+}
+
+func log2File(level, msg string) {
+	switch level {
+	case "error":
+		z.Error(msg)
+	case "warning":
+		z.Warn(msg)
+	case "info":
+		z.Info(msg)
+	default:
+		z.Debug(msg)
+	}
+}
+
+func level2AtomicLevel(level string) zap.AtomicLevel {
+	switch level {
+	case "error":
+		return zap.NewAtomicLevelAt(zap.ErrorLevel)
+	case "warning":
+		return zap.NewAtomicLevelAt(zap.WarnLevel)
+	case "info":
+		return zap.NewAtomicLevelAt(zap.InfoLevel)
+	default:
+		return zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+}
+
+func msg(level string, withoutColor bool, msg ...interface{}) {
 	var message string
+	var confLevel, msgLevel int
+
+	if LuetCfg.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = level2Number(LuetCfg.GetLogging().Level)
+	}
+	msgLevel = level2Number(level)
+	if msgLevel > confLevel {
+		return
+	}
+
 	for _, m := range msg {
 		message += " " + fmt.Sprintf("%v", m)
 	}
 
 	var levelMsg string
-	switch level {
-	case "warning":
-		levelMsg = Bold(Yellow(":construction: " + message)).BgBlack().String()
-	case "debug":
-		levelMsg = White(message).BgBlack().String()
-	case "info":
-		levelMsg = Bold(White(message)).BgBlack().String()
-	case "error":
-		levelMsg = Bold(Red(":bomb: " + message + ":fire:")).BgBlack().String()
+
+	if withoutColor {
+		levelMsg = message
+	} else {
+		switch level {
+		case "warning":
+			levelMsg = Bold(Yellow(":construction: " + message)).BgBlack().String()
+		case "debug":
+			levelMsg = White(message).BgBlack().String()
+		case "info":
+			levelMsg = Bold(White(message)).BgBlack().String()
+		case "error":
+			levelMsg = Bold(Red(":bomb: " + message + ":fire:")).BgBlack().String()
+		}
 	}
 
 	levelMsg = emoji.Sprint(levelMsg)
 
-	// if s.Active() {
-	// 	SpinnerText(levelMsg, "")
-	// 	return
-	// }
-
-	cmd := []interface{}{}
-	for _, f := range msg {
-		cmd = append(cmd, f)
+	if z != nil {
+		log2File(level, message)
 	}
 
 	fmt.Println(levelMsg)
-	//fmt.Println(cmd...)
 }
 
 func Warning(mess ...interface{}) {
-	msg("warning", mess...)
+	msg("warning", false, mess...)
+	if LuetCfg.GetGeneral().FatalWarns {
+		os.Exit(2)
+	}
 }
 
 func Debug(mess ...interface{}) {
-	msg("debug", mess...)
+	msg("debug", false, mess...)
+}
+
+func DebugC(mess ...interface{}) {
+	msg("debug", true, mess...)
 }
 
 func Info(mess ...interface{}) {
-	msg("info", mess...)
+	msg("info", false, mess...)
+}
+
+func InfoC(mess ...interface{}) {
+	msg("info", true, mess...)
 }
 
 func Error(mess ...interface{}) {
-	msg("error", mess...)
+	msg("error", false, mess...)
 }
 
 func Fatal(mess ...interface{}) {

pkg/package/database_boltdb.go

@@ -22,7 +22,6 @@ import (
 	"sync"
 	"time"
 
-	version "github.com/hashicorp/go-version"
 	"github.com/pkg/errors"
 
 	storm "github.com/asdine/storm"
@@ -233,16 +232,11 @@ func (db *BoltDatabase) getProvide(p Package) (Package, error) {
 
 		for ve, _ := range versions {
 
-			v, err := version.NewVersion(p.GetVersion())
+			match, err := p.VersionMatchSelector(ve)
 			if err != nil {
-				return nil, err
+				return nil, errors.Wrap(err, "Error on match version")
 			}
-			constraints, err := version.NewConstraint(ve)
-			if err != nil {
-				return nil, err
-			}
-
-			if constraints.Check(v) {
+			if match {
 				pa, ok := db.ProvidesDatabase[p.GetPackageName()][ve]
 				if !ok {
 					return nil, errors.New("No versions found for package")
@@ -367,15 +361,11 @@ func (db *BoltDatabase) FindPackages(p Package) ([]Package, error) {
 			continue
 		}
 
-		v, err := version.NewVersion(w.GetVersion())
+		match, err := p.SelectorMatchVersion(w.GetVersion())
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Error on match selector")
 		}
-		constraints, err := version.NewConstraint(p.GetVersion())
-		if err != nil {
-			return nil, err
-		}
-		if constraints.Check(v) {
+		if match {
 			versionsInWorld = append(versionsInWorld, w)
 		}
 	}

pkg/package/database_mem.go

@@ -20,7 +20,6 @@ import (
 	"encoding/json"
 	"sync"
 
-	version "github.com/hashicorp/go-version"
 	"github.com/pkg/errors"
 )
 
@@ -178,16 +177,11 @@ func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
 
 		for ve, _ := range versions {
 
-			v, err := version.NewVersion(p.GetVersion())
+			match, err := p.VersionMatchSelector(ve)
 			if err != nil {
-				return nil, err
+				return nil, errors.Wrap(err, "Error on match version")
 			}
-			constraints, err := version.NewConstraint(ve)
-			if err != nil {
-				return nil, err
-			}
-
-			if constraints.Check(v) {
+			if match {
 				pa, ok := db.ProvidesDatabase[p.GetPackageName()][ve]
 				if !ok {
 					return nil, errors.New("No versions found for package")
@@ -257,15 +251,12 @@ func (db *InMemoryDatabase) FindPackages(p Package) ([]Package, error) {
 	}
 	var versionsInWorld []Package
 	for ve, _ := range versions {
-		v, err := version.NewVersion(ve)
+		match, err := p.SelectorMatchVersion(ve)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Error on match selector")
 		}
-		constraints, err := version.NewConstraint(p.GetVersion())
-		if err != nil {
-			return nil, err
-		}
-		if constraints.Check(v) {
+
+		if match {
 			w, err := db.FindPackage(&DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
 			if err != nil {
 				return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")

pkg/package/database_test.go

@@ -77,6 +77,27 @@ var _ = Describe("Database", func() {
 
 		})
 
+		It("Find specific package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
 		It("Provides replaces definitions", func() {
 			db := NewInMemoryDatabase(false)
 			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})

pkg/package/package.go

@@ -17,8 +17,10 @@ package pkg
 
 import (
 	"bytes"
+	"crypto/md5"
 	"encoding/json"
 	"fmt"
+	"io"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -83,6 +85,12 @@ type Package interface {
 	GetLicense() string
 
 	IsSelector() bool
+	VersionMatchSelector(string) (bool, error)
+	SelectorMatchVersion(string) (bool, error)
+
+	String() string
+	HumanReadableString() string
+	HashFingerprint() string
 }
 
 type Tree interface {
@@ -126,11 +134,11 @@ func (t *DefaultPackage) JSON() ([]byte, error) {
 
 // DefaultPackage represent a standard package definition
 type DefaultPackage struct {
-	ID               int      `json:"-" storm:"id,increment"` // primary key with auto increment
-	Name             string   `json:"name"`                   // Affects YAML field names too.
-	Version          string   `json:"version"`                // Affects YAML field names too.
-	Category         string   `json:"category"`               // Affects YAML field names too.
-	UseFlags         []string `json:"use_flags"`              // Affects YAML field names too.
+	ID               int      `storm:"id,increment" json:"id"` // primary key with auto increment
+	Name             string   `json:"name"`                    // Affects YAML field names too.
+	Version          string   `json:"version"`                 // Affects YAML field names too.
+	Category         string   `json:"category"`                // Affects YAML field names too.
+	UseFlags         []string `json:"use_flags"`               // Affects YAML field names too.
 	State            State
 	PackageRequires  []*DefaultPackage `json:"requires"`  // Affects YAML field names too.
 	PackageConflicts []*DefaultPackage `json:"conflicts"` // Affects YAML field names too.
@@ -158,7 +166,7 @@ func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*D
 func (p *DefaultPackage) String() string {
 	b, err := p.JSON()
 	if err != nil {
-		return fmt.Sprintf("{ id: \"%d\", name: \"%s\" }", p.ID, p.Name)
+		return fmt.Sprintf("{ id: \"%d\", name: \"%s\", version: \"%s\", category: \"%s\"  }", p.ID, p.Name, p.Version, p.Category)
 	}
 	return fmt.Sprintf("%s", string(b))
 }
@@ -169,6 +177,26 @@ func (p *DefaultPackage) GetFingerPrint() string {
 	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version)
 }
 
+func (p *DefaultPackage) HashFingerprint() string {
+	h := md5.New()
+	io.WriteString(h, p.GetFingerPrint())
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func (p *DefaultPackage) HumanReadableString() string {
+	return fmt.Sprintf("%s/%s-%s", p.Category, p.Name, p.Version)
+}
+
+func FromString(s string) Package {
+	var unescaped DefaultPackage
+
+	err := json.Unmarshal([]byte(s), &unescaped)
+	if err != nil {
+		return &unescaped
+	}
+	return &unescaped
+}
+
 func (p *DefaultPackage) GetPackageName() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.Category)
 }
@@ -322,15 +350,11 @@ func (p *DefaultPackage) Expand(definitiondb PackageDatabase) ([]Package, error)
 		return nil, err
 	}
 	for _, w := range all {
-		v, err := version.NewVersion(w.GetVersion())
+		match, err := p.SelectorMatchVersion(w.GetVersion())
 		if err != nil {
 			return nil, err
 		}
-		constraints, err := version.NewConstraint(p.GetVersion())
-		if err != nil {
-			return nil, err
-		}
-		if constraints.Check(v) {
+		if match {
 			versionsInWorld = append(versionsInWorld, w)
 		}
 	}
@@ -538,20 +562,25 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 			if err != nil || len(packages) == 0 {
 				required = requiredDef
 			} else {
-				for _, p := range packages {
-					encodedB, err := p.Encode(db)
-					if err != nil {
-						return nil, err
-					}
-					B := bf.Var(encodedB)
-					formulas = append(formulas, bf.Or(bf.Not(A),
-						bf.Not(B)))
+				if len(packages) == 1 {
+					required = packages[0]
+				} else {
+					for _, p := range packages {
+						encodedB, err := p.Encode(db)
+						if err != nil {
+							return nil, err
+						}
+						B := bf.Var(encodedB)
+						formulas = append(formulas, bf.Or(bf.Not(A),
+							bf.Not(B)))
 
-					f, err := p.BuildFormula(definitiondb, db)
-					if err != nil {
-						return nil, err
+						f, err := p.BuildFormula(definitiondb, db)
+						if err != nil {
+							return nil, err
+						}
+						formulas = append(formulas, f...)
 					}
-					formulas = append(formulas, f...)
+					continue
 				}
 			}
 		}

pkg/package/package_suite_test.go

@@ -18,11 +18,15 @@ package pkg_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Package Suite")
 }

pkg/package/package_test.go

@@ -23,6 +23,22 @@ import (
 
 var _ = Describe("Package", func() {
 
+	Context("Encoding/Decoding", func() {
+		a := &DefaultPackage{Name: "test", Version: "1", Category: "t"}
+		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		It("Encodes and decodes correctly", func() {
+			Expect(a.String()).ToNot(Equal(""))
+			p := FromString(a.String())
+			Expect(p).To(Equal(a))
+		})
+
+		It("Generates packages fingerprint's hashes", func() {
+			Expect(a.HashFingerprint()).ToNot(Equal(a1.HashFingerprint()))
+			Expect(a.HashFingerprint()).To(Equal("c64caa391b79adb598ad98e261aa79a0"))
+		})
+	})
+
 	Context("Simple package", func() {
 		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})

pkg/package/version.go

@@ -0,0 +1,364 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>,
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	version "github.com/hashicorp/go-version"
+)
+
+// Package Selector Condition
+type PkgSelectorCondition int
+
+type PkgVersionSelector struct {
+	Version       string
+	VersionSuffix string
+	Condition     PkgSelectorCondition
+	// TODO: Integrate support for multiple repository
+}
+
+const (
+	PkgCondInvalid = 0
+	// >
+	PkgCondGreater = 1
+	// >=
+	PkgCondGreaterEqual = 2
+	// <
+	PkgCondLess = 3
+	// <=
+	PkgCondLessEqual = 4
+	// =
+	PkgCondEqual = 5
+	// !
+	PkgCondNot = 6
+	// ~
+	PkgCondAnyRevision = 7
+	// =<pkg>*
+	PkgCondMatchVersion = 8
+)
+
+func PkgSelectorConditionFromInt(c int) (ans PkgSelectorCondition) {
+	if c == PkgCondGreater {
+		ans = PkgCondGreater
+	} else if c == PkgCondGreaterEqual {
+		ans = PkgCondGreaterEqual
+	} else if c == PkgCondLess {
+		ans = PkgCondLess
+	} else if c == PkgCondLessEqual {
+		ans = PkgCondLessEqual
+	} else if c == PkgCondNot {
+		ans = PkgCondNot
+	} else if c == PkgCondAnyRevision {
+		ans = PkgCondAnyRevision
+	} else if c == PkgCondMatchVersion {
+		ans = PkgCondMatchVersion
+	} else {
+		ans = PkgCondInvalid
+	}
+	return
+}
+
+func (p PkgSelectorCondition) String() (ans string) {
+	if p == PkgCondInvalid {
+		ans = ""
+	} else if p == PkgCondGreater {
+		ans = ">"
+	} else if p == PkgCondGreaterEqual {
+		ans = ">="
+	} else if p == PkgCondLess {
+		ans = "<"
+	} else if p == PkgCondLessEqual {
+		ans = "<="
+	} else if p == PkgCondEqual {
+		// To permit correct matching on database
+		// we currently use directly package version without =
+		ans = ""
+	} else if p == PkgCondNot {
+		ans = "!"
+	} else if p == PkgCondAnyRevision {
+		ans = "~"
+	} else if p == PkgCondMatchVersion {
+		ans = "=*"
+	}
+	return
+}
+
+func (p PkgSelectorCondition) Int() (ans int) {
+	if p == PkgCondInvalid {
+		ans = PkgCondInvalid
+	} else if p == PkgCondGreater {
+		ans = PkgCondGreater
+	} else if p == PkgCondGreaterEqual {
+		ans = PkgCondGreaterEqual
+	} else if p == PkgCondLess {
+		ans = PkgCondLess
+	} else if p == PkgCondLessEqual {
+		ans = PkgCondLessEqual
+	} else if p == PkgCondEqual {
+		// To permit correct matching on database
+		// we currently use directly package version without =
+		ans = PkgCondEqual
+	} else if p == PkgCondNot {
+		ans = PkgCondNot
+	} else if p == PkgCondAnyRevision {
+		ans = PkgCondAnyRevision
+	} else if p == PkgCondMatchVersion {
+		ans = PkgCondMatchVersion
+	}
+	return
+}
+
+func ParseVersion(v string) (PkgVersionSelector, error) {
+	var ans PkgVersionSelector = PkgVersionSelector{
+		Version:       "",
+		VersionSuffix: "",
+		Condition:     PkgCondInvalid,
+	}
+
+	if strings.HasPrefix(v, ">=") {
+		v = v[2:]
+		ans.Condition = PkgCondGreaterEqual
+	} else if strings.HasPrefix(v, ">") {
+		v = v[1:]
+		ans.Condition = PkgCondGreater
+	} else if strings.HasPrefix(v, "<=") {
+		v = v[2:]
+		ans.Condition = PkgCondLessEqual
+	} else if strings.HasPrefix(v, "<") {
+		v = v[1:]
+		ans.Condition = PkgCondLess
+	} else if strings.HasPrefix(v, "=") {
+		v = v[1:]
+		if strings.HasSuffix(v, "*") {
+			ans.Condition = PkgCondMatchVersion
+			v = v[0 : len(v)-1]
+		} else {
+			ans.Condition = PkgCondEqual
+		}
+	} else if strings.HasPrefix(v, "~") {
+		v = v[1:]
+		ans.Condition = PkgCondAnyRevision
+	} else if strings.HasPrefix(v, "!") {
+		v = v[1:]
+		ans.Condition = PkgCondNot
+	}
+
+	// Check if build number is present
+	buildIdx := strings.Index(v, "+")
+	buildVersion := ""
+	if buildIdx > 0 {
+		// <pre-release> ::= <dot-separated pre-release identifiers>
+		//
+		// <dot-separated pre-release identifiers> ::=
+		//      <pre-release identifier> | <pre-release identifier> "."
+		//      <dot-separated pre-release identifiers>
+		//
+		// <build> ::= <dot-separated build identifiers>
+		//
+		// <dot-separated build identifiers> ::= <build identifier>
+		//      | <build identifier> "." <dot-separated build identifiers>
+		//
+		// <pre-release identifier> ::= <alphanumeric identifier>
+		//                            | <numeric identifier>
+		//
+		// <build identifier> ::= <alphanumeric identifier>
+		//      | <digits>
+		//
+		// <alphanumeric identifier> ::= <non-digit>
+		//      | <non-digit> <identifier characters>
+		//      | <identifier characters> <non-digit>
+		//      | <identifier characters> <non-digit> <identifier characters>
+		buildVersion = v[buildIdx:]
+		v = v[0:buildIdx]
+	}
+
+	regexPkg := regexp.MustCompile(
+		fmt.Sprintf("(%s|%s|%s|%s|%s|%s)((%s|%s|%s|%s|%s|%s|%s)+)*$",
+			// Version regex
+			// 1.1
+			"[0-9]+[.][0-9]+[a-z]*",
+			// 1
+			"[0-9]+[a-z]*",
+			// 1.1.1
+			"[0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+			// 1.1.1.1
+			"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+			// 1.1.1.1.1
+			"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+			// 1.1.1.1.1.1
+			"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+			// suffix
+			"-r[0-9]+",
+			"_p[0-9]+",
+			"_pre[0-9]*",
+			"_rc[0-9]+",
+			// handle also rc without number
+			"_rc",
+			"_alpha",
+			"_beta",
+		),
+	)
+	matches := regexPkg.FindAllString(v, -1)
+
+	if len(matches) > 0 {
+		// Check if there patch
+		if strings.Contains(matches[0], "_p") {
+			ans.Version = matches[0][0:strings.Index(matches[0], "_p")]
+			ans.VersionSuffix = matches[0][strings.Index(matches[0], "_p"):]
+		} else if strings.Contains(matches[0], "_rc") {
+			ans.Version = matches[0][0:strings.Index(matches[0], "_rc")]
+			ans.VersionSuffix = matches[0][strings.Index(matches[0], "_rc"):]
+		} else if strings.Contains(matches[0], "_alpha") {
+			ans.Version = matches[0][0:strings.Index(matches[0], "_alpha")]
+			ans.VersionSuffix = matches[0][strings.Index(matches[0], "_alpha"):]
+		} else if strings.Contains(matches[0], "_beta") {
+			ans.Version = matches[0][0:strings.Index(matches[0], "_beta")]
+			ans.VersionSuffix = matches[0][strings.Index(matches[0], "_beta"):]
+		} else if strings.Contains(matches[0], "-r") {
+			ans.Version = matches[0][0:strings.Index(matches[0], "-r")]
+			ans.VersionSuffix = matches[0][strings.Index(matches[0], "-r"):]
+		} else {
+			ans.Version = matches[0]
+		}
+	}
+
+	// Set condition if there isn't a prefix but only a version
+	if ans.Condition == PkgCondInvalid && ans.Version != "" {
+		ans.Condition = PkgCondEqual
+	}
+
+	ans.Version += buildVersion
+
+	// NOTE: Now suffix complex like _alpha_rc1 are not supported.
+	return ans, nil
+}
+
+func PackageAdmit(selector, i PkgVersionSelector) (bool, error) {
+	var v1 *version.Version = nil
+	var v2 *version.Version = nil
+	var ans bool
+	var err error
+
+	if selector.Version != "" {
+		v1, err = version.NewVersion(selector.Version)
+		if err != nil {
+			return false, err
+		}
+	}
+	if i.Version != "" {
+		v2, err = version.NewVersion(i.Version)
+		if err != nil {
+			return false, err
+		}
+	} else {
+		// If version is not defined match always package
+		ans = true
+	}
+
+	// If package doesn't define version admit all versions of the package.
+	if selector.Version == "" {
+		ans = true
+	} else {
+		if selector.Condition == PkgCondInvalid || selector.Condition == PkgCondEqual {
+			// case 1: source-pkg-1.0 and dest-pkg-1.0 or dest-pkg without version
+			if i.Version != "" && i.Version == selector.Version && selector.VersionSuffix == i.VersionSuffix {
+				ans = true
+			}
+		} else if selector.Condition == PkgCondAnyRevision {
+			if v1 != nil && v2 != nil {
+				ans = v1.Equal(v2)
+			}
+		} else if selector.Condition == PkgCondMatchVersion {
+			// TODO: case of 7.3* where 7.30 is accepted.
+			if v1 != nil && v2 != nil {
+				segments := v1.Segments()
+				n := strings.Count(selector.Version, ".")
+				switch n {
+				case 0:
+					segments[0]++
+				case 1:
+					segments[1]++
+				case 2:
+					segments[2]++
+				default:
+					segments[len(segments)-1]++
+				}
+				nextVersion := strings.Trim(strings.Replace(fmt.Sprint(segments), " ", ".", -1), "[]")
+				constraints, err := version.NewConstraint(
+					fmt.Sprintf(">= %s, < %s", selector.Version, nextVersion),
+				)
+				if err != nil {
+					return false, err
+				}
+				ans = constraints.Check(v2)
+			}
+		} else if v1 != nil && v2 != nil {
+
+			// TODO: Integrate check of version suffix
+			switch selector.Condition {
+			case PkgCondGreaterEqual:
+				ans = v2.GreaterThanOrEqual(v1)
+			case PkgCondLessEqual:
+				ans = v2.LessThanOrEqual(v1)
+			case PkgCondGreater:
+				ans = v2.GreaterThan(v1)
+			case PkgCondLess:
+				ans = v2.LessThan(v1)
+			case PkgCondNot:
+				ans = !v2.Equal(v1)
+			}
+		}
+	}
+
+	return ans, nil
+}
+
+func (p *DefaultPackage) SelectorMatchVersion(v string) (bool, error) {
+	if !p.IsSelector() {
+		return false, errors.New("Package is not a selector")
+	}
+
+	vS, err := ParseVersion(p.GetVersion())
+	if err != nil {
+		return false, err
+	}
+
+	vSI, err := ParseVersion(v)
+	if err != nil {
+		return false, err
+	}
+
+	return PackageAdmit(vS, vSI)
+}
+
+func (p *DefaultPackage) VersionMatchSelector(selector string) (bool, error) {
+	vS, err := ParseVersion(selector)
+	if err != nil {
+		return false, err
+	}
+
+	vSI, err := ParseVersion(p.GetVersion())
+	if err != nil {
+		return false, err
+	}
+
+	return PackageAdmit(vS, vSI)
+}

pkg/package/version_test.go

@@ -0,0 +1,288 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Versions", func() {
+
+	Context("Versions Parser1", func() {
+		v, err := ParseVersion(">=1.0")
+		It("ParseVersion1", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser2", func() {
+		v, err := ParseVersion(">1.0")
+		It("ParseVersion2", func() {
+			var c PkgSelectorCondition = PkgCondGreater
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser3", func() {
+		v, err := ParseVersion("<=1.0")
+		It("ParseVersion3", func() {
+			var c PkgSelectorCondition = PkgCondLessEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser4", func() {
+		v, err := ParseVersion("<1.0")
+		It("ParseVersion4", func() {
+			var c PkgSelectorCondition = PkgCondLess
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser5", func() {
+		v, err := ParseVersion("=1.0")
+		It("ParseVersion5", func() {
+			var c PkgSelectorCondition = PkgCondEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser6", func() {
+		v, err := ParseVersion("!1.0")
+		It("ParseVersion6", func() {
+			var c PkgSelectorCondition = PkgCondNot
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("1.0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser7", func() {
+		v, err := ParseVersion("")
+		It("ParseVersion7", func() {
+			var c PkgSelectorCondition = PkgCondInvalid
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal(""))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser8", func() {
+		v, err := ParseVersion("=12.1.0.2_p1")
+		It("ParseVersion8", func() {
+			var c PkgSelectorCondition = PkgCondEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("12.1.0.2"))
+			Expect(v.VersionSuffix).Should(Equal("_p1"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser9", func() {
+		v, err := ParseVersion(">=0.0.20190406.4.9.172-r1")
+		It("ParseVersion9", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.0.20190406.4.9.172"))
+			Expect(v.VersionSuffix).Should(Equal("-r1"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser10", func() {
+		v, err := ParseVersion(">=0.0.20190406.4.9.172_alpha")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.0.20190406.4.9.172"))
+			Expect(v.VersionSuffix).Should(Equal("_alpha"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser11 - semver", func() {
+		v, err := ParseVersion("0.1.0+0")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.1.0+0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser12 - semver", func() {
+		v, err := ParseVersion(">=0.1.0_alpha+AB")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.1.0+AB"))
+			Expect(v.VersionSuffix).Should(Equal("_alpha"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser13 - semver", func() {
+		v, err := ParseVersion(">=0.1.0_alpha+0.1.22")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.1.0+0.1.22"))
+			Expect(v.VersionSuffix).Should(Equal("_alpha"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser14 - semver", func() {
+		v, err := ParseVersion(">=0.1.0_alpha+0.1.22")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.1.0+0.1.22"))
+			Expect(v.VersionSuffix).Should(Equal("_alpha"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser15 - semver", func() {
+		v, err := ParseVersion("<=0.3.222.4.5+AB")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondLessEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.3.222.4.5+AB"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Selector1", func() {
+		v1, err := ParseVersion(">=0.0.20190406.4.9.172-r1")
+		v2, err2 := ParseVersion("1.0.111")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector1", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(true))
+		})
+	})
+
+	Context("Selector2", func() {
+		v1, err := ParseVersion(">=0.0.20190406.4.9.172-r1")
+		v2, err2 := ParseVersion("0")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector2", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(false))
+		})
+	})
+
+	Context("Selector3", func() {
+		v1, err := ParseVersion(">0")
+		v2, err2 := ParseVersion("0.0.40-alpha")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector3", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(true))
+		})
+	})
+
+	Context("Selector4", func() {
+		v1, err := ParseVersion(">0")
+		v2, err2 := ParseVersion("")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector4", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(true))
+		})
+	})
+
+	Context("Selector5", func() {
+		v1, err := ParseVersion(">0.1.0+0.4")
+		v2, err2 := ParseVersion("0.1.0+0.3")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector5", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(false))
+		})
+	})
+
+	Context("Selector6", func() {
+		v1, err := ParseVersion(">=0.1.0+0.4")
+		v2, err2 := ParseVersion("0.1.0+0.5")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector6", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(true))
+		})
+	})
+
+	PContext("Selector7", func() {
+		v1, err := ParseVersion(">0.1.0+0.4")
+		v2, err2 := ParseVersion("0.1.0+0.5")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector7", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(true))
+		})
+	})
+
+	Context("Condition Converter 1", func() {
+		gp, err := gentoo.ParsePackageStr("=layer/build-1.0")
+		var cond gentoo.PackageCond = gentoo.PkgCondEqual
+		It("Converter1", func() {
+			Expect(err).Should(BeNil())
+			Expect((*gp).Condition).Should(Equal(cond))
+			Expect(PkgSelectorConditionFromInt((*gp).Condition.Int()).String()).Should(Equal(""))
+		})
+	})
+
+})

pkg/repository/loader.go

@@ -0,0 +1,85 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package repository
+
+import (
+	"io/ioutil"
+	"path"
+	"regexp"
+
+	"github.com/ghodss/yaml"
+
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+func LoadRepositories(c *LuetConfig) error {
+	var regexRepo = regexp.MustCompile(`.yml$`)
+
+	for _, rdir := range c.RepositoriesConfDir {
+		Debug("Parsing Repository Directory", rdir, "...")
+
+		files, err := ioutil.ReadDir(rdir)
+		if err != nil {
+			Warning("Skip dir", rdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexRepo.MatchString(file.Name()) {
+				Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(rdir, file.Name()))
+			if err != nil {
+				Warning("On read file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := LoadRepository(content)
+			if err != nil {
+				Warning("On parse file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Urls) == 0 || r.Type == "" {
+				Warning("Invalid repository ", file.Name())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddSystemRepository(*r)
+		}
+	}
+	return nil
+}
+
+func LoadRepository(data []byte) (*LuetRepository, error) {
+	ans := NewEmptyLuetRepository()
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}

pkg/repository/repository_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package repository_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Repository Suite")
+}

pkg/repository/repository_test.go

@@ -0,0 +1,46 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package repository_test
+
+import (
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/repository"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	viper "github.com/spf13/viper"
+)
+
+var _ = Describe("Repository", func() {
+	Context("Load Repository1", func() {
+		cfg := NewLuetConfig(viper.New())
+		cfg.RepositoriesConfDir = []string{
+			"../../tests/fixtures/repos.conf.d",
+		}
+		err := LoadRepositories(cfg)
+
+		It("Chec Load Repository 1", func() {
+			Expect(err).Should(BeNil())
+			Expect(len(cfg.SystemRepositories)).Should(Equal(1))
+			Expect(cfg.SystemRepositories[0].Name).Should(Equal("test1"))
+			Expect(cfg.SystemRepositories[0].Priority).Should(Equal(999))
+			Expect(cfg.SystemRepositories[0].Type).Should(Equal("disk"))
+			Expect(len(cfg.SystemRepositories[0].Urls)).Should(Equal(1))
+			Expect(cfg.SystemRepositories[0].Urls[0]).Should(Equal("tests/repos/test1"))
+		})
+	})
+})

pkg/solver/decoder.go

@@ -196,7 +196,7 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 	for _, res := range result {
 		a, ok := tmpMap[res]
 		if !ok {
-			panic("fail")
+			panic("fail looking for " + res)
 			//	continue
 		}
 		orderedAssertions = append(orderedAssertions, a)

pkg/solver/resolver.go

@@ -0,0 +1,336 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/crillab/gophersat/bf"
+	"github.com/mudler/luet/pkg/helpers"
+	"gopkg.in/yaml.v2"
+
+	"github.com/ecooper/qlearning"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/pkg/errors"
+)
+
+type ActionType int
+
+const (
+	NoAction      = 0
+	Solved        = iota
+	NoSolution    = iota
+	Going         = iota
+	ActionRemoved = iota
+	ActionAdded   = iota
+
+	DoNoop = false
+
+	ActionDomains = 3 // Bump it if you increase the number of actions
+
+	DefaultMaxAttempts     = 9000
+	DefaultLearningRate    = 0.7
+	DefaultDiscount        = 1.0
+	DefaultInitialObserved = 999999
+
+	QLearningResolverType = "qlearning"
+)
+
+//. "github.com/mudler/luet/pkg/logger"
+
+// PackageResolver assists PackageSolver on unsat cases
+type PackageResolver interface {
+	Solve(bf.Formula, PackageSolver) (PackagesAssertions, error)
+}
+
+type DummyPackageResolver struct {
+}
+
+func (*DummyPackageResolver) Solve(bf.Formula, PackageSolver) (PackagesAssertions, error) {
+	return nil, errors.New("Could not satisfy the constraints. Try again by removing deps ")
+}
+
+type QLearningResolver struct {
+	Attempts int
+
+	ToAttempt int
+
+	attempts int
+
+	Attempted map[string]bool
+
+	Solver  PackageSolver
+	Formula bf.Formula
+
+	Targets []pkg.Package
+	Current []pkg.Package
+
+	observedDelta       int
+	observedDeltaChoice []pkg.Package
+
+	Agent *qlearning.SimpleAgent
+}
+
+func SimpleQLearningSolver() PackageResolver {
+	return NewQLearningResolver(DefaultLearningRate, DefaultDiscount, DefaultMaxAttempts, DefaultInitialObserved)
+}
+
+// Defaults LearningRate 0.7, Discount 1.0
+func NewQLearningResolver(LearningRate, Discount float32, MaxAttempts, initialObservedDelta int) PackageResolver {
+	return &QLearningResolver{
+		Agent:         qlearning.NewSimpleAgent(LearningRate, Discount),
+		observedDelta: initialObservedDelta,
+		Attempts:      MaxAttempts,
+	}
+}
+
+func (resolver *QLearningResolver) Solve(f bf.Formula, s PackageSolver) (PackagesAssertions, error) {
+	//	Info("Using QLearning solver to resolve conflicts. Please be patient.")
+	resolver.Solver = s
+
+	s.SetResolver(&DummyPackageResolver{}) // Set dummy. Otherwise the attempts will run again a QLearning instance.
+	defer s.SetResolver(resolver)          // Set back ourselves as resolver
+
+	resolver.Formula = f
+
+	// Our agent by default has a learning rate of 0.7 and discount of 1.0.
+	if resolver.Agent == nil {
+		resolver.Agent = qlearning.NewSimpleAgent(DefaultLearningRate, DefaultDiscount) // FIXME: Remove hardcoded values
+	}
+
+	// 3 are the action domains, counting noop regardless if enabled or not
+	// get the permutations to attempt
+	resolver.ToAttempt = int(helpers.Factorial(uint64(len(resolver.Solver.(*Solver).Wanted)-1) * ActionDomains)) // TODO: type assertions must go away
+	resolver.Targets = resolver.Solver.(*Solver).Wanted
+
+	resolver.attempts = resolver.Attempts
+
+	resolver.Attempted = make(map[string]bool, len(resolver.Targets))
+
+	for resolver.IsComplete() == Going {
+		// Pick the next move, which is going to be a letter choice.
+		action := qlearning.Next(resolver.Agent, resolver)
+
+		// Whatever that choice is, let's update our model for its
+		// impact. If the package chosen makes the formula sat,
+		// then this action will be positive. Otherwise, it will be
+		// negative.
+		resolver.Agent.Learn(action, resolver)
+
+		// Reward doesn't change state so we can check what the
+		// reward would be for this action, and report how the
+		// env changed.
+		//	score := resolver.Reward(action)
+		//	if score > 0.0 {
+		//	resolver.Log("%s was correct", action.Action.String())
+		//	} else {
+		//	resolver.Log("%s was incorrect", action.Action.String())
+		//	}
+	}
+
+	// If we get good result, take it
+	// Take the result also if we did  reached overall maximum attempts
+	if resolver.IsComplete() == Solved || resolver.IsComplete() == NoSolution {
+
+		if len(resolver.observedDeltaChoice) != 0 {
+			// Take the minimum delta observed choice result, and consume it (Try sets the wanted list)
+			resolver.Solver.(*Solver).Wanted = resolver.observedDeltaChoice
+		}
+
+		return resolver.Solver.Solve()
+	} else {
+		return nil, errors.New("QLearning resolver failed ")
+	}
+
+}
+
+// Returns the current state.
+func (resolver *QLearningResolver) IsComplete() int {
+	if resolver.attempts < 1 {
+		return NoSolution
+	}
+
+	if resolver.ToAttempt > 0 {
+		return Going
+	}
+
+	return Solved
+}
+
+func (resolver *QLearningResolver) Try(c Choice) error {
+	pack := c.Package
+	packtoAdd := pkg.FromString(pack)
+	resolver.Attempted[pack+strconv.Itoa(int(c.Action))] = true // increase the count
+	s, _ := resolver.Solver.(*Solver)
+	var filtered []pkg.Package
+
+	switch c.Action {
+	case ActionAdded:
+		found := false
+		for _, p := range s.Wanted {
+			if p.String() == pack {
+				found = true
+				break
+			}
+		}
+		if !found {
+			resolver.Solver.(*Solver).Wanted = append(resolver.Solver.(*Solver).Wanted, packtoAdd)
+		}
+
+	case ActionRemoved:
+		for _, p := range s.Wanted {
+			if p.String() != pack {
+				filtered = append(filtered, p)
+			}
+		}
+
+		resolver.Solver.(*Solver).Wanted = filtered
+	}
+
+	_, err := resolver.Solver.Solve()
+
+	return err
+}
+
+// Choose applies a pack attempt, returning
+// true if the formula returns sat.
+//
+// Choose updates the resolver's state.
+func (resolver *QLearningResolver) Choose(c Choice) bool {
+	//pack := pkg.FromString(c.Package)
+
+	err := resolver.Try(c)
+
+	if err == nil {
+		resolver.ToAttempt--
+		resolver.attempts-- // Decrease attempts - it's a barrier. We could also do not decrease it here, allowing more attempts to be made
+	} else {
+		resolver.attempts--
+		return false
+	}
+
+	return true
+}
+
+// Reward returns a score for a given qlearning.StateAction. Reward is a
+// member of the qlearning.Rewarder interface. If the choice will make sat the formula, a positive score is returned.
+// Otherwise, a static -1000 is returned.
+func (resolver *QLearningResolver) Reward(action *qlearning.StateAction) float32 {
+	choice := action.Action.(*Choice)
+
+	//_, err := resolver.Solver.Solve()
+	err := resolver.Try(*choice)
+
+	toBeInstalled := len(resolver.Solver.(*Solver).Wanted)
+	originalTarget := len(resolver.Targets)
+	noaction := choice.Action == NoAction
+	delta := originalTarget - toBeInstalled
+
+	if err == nil {
+		// if toBeInstalled == originalTarget { // Base case: all the targets matches (it shouldn't happen, but lets put a higher)
+		// 	Debug("Target match, maximum score")
+		// 	return 24.0 / float32(len(resolver.Attempted))
+
+		// }
+		if DoNoop {
+			if noaction && toBeInstalled == 0 { // We decided to stay in the current state, and no targets have been chosen
+				return -100
+			}
+		}
+
+		if delta <= resolver.observedDelta { // Try to maximise observedDelta
+			resolver.observedDelta = delta
+			resolver.observedDeltaChoice = resolver.Solver.(*Solver).Wanted // we store it as this is our return value at the end
+			return 24.0 / float32(len(resolver.Attempted))
+		} else if toBeInstalled > 0 { // If we installed something, at least give a good score
+			return 24.0 / float32(len(resolver.Attempted))
+		}
+
+	}
+
+	return -1000
+}
+
+// Next creates a new slice of qlearning.Action instances. A possible
+// action is created for each package that could be removed from the formula's target
+func (resolver *QLearningResolver) Next() []qlearning.Action {
+	actions := make([]qlearning.Action, 0, (len(resolver.Targets)-1)*3)
+
+TARGETS:
+	for _, pack := range resolver.Targets {
+		for _, current := range resolver.Solver.(*Solver).Wanted {
+			if current.String() == pack.String() {
+				actions = append(actions, &Choice{Package: pack.String(), Action: ActionRemoved})
+				continue TARGETS
+			}
+
+		}
+		actions = append(actions, &Choice{Package: pack.String(), Action: ActionAdded})
+	}
+
+	if DoNoop {
+		actions = append(actions, &Choice{Package: "", Action: NoAction}) // NOOP
+	}
+
+	return actions
+}
+
+// Log is a wrapper of fmt.Printf. If Game.debug is true, Log will print
+// to stdout.
+func (resolver *QLearningResolver) Log(msg string, args ...interface{}) {
+	logMsg := fmt.Sprintf("(%d moves, %d remaining attempts) %s\n", len(resolver.Attempted), resolver.attempts, msg)
+	fmt.Println(fmt.Sprintf(logMsg, args...))
+}
+
+// String returns a consistent hash for the current env state to be
+// used in a qlearning.Agent.
+func (resolver *QLearningResolver) String() string {
+	return fmt.Sprintf("%v", resolver.Solver.(*Solver).Wanted)
+}
+
+// Choice implements qlearning.Action for a package choice for removal from wanted targets
+type Choice struct {
+	Package string     `json:"pack"`
+	Action  ActionType `json:"action"`
+}
+
+func ChoiceFromString(s string) (*Choice, error) {
+	var p *Choice
+	err := yaml.Unmarshal([]byte(s), &p)
+	if err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+// String returns the character for the current action.
+func (choice *Choice) String() string {
+	data, err := json.Marshal(choice)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+// Apply updates the state of the solver for the package choice.
+func (choice *Choice) Apply(state qlearning.State) qlearning.State {
+	resolver := state.(*QLearningResolver)
+	resolver.Choose(*choice)
+
+	return resolver
+}

pkg/solver/resolver_test.go

@@ -0,0 +1,182 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver_test
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "github.com/mudler/luet/pkg/solver"
+)
+
+var _ = Describe("Resolver", func() {
+
+	db := pkg.NewInMemoryDatabase(false)
+	dbInstalled := pkg.NewInMemoryDatabase(false)
+	dbDefinitions := pkg.NewInMemoryDatabase(false)
+	s := NewSolver(dbInstalled, dbDefinitions, db)
+
+	BeforeEach(func() {
+		db = pkg.NewInMemoryDatabase(false)
+		dbInstalled = pkg.NewInMemoryDatabase(false)
+		dbDefinitions = pkg.NewInMemoryDatabase(false)
+		s = NewSolver(dbInstalled, dbDefinitions, db)
+	})
+
+	Context("Conflict set", func() {
+		Context("DummyPackageResolver", func() {
+			It("is unsolvable - as we something we ask to install conflict with system stuff", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A})
+				Expect(len(solution)).To(Equal(0))
+				Expect(err).To(HaveOccurred())
+			})
+			It("succeeds to install D and F if explictly requested", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D, E, F} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{D, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(len(solution)).To(Equal(6))
+
+				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
+
+			})
+
+		})
+		Context("QLearningResolver", func() {
+			It("will find out that we can install D by ignoring A", func() {
+				s.SetResolver(SimpleQLearningSolver())
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A, D})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+
+				Expect(len(solution)).To(Equal(4))
+			})
+
+			It("will find out that we can install D and F by ignoring E and A", func() {
+				s.SetResolver(SimpleQLearningSolver())
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D, E, F} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A, D, E, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) // Was already installed
+				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
+				Expect(len(solution)).To(Equal(6))
+
+			})
+		})
+
+		Context("DummyPackageResolver", func() {
+			It("cannot find a solution", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A, D})
+				Expect(err).To(HaveOccurred())
+
+				Expect(len(solution)).To(Equal(0))
+			})
+
+		})
+	})
+
+})

pkg/solver/solver.go

@@ -33,6 +33,10 @@ type PackageSolver interface {
 	ConflictsWith(p pkg.Package, ls []pkg.Package) (bool, error)
 	World() []pkg.Package
 	Upgrade() ([]pkg.Package, PackagesAssertions, error)
+
+	SetResolver(PackageResolver)
+
+	Solve() (PackagesAssertions, error)
 }
 
 // Solver is the default solver for luet
@@ -41,20 +45,33 @@ type Solver struct {
 	SolverDatabase     pkg.PackageDatabase
 	Wanted             []pkg.Package
 	InstalledDatabase  pkg.PackageDatabase
+
+	Resolver PackageResolver
 }
 
 // NewSolver accepts as argument two lists of packages, the first is the initial set,
 // the second represent all the known packages.
 func NewSolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase) PackageSolver {
-	return &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb}
+	return NewResolver(installed, definitiondb, solverdb, &DummyPackageResolver{})
 }
 
-// SetWorld is a setter for the list of all known packages to the solver
+// NewReSolver accepts as argument two lists of packages, the first is the initial set,
+// the second represent all the known packages.
+func NewResolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase, re PackageResolver) PackageSolver {
+	return &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb, Resolver: re}
+}
+
+// SetDefinitionDatabase is a setter for the definition Database
 
 func (s *Solver) SetDefinitionDatabase(db pkg.PackageDatabase) {
 	s.DefinitionDatabase = db
 }
 
+// SetResolver is a setter for the unsat resolver backend
+func (s *Solver) SetResolver(r PackageResolver) {
+	s.Resolver = r
+}
+
 func (s *Solver) World() []pkg.Package {
 	return s.DefinitionDatabase.World()
 }
@@ -221,6 +238,7 @@ func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
 	}
 
 	s2 := NewSolver(installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+	s2.SetResolver(s.Resolver)
 	// Then try to uninstall the versions in the system, and store that tree
 	for _, p := range toUninstall {
 		r, err := s.Uninstall(p)
@@ -361,13 +379,20 @@ func (s *Solver) solve(f bf.Formula) (map[string]bool, bf.Formula, error) {
 
 // Solve builds the formula given the current state and returns package assertions
 func (s *Solver) Solve() (PackagesAssertions, error) {
+	var model map[string]bool
+	var err error
+
 	f, err := s.BuildFormula()
 
 	if err != nil {
 		return nil, err
 	}
 
-	model, _, err := s.solve(f)
+	model, _, err = s.solve(f)
+	if err != nil && s.Resolver != nil {
+		return s.Resolver.Solve(f, s)
+	}
+
 	if err != nil {
 		return nil, err
 	}

pkg/solver/solver_suite_test.go

@@ -18,11 +18,15 @@ package solver_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Solver Suite")
 }

pkg/tree/builder/gentoo/builder_suite_test.go

@@ -18,11 +18,15 @@ package gentoo_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestGentooBuilder(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Gentoo Suite")
 }

pkg/tree/tree_suite_test.go

@@ -18,11 +18,15 @@ package tree_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestTree(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Tree Suite")
 }

tests/fixtures/qlearning/a/build.yaml

@@ -0,0 +1,8 @@
+prelude:
+  - echo a > /aprelude
+steps:
+  - echo a > /a
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/qlearning/a/definition.yaml

@@ -0,0 +1,8 @@
+category: "test"
+name: "a"
+version: "1.0"
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"
+

tests/fixtures/qlearning/b/build.yaml

@@ -0,0 +1,5 @@
+image: "alpine"
+prelude:
+  - echo b > /bprelude
+steps:
+  - echo b > /b

tests/fixtures/qlearning/b/definition.yaml

@@ -0,0 +1,7 @@
+category: "test"
+name: "b"
+version: "1.0"
+conflicts:
+ - category: "test"
+   name: "c"
+   version: "1.0"

tests/fixtures/qlearning/c/build.yaml

@@ -0,0 +1,5 @@
+image: "alpine"
+prelude:
+  - echo c > /cprelude
+steps:
+  - echo c > /c

tests/fixtures/qlearning/c/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.0"

tests/fixtures/qlearning/d/build.yaml

@@ -0,0 +1,5 @@
+image: "alpine"
+prelude:
+  - echo d > /dprelude
+steps:
+  - echo d > /d

tests/fixtures/qlearning/d/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "d"
+version: "1.0"

tests/fixtures/qlearning/e/build.yaml

@@ -0,0 +1,9 @@
+prelude:
+  - echo e > /eprelude
+steps:
+  - echo e > /e
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"
+

tests/fixtures/qlearning/e/definition.yaml

@@ -0,0 +1,8 @@
+category: "test"
+name: "e"
+version: "1.0"
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"
+

tests/fixtures/qlearning/f/build.yaml

@@ -0,0 +1,5 @@
+prelude:
+  - echo f > /eprelude
+steps:
+  - echo f > /f
+image: "alpine"

tests/fixtures/qlearning/f/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "f"
+version: "1.0"

tests/fixtures/repos.conf.d/repo-test.yml

@@ -0,0 +1,9 @@
+name: "test1"
+description: "Test1 Repository"
+type: "disk"
+urls:
+  - "tests/repos/test1"
+priority: 999
+enable: true
+# auth:
+#   token: "xxxxx"

tests/fixtures/repos.conf.d/repo.yml.skipped

@@ -0,0 +1,8 @@
+name: "test1-skipped"
+description: "Test1 Repository"
+type: "disk"
+path: "tests/repos/test1"
+priority: 1
+enable: true
+# auth:
+#   token: "xxxxx"

tests/fixtures/retrieve-integration/a/build.yaml

@@ -0,0 +1,3 @@
+image: "alpine"
+steps:
+  - echo a > /a

tests/fixtures/retrieve-integration/a/definition.yaml

@@ -0,0 +1,3 @@
+name: "a"
+version: "1.0"
+category: "test"

tests/fixtures/retrieve-integration/b/build.yaml

@@ -0,0 +1,10 @@
+
+steps:
+- tar xvf a-test-1.0.package.* -C ./
+- mv a /b
+requires:
+- name: "a"
+  version: "1.0"
+  category: "test"
+retrieve:
+- a-test-1.0.package.*

tests/fixtures/retrieve-integration/b/definition.yaml

@@ -0,0 +1,7 @@
+name: "b"
+version: "1.0"
+category: "test"
+requires:
+- name: "a"
+  version: "1.0"
+  category: "test"

tests/fixtures/retrieve/a/build.yaml

@@ -0,0 +1,10 @@
+image: "luet/base"
+seed: "alpine"
+steps:
+  - echo foo > /test
+  - echo bar > /test2
+retrieve:
+- test
+- http://www.google.com
+env:
+- test=1

tests/fixtures/retrieve/a/definition.yaml

@@ -0,0 +1,3 @@
+name: "a"
+version: "1.0"
+category: "test"

tests/integration/01_simple.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c > /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/c
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testReInstall() {
+    output=$(luet install --config $tmpdir/luet.yaml  test/c-1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertContains 'contains warning' "$output" 'Filtering out'
+}
+
+testUnInstall() {
+    luet uninstall --config $tmpdir/luet.yaml test/c
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+testInstallAgain() {
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    output=$(luet install --config $tmpdir/luet.yaml test/c-1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'Filtering out'
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/01_simple_gzip.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c-1.0 > /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --tree-compression gzip \
+    --tree-path foo.tar \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+    assertTrue 'create named tree in gzip' "[ -e '$tmpdir/testbuild/foo.tar.gz' ]"
+    assertTrue 'create tree in gzip-only' "[ ! -e '$tmpdir/testbuild/foo.tar' ]"
+
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/c-1.0
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testReInstall() {
+    output=$(luet install --config $tmpdir/luet.yaml  test/c-1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertContains 'contains warning' "$output" 'Filtering out'
+}
+
+testUnInstall() {
+    luet uninstall --config $tmpdir/luet.yaml test/c-1.0
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+testInstallAgain() {
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    output=$(luet install --config $tmpdir/luet.yaml test/c-1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'Filtering out'
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/02_create_repo_from_config.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c-1.0 > /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    --config $tmpdir/luet.yaml \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --repo "main" \
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/c-1.0
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testReInstall() {
+    output=$(luet install --config $tmpdir/luet.yaml  test/c-1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertContains 'contains warning' "$output" 'Filtering out'
+}
+
+testUnInstall() {
+    luet uninstall --config $tmpdir/luet.yaml test/c-1.0
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+testInstallAgain() {
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    output=$(luet install --config $tmpdir/luet.yaml test/c-1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'Filtering out'
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/03_qlearning.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --all --tree "$ROOT_DIR/tests/fixtures/qlearning" --destination $tmpdir/testbuild --compression gzip
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/qlearning" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/c
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package C installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testFullInstall() {
+    output=$(luet install --config $tmpdir/luet.yaml test/d test/f test/e test/a)
+    installst=$?
+    assertEquals 'cannot install' "$installst" "1"
+    assertTrue 'package D installed' "[ ! -e '$tmpdir/testrootfs/d' ]"
+    assertTrue 'package F installed' "[ ! -e '$tmpdir/testrootfs/f' ]"
+}
+
+testInstallAgain() {
+    output=$(luet install --solver-type qlearning --config $tmpdir/luet.yaml test/d test/f test/e test/a)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'Filtering out'
+    assertTrue 'package D installed' "[ -e '$tmpdir/testrootfs/d' ]"
+    assertTrue 'package F installed' "[ -e '$tmpdir/testrootfs/f' ]"
+    assertTrue 'package E not installed' "[ ! -e '$tmpdir/testrootfs/e' ]"
+    assertTrue 'package A not installed' "[ ! -e '$tmpdir/testrootfs/a' ]"
+}
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/04_retrieve.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/retrieve-integration" --destination $tmpdir/testbuild --compression gzip test/b
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/retrieve-integration" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/b
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package B installed' "[ -e '$tmpdir/testrootfs/b' ]"
+    val=$(cat "$tmpdir/testrootfs/b")
+    assertEquals 'package B content comes from a' "$val" "a"
+    assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/a' ]"
+}
+
+
+testUnInstall() {
+    luet uninstall --config $tmpdir/luet.yaml test/b
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/b' ]"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/a' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/b-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+export ROOT_DIR="$(git rev-parse --show-toplevel)"
+
+pushd $ROOT_DIR
+    go build -o "$ROOT_DIR/tests/integration/bin/luet" 
+popd
+
+[ ! -d "$ROOT_DIR/tests/integration/shunit2" ] && git clone https://github.com/kward/shunit2.git "$ROOT_DIR/tests/integration/shunit2" 
+
+export PATH=$ROOT_DIR/tests/integration/bin/:$PATH
+
+for script in $(ls "$ROOT_DIR/tests/integration/" | grep '^[0-9]*_.*.sh'); do
+  echo "Executing script '$script'."
+  $ROOT_DIR/tests/integration/$script
+done

vendor/github.com/BurntSushi/toml/.gitignore

@@ -0,0 +1,5 @@
+TAGS
+tags
+.*.swp
+tomlcheck/tomlcheck
+toml.test

vendor/github.com/BurntSushi/toml/.travis.yml

@@ -0,0 +1,15 @@
+language: go
+go:
+  - 1.1
+  - 1.2
+  - 1.3
+  - 1.4
+  - 1.5
+  - 1.6
+  - tip
+install:
+  - go install ./...
+  - go get github.com/BurntSushi/toml-test
+script:
+  - export PATH="$PATH:$HOME/gopath/bin"
+  - make test