Tag 0.8.10

The parallel solver made the issue more visible, the constraints needed
to be less relaxed and needed to be exclusive so our candidate is looked
up at it first

.github/workflows/release.yml

@@ -0,0 +1,23 @@
+on: push
+name: Build and release on push
+jobs:
+  release:
+    name: Test and Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Tests
+        run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage
+      - name: Build
+        run: sudo -E env "PATH=$PATH" make multiarch-build && sudo chmod -R 777 release/
+      - name: Release
+        uses: fnkr/github-action-ghr@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GHR_PATH: release/
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+
+on: pull_request
+name: Build and Test
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage

.travis.yml

@@ -2,17 +2,18 @@ language: go
 services:
   - docker
 go:
-  - "1.12"
+  - "1.14"
 env:
   - "GO15VENDOREXPERIMENT=1"
 before_install:
-  - make deps
-  - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && mkdir -p $HOME/bin && export PATH=$PATH:$HOME/bin && mv container-diff-linux-amd64 $HOME/bin/container-diff
+  - sudo -E env "PATH=$PATH" apt-get install -y libcap2-bin
+  - sudo -E env "PATH=$PATH" make deps
 script:
-  - make multiarch-build test test-integration
-after_success:
-  - make coverage
-  - bash <(curl -s https://codecov.io/bash)
-  - git config --global user.name "Deployer" && git config --global user.email foo@bar.com
-  - go get github.com/tcnksm/ghr
-  - ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+  - sudo -E env "PATH=$PATH" make multiarch-build test-integration test-coverage
+#after_success:
+#  - |
+#      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+#        sudo -E env "PATH=$PATH" git config --global user.name "Deployer" && git config --global user.email foo@bar.com
+#        sudo -E env "PATH=$PATH" go get github.com/tcnksm/ghr
+#        sudo -E env "PATH=$PATH" ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+#      fi

Dockerfile

@@ -1,6 +1,7 @@
 FROM golang as builder
+RUN apt-get update && apt-get install -y upx
 ADD . /luet
-RUN cd /luet && make build
+RUN cd /luet && make build-small
 
 FROM scratch
 ENV LUET_NOLOCK=true

Makefile

@@ -1,10 +1,14 @@
+
+# go tool nm ./luet | grep Commit
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S %Z')"
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildCommit=$(shell git rev-parse HEAD)"
+
 NAME ?= luet
 PACKAGE_NAME ?= $(NAME)
 PACKAGE_CONFLICT ?= $(PACKAGE_NAME)-beta
 REVISION := $(shell git rev-parse --short HEAD || echo dev)
 VERSION := $(shell git describe --tags || echo $(REVISION))
 VERSION := $(shell echo $(VERSION) | sed -e 's/^v//g')
-ITTERATION := $(shell date +%s)
 BUILD_PLATFORMS ?= -osarch="linux/amd64" -osarch="linux/386" -osarch="linux/arm"
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
@@ -19,7 +23,7 @@ fmt:
 test:
 	GO111MODULE=off go get github.com/onsi/ginkgo/ginkgo
 	GO111MODULE=off go get github.com/onsi/gomega/...
-	ginkgo -race -r ./...
+	ginkgo -race -r -flakeAttempts 3 ./...
 
 .PHONY: test-integration
 test-integration:
@@ -29,6 +33,10 @@ test-integration:
 coverage:
 	go test ./... -race -coverprofile=coverage.txt -covermode=atomic
 
+.PHONY: test-coverage
+test-coverage:
+	scripts/ginkgo.coverage.sh --codecov
+
 .PHONY: help
 help:
 	# make all => deps test lint build
@@ -55,16 +63,17 @@ deps:
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 go build
+	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)'
+
+.PHONY: build-small
+build-small:
+	@$(MAKE) LDFLAGS+="-s -w" build
+	upx --brute -1 $(NAME)
 
 .PHONY: image
 image:
 	docker build --rm -t luet/base .
 
-.PHONY: gox-build
-gox-build:
-	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
-
 .PHONY: lint
 lint:
 	golint ./... | grep -v "be unexported"
@@ -81,4 +90,4 @@ test-docker:
 
 .PHONY: multiarch-build
 multiarch-build:
-	gox $(BUILD_PLATFORMS) -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"

README.md

@@ -1,4 +1,5 @@
 # luet - Container-based Package manager
+[![Docker Repository on Quay](https://quay.io/repository/luet/base/status "Docker Repository on Quay")](https://quay.io/repository/luet/base)
 [![Go Report Card](https://goreportcard.com/badge/github.com/mudler/luet)](https://goreportcard.com/report/github.com/mudler/luet)
 [![Build Status](https://travis-ci.org/mudler/luet.svg?branch=master)](https://travis-ci.org/mudler/luet)
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
@@ -30,6 +31,23 @@ To install luet, you can grab a release on the [Release page](https://github.com
 
 Luet is not feature-complete yet, it can build, install/uninstall/upgrade packages - but it doesn't support yet all the features you would normally expect from a Package Manager nowadays.
 
+# Dependency solving
+
+Luet uses SAT and Reinforcement learning engine for dependency solving.
+It encodes the package requirements into a SAT problem, using gophersat to solve the dependency tree and give a concrete model as result.
+
+## SAT encoding
+
+Each package and its constraints are encoded and built around [OPIUM](https://ranjitjhala.github.io/static/opium.pdf). Additionally, Luet treats
+also selectors seamlessly while building the model, adding *ALO* ( *At least one* ) and *AMO* ( *At most one* ) rules to guarantee coherence within the installed system.
+
+## Reinforcement learning
+
+Luet also implements a small and portable qlearning agent that will try to solve conflict on your behalf
+when they arises while trying to validate your queries against the system model.
+
+To leverage it, simply pass ```--solver-type qlearning``` to the subcommands that supports it ( you can check out by invoking ```--help``` ).
+
 ## Documentation
 
 [Documentation](https://luet-lab.github.io/docs) is available, or

cmd/box.go

@@ -0,0 +1,36 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/box"
+
+	"github.com/spf13/cobra"
+)
+
+var boxGroupCmd = &cobra.Command{
+	Use:   "box [command] [OPTIONS]",
+	Short: "Manage luet boxes",
+}
+
+func init() {
+	RootCmd.AddCommand(boxGroupCmd)
+
+	boxGroupCmd.AddCommand(
+		NewBoxExecCommand(),
+	)
+}

cmd/box/exec.go

@@ -0,0 +1,81 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_box
+
+import (
+	"os"
+
+	b64 "encoding/base64"
+
+	"github.com/mudler/luet/pkg/box"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+func NewBoxExecCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "exec [OPTIONS]",
+		Short: "Execute a binary in a box",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			stdin, _ := cmd.Flags().GetBool("stdin")
+			stdout, _ := cmd.Flags().GetBool("stdout")
+			stderr, _ := cmd.Flags().GetBool("stderr")
+			rootfs, _ := cmd.Flags().GetString("rootfs")
+			base, _ := cmd.Flags().GetBool("decode")
+			entrypoint, _ := cmd.Flags().GetString("entrypoint")
+			envs, _ := cmd.Flags().GetStringArray("env")
+			mounts, _ := cmd.Flags().GetStringArray("mount")
+
+			if base {
+				var ss []string
+				for _, a := range args {
+					sDec, _ := b64.StdEncoding.DecodeString(a)
+					ss = append(ss, string(sDec))
+				}
+				//If the command to run is complex,using base64 to avoid bad input
+
+				args = ss
+			}
+			Info("Executing", args, "in", rootfs)
+
+			b := box.NewBox(entrypoint, args, mounts, envs, rootfs, stdin, stdout, stderr)
+			err := b.Run()
+			if err != nil {
+				Fatal(err)
+			}
+		},
+	}
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	ans.Flags().String("rootfs", path, "Rootfs path")
+	ans.Flags().Bool("stdin", false, "Attach to stdin")
+	ans.Flags().Bool("stdout", true, "Attach to stdout")
+	ans.Flags().Bool("stderr", true, "Attach to stderr")
+	ans.Flags().Bool("decode", false, "Base64 decode")
+	ans.Flags().StringArrayP("env", "e", []string{}, "Environment settings")
+	ans.Flags().StringArrayP("mount", "m", []string{}, "List of paths to bind-mount from the host")
+
+	ans.Flags().String("entrypoint", "/bin/sh", "Entrypoint command (/bin/sh)")
+
+	return ans
+}

cmd/build.go

@@ -15,18 +15,18 @@
 package cmd
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
 
-	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -45,11 +45,25 @@ var buildCmd = &cobra.Command{
 		viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
 		viper.BindPFlag("all", cmd.Flags().Lookup("all"))
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+
+		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
+		viper.BindPFlag("push", cmd.Flags().Lookup("push"))
+		viper.BindPFlag("pull", cmd.Flags().Lookup("pull"))
+		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
+
+		LuetCfg.Viper.BindPFlag("keep-exported-images", cmd.Flags().Lookup("keep-exported-images"))
+
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
 		clean := viper.GetBool("clean")
-		src := viper.GetString("tree")
+		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("destination")
 		concurrency := LuetCfg.GetGeneral().Concurrency
 		backendType := viper.GetString("backend")
@@ -58,6 +72,17 @@ var buildCmd = &cobra.Command{
 		all := viper.GetBool("all")
 		databaseType := viper.GetString("database")
 		compressionType := viper.GetString("compression")
+		imageRepository := viper.GetString("image-repository")
+		push := viper.GetBool("push")
+		pull := viper.GetBool("pull")
+		keepImages := viper.GetBool("keep-images")
+		nodeps := viper.GetBool("nodeps")
+		onlydeps := viper.GetBool("onlydeps")
+		keepExportedImages := viper.GetBool("keep-exported-images")
+		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
+		full, _ := cmd.Flags().GetBool("full")
+		skip, _ := cmd.Flags().GetBool("skip-if-metadata-exists")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 
 		compilerSpecs := compiler.NewLuetCompilationspecs()
 		var compilerBackend compiler.CompilerBackend
@@ -85,40 +110,74 @@ var buildCmd = &cobra.Command{
 
 		generalRecipe := tree.NewCompilerRecipe(db)
 
-		Info("Loading", src)
+		if len(treePaths) <= 0 {
+			Fatal("No tree path supplied!")
+		}
+
+		for _, src := range treePaths {
+			Info("Loading tree", src)
+
+			err := generalRecipe.Load(src)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+		}
+
 		Info("Building in", dst)
 
-		err := generalRecipe.Load(src)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
 		opts := compiler.NewDefaultCompilerOptions()
+		opts.SolverOptions = *LuetCfg.GetSolverOptions()
+		opts.ImageRepository = imageRepository
 		opts.Clean = clean
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts)
+		opts.PullFirst = pull
+		opts.KeepImg = keepImages
+		opts.Push = push
+		opts.OnlyDeps = onlydeps
+		opts.NoDeps = nodeps
+		opts.KeepImageExport = keepExportedImages
+		opts.SkipIfMetadataExists = skip
+		opts.PackageTargetOnly = onlyTarget
+
+		var solverOpts solver.Options
+		if concurrent {
+			solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
+		} else {
+			solverOpts = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+		}
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts, solverOpts)
 		luetCompiler.SetConcurrency(concurrency)
 		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
-		if !all {
+		if full {
+			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
+			if err != nil {
+				Fatal(err.Error())
+			}
+			for _, spec := range specs {
+				Info(":package: Selecting ", spec.GetPackage().GetName(), spec.GetPackage().GetVersion())
+
+				compilerSpecs.Add(spec)
+			}
+		} else if !all {
 			for _, a := range args {
-				gp, err := _gentoo.ParsePackageStr(a)
+
+				pack, err := helpers.ParsePackageStr(a)
 				if err != nil {
 					Fatal("Invalid package string ", a, ": ", err.Error())
 				}
 
-				if gp.Version == "" {
-					gp.Version = "0"
-					gp.Condition = _gentoo.PkgCondGreaterEqual
-				}
-
-				pack := &pkg.DefaultPackage{
-					Name: gp.Name,
-					Version: fmt.Sprintf("%s%s%s",
-						pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
-						gp.Version,
-						gp.VersionSuffix,
-					),
-					Category: gp.Category,
-					Uri:      make([]string, 0),
-				}
 				spec, err := luetCompiler.FromPackage(pack)
 				if err != nil {
 					Fatal("Error: " + err.Error())
@@ -168,14 +227,30 @@ func init() {
 		Fatal(err)
 	}
 	buildCmd.Flags().Bool("clean", true, "Build all packages without considering the packages present in the build directory")
-	buildCmd.Flags().String("tree", path, "Source luet tree")
+	buildCmd.Flags().StringSliceP("tree", "t", []string{}, "Path of the tree to use.")
 	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
 	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
 	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
-	buildCmd.Flags().Bool("all", false, "Build all packages in the tree")
+	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
+	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
+
 	buildCmd.Flags().String("destination", path, "Destination folder")
 	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
+	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+	buildCmd.Flags().Bool("push", false, "Push images to a hub")
+	buildCmd.Flags().Bool("pull", false, "Pull images from a hub")
+	buildCmd.Flags().Bool("keep-images", true, "Keep built docker images in the host")
+	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
+	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
+	buildCmd.Flags().Bool("keep-exported-images", false, "Keep exported images used during building")
+	buildCmd.Flags().Bool("skip-if-metadata-exists", false, "Skip package if metadata exists")
+	buildCmd.Flags().Bool("only-target-package", false, "Build packages of only the required target. Otherwise builds all the necessary ones not present in the destination")
+	buildCmd.Flags().String("solver-type", "", "Solver strategy")
+	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 
 	RootCmd.AddCommand(buildCmd)
 }

cmd/cleanup.go

@@ -35,9 +35,9 @@ var cleanupCmd = &cobra.Command{
 		var cleaned int = 0
 
 		// Check if cache dir exists
-		if helpers.Exists(helpers.GetSystemPkgsCacheDirPath()) {
+		if helpers.Exists(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
 
-			files, err := ioutil.ReadDir(helpers.GetSystemPkgsCacheDirPath())
+			files, err := ioutil.ReadDir(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
 			if err != nil {
 				Fatal("Error on read cachedir ", err.Error())
 			}
@@ -52,7 +52,7 @@ var cleanupCmd = &cobra.Command{
 				}
 
 				err := os.RemoveAll(
-					filepath.Join(helpers.GetSystemPkgsCacheDirPath(), file.Name()))
+					filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
 				if err != nil {
 					Fatal("Error on removing", file.Name())
 				}

cmd/config.go

@@ -19,14 +19,16 @@ import (
 	"fmt"
 
 	config "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
 
 	"github.com/spf13/cobra"
 )
 
 var configCmd = &cobra.Command{
-	Use:   "config",
-	Short: "Print config",
-	Long:  `Show luet configuration`,
+	Use:     "config",
+	Short:   "Print config",
+	Long:    `Show luet configuration`,
+	Aliases: []string{"c"},
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println(config.LuetCfg.GetLogging())
 		fmt.Println(config.LuetCfg.GetGeneral())
@@ -51,6 +53,23 @@ var configCmd = &cobra.Command{
 			}
 		}
 
+		if len(config.LuetCfg.ConfigProtectConfDir) > 0 {
+
+			// Load config protect configs
+			installer.LoadConfigProtectConfs(config.LuetCfg)
+
+			fmt.Println("config_protect_confdir:")
+			for _, dir := range config.LuetCfg.ConfigProtectConfDir {
+				fmt.Println("  - ", dir)
+			}
+
+			if len(config.LuetCfg.GetConfigProtectConfFiles()) > 0 {
+				fmt.Println("protect_conf_files:")
+				for _, file := range config.LuetCfg.GetConfigProtectConfFiles() {
+					fmt.Println("  - ", file.String())
+				}
+			}
+		}
 	},
 }
 

cmd/convert.go

@@ -28,7 +28,7 @@ import (
 )
 
 var convertCmd = &cobra.Command{
-	Use:   "convert",
+	Use:   "convert [portage-tree] [luet-tree]",
 	Short: "convert other package manager tree into luet",
 	Long:  `Parses external PM and produces a luet parsable tree`,
 	PreRun: func(cmd *cobra.Command, args []string) {

cmd/create-repo.go

@@ -40,7 +40,9 @@ var createrepoCmd = &cobra.Command{
 		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
 		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
 		viper.BindPFlag("tree-compression", cmd.Flags().Lookup("tree-compression"))
-		viper.BindPFlag("tree-path", cmd.Flags().Lookup("tree-path"))
+		viper.BindPFlag("tree-filename", cmd.Flags().Lookup("tree-filename"))
+		viper.BindPFlag("meta-compression", cmd.Flags().Lookup("meta-compression"))
+		viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename"))
 		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
 		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
 	},
@@ -48,7 +50,7 @@ var createrepoCmd = &cobra.Command{
 		var err error
 		var repo installer.Repository
 
-		tree := viper.GetString("tree")
+		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("output")
 		packages := viper.GetString("packages")
 		name := viper.GetString("name")
@@ -57,9 +59,14 @@ var createrepoCmd = &cobra.Command{
 		t := viper.GetString("type")
 		reset := viper.GetBool("reset-revision")
 		treetype := viper.GetString("tree-compression")
-		treepath := viper.GetString("tree-path")
+		treeName := viper.GetString("tree-filename")
+		metatype := viper.GetString("meta-compression")
+		metaName := viper.GetString("meta-filename")
 		source_repo := viper.GetString("repo")
 
+		treeFile := installer.NewDefaultTreeRepositoryFile()
+		metaFile := installer.NewDefaultMetaRepositoryFile()
+
 		if source_repo != "" {
 			// Search for system repository
 			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
@@ -67,8 +74,8 @@ var createrepoCmd = &cobra.Command{
 				Fatal("Error: " + err.Error())
 			}
 
-			if tree == "" {
-				tree = lrepo.TreePath
+			if len(treePaths) <= 0 {
+				treePaths = []string{lrepo.TreePath}
 			}
 
 			if t == "" {
@@ -80,12 +87,12 @@ var createrepoCmd = &cobra.Command{
 				lrepo.Urls,
 				lrepo.Priority,
 				packages,
-				tree,
+				treePaths,
 				pkg.NewInMemoryDatabase(false))
 
 		} else {
 			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
-				tree, pkg.NewInMemoryDatabase(false))
+				treePaths, pkg.NewInMemoryDatabase(false))
 		}
 
 		if err != nil {
@@ -93,13 +100,24 @@ var createrepoCmd = &cobra.Command{
 		}
 
 		if treetype != "" {
-			repo.SetTreeCompressionType(compiler.CompressionImplementation(treetype))
+			treeFile.SetCompressionType(compiler.CompressionImplementation(treetype))
 		}
 
-		if treepath != "" {
-			repo.SetTreePath(treepath)
+		if treeName != "" {
+			treeFile.SetFileName(treeName)
 		}
 
+		if metatype != "" {
+			metaFile.SetCompressionType(compiler.CompressionImplementation(metatype))
+		}
+
+		if metaName != "" {
+			metaFile.SetFileName(metaName)
+		}
+
+		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
+		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
+
 		err = repo.Write(dst, reset)
 		if err != nil {
 			Fatal("Error: " + err.Error())
@@ -113,7 +131,7 @@ func init() {
 		Fatal(err)
 	}
 	createrepoCmd.Flags().String("packages", path, "Packages folder (output from build)")
-	createrepoCmd.Flags().String("tree", path, "Source luet tree")
+	createrepoCmd.Flags().StringSliceP("tree", "t", []string{}, "Path of the source trees to use.")
 	createrepoCmd.Flags().String("output", path, "Destination folder")
 	createrepoCmd.Flags().String("name", "luet", "Repository name")
 	createrepoCmd.Flags().String("descr", "luet", "Repository description")
@@ -122,8 +140,10 @@ func init() {
 	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
 	createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
 
-	createrepoCmd.Flags().String("tree-compression", "none", "Compression alg: none, gzip")
-	createrepoCmd.Flags().String("tree-path", installer.TREE_TARBALL, "Repository tree filename")
+	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
+	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

cmd/database.go

@@ -0,0 +1,37 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/database"
+
+	"github.com/spf13/cobra"
+)
+
+var databaseGroupCmd = &cobra.Command{
+	Use:   "database [command] [OPTIONS]",
+	Short: "Manage system database (dangerous commands ahead!)",
+}
+
+func init() {
+	RootCmd.AddCommand(databaseGroupCmd)
+
+	databaseGroupCmd.AddCommand(
+		NewDatabaseCreateCommand(),
+		NewDatabaseRemoveCommand(),
+	)
+}

cmd/database/create.go

@@ -0,0 +1,78 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseCreateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "create <artifact_metadata1.yaml> <artifact_metadata1.yaml>",
+		Short: "Insert a package in the system DB",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			var systemDB pkg.PackageDatabase
+
+			for _, a := range args {
+				dat, err := ioutil.ReadFile(a)
+				if err != nil {
+					Fatal("Failed reading ", a, ": ", err.Error())
+				}
+				art, err := compiler.NewPackageArtifactFromYaml(dat)
+				if err != nil {
+					Fatal("Failed reading yaml ", a, ": ", err.Error())
+				}
+
+				if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+					systemDB = pkg.NewBoltDatabase(
+						filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+				} else {
+					systemDB = pkg.NewInMemoryDatabase(true)
+				}
+
+				files := art.GetFiles()
+
+				if _, err := systemDB.CreatePackage(art.GetCompileSpec().GetPackage()); err != nil {
+					Fatal("Failed to create ", a, ": ", err.Error())
+				}
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.GetCompileSpec().GetPackage().GetFingerPrint(), Files: files}); err != nil {
+					Fatal("Failed setting package files for ", a, ": ", err.Error())
+				}
+
+				Info(art.GetCompileSpec().GetPackage().HumanReadableString(), " created")
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/database/remove.go

@@ -0,0 +1,69 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"path/filepath"
+
+	. "github.com/mudler/luet/pkg/logger"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseRemoveCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "remove [package1] [package2] ...",
+		Short: "Remove a package from the system DB (forcefully - you normally don't want to do that)",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var systemDB pkg.PackageDatabase
+
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					Fatal("Invalid package string ", a, ": ", err.Error())
+				}
+
+				if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+					systemDB = pkg.NewBoltDatabase(
+						filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+				} else {
+					systemDB = pkg.NewInMemoryDatabase(true)
+				}
+
+				if err := systemDB.RemovePackage(pack); err != nil {
+					Fatal("Failed removing ", a, ": ", err.Error())
+				}
+
+				if err := systemDB.RemovePackageFiles(pack); err != nil {
+					Fatal("Failed removing files for ", a, ": ", err.Error())
+				}
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/exec.go

@@ -0,0 +1,86 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	b64 "encoding/base64"
+
+	"github.com/mudler/luet/pkg/box"
+	. "github.com/mudler/luet/pkg/logger"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+var execCmd = &cobra.Command{
+	Use:   "exec --rootfs /path [command]",
+	Short: "Execute a command in the rootfs context",
+	Long:  `Uses unshare technique and pivot root to execute a command inside a folder containing a valid rootfs`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+	},
+	// If you change this, look at pkg/box/exec that runs this command and adapt
+	Run: func(cmd *cobra.Command, args []string) {
+
+		stdin, _ := cmd.Flags().GetBool("stdin")
+		stdout, _ := cmd.Flags().GetBool("stdout")
+		stderr, _ := cmd.Flags().GetBool("stderr")
+		rootfs, _ := cmd.Flags().GetString("rootfs")
+		base, _ := cmd.Flags().GetBool("decode")
+
+		entrypoint, _ := cmd.Flags().GetString("entrypoint")
+		envs, _ := cmd.Flags().GetStringArray("env")
+		mounts, _ := cmd.Flags().GetStringArray("mount")
+
+		if base {
+			var ss []string
+			for _, a := range args {
+				sDec, _ := b64.StdEncoding.DecodeString(a)
+				ss = append(ss, string(sDec))
+			}
+			//If the command to run is complex,using base64 to avoid bad input
+
+			args = ss
+		}
+		Info("Executing", args, "in", rootfs)
+
+		b := box.NewBox(entrypoint, args, mounts, envs, rootfs, stdin, stdout, stderr)
+		err := b.Exec()
+		if err != nil {
+			Fatal(errors.Wrap(err, fmt.Sprintf("entrypoint: %s rootfs: %s", entrypoint, rootfs)))
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	execCmd.Hidden = true
+	execCmd.Flags().String("rootfs", path, "Rootfs path")
+	execCmd.Flags().Bool("stdin", false, "Attach to stdin")
+	execCmd.Flags().Bool("stdout", false, "Attach to stdout")
+	execCmd.Flags().Bool("stderr", false, "Attach to stderr")
+	execCmd.Flags().Bool("decode", false, "Base64 decode")
+
+	execCmd.Flags().StringArrayP("env", "e", []string{}, "Environment settings")
+	execCmd.Flags().StringArrayP("mount", "m", []string{}, "List of paths to bind-mount from the host")
+
+	execCmd.Flags().String("entrypoint", "/bin/sh", "Entrypoint command (/bin/sh)")
+
+	RootCmd.AddCommand(execCmd)
+}

cmd/helpers/cli.go

@@ -0,0 +1,78 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_helpers
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+
+	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
+	pkg "github.com/mudler/luet/pkg/package"
+	version "github.com/mudler/luet/pkg/versioner"
+)
+
+func CreateRegexArray(rgx []string) ([]*regexp.Regexp, error) {
+	ans := make([]*regexp.Regexp, len(rgx))
+	if len(rgx) > 0 {
+		for idx, reg := range rgx {
+			re := regexp.MustCompile(reg)
+			if re == nil {
+				return nil, errors.New("Invalid regex " + reg + "!")
+			}
+			ans[idx] = re
+		}
+	}
+
+	return ans, nil
+}
+
+func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
+	gp, err := _gentoo.ParsePackageStr(p)
+	if err != nil {
+		return nil, err
+	}
+	if gp.Version == "" {
+		gp.Version = "0"
+		gp.Condition = _gentoo.PkgCondGreaterEqual
+	}
+
+	pkgVersion := ""
+	if gp.VersionBuild != "" {
+		pkgVersion = fmt.Sprintf("%s%s%s+%s",
+			version.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+			gp.Version,
+			gp.VersionSuffix,
+			gp.VersionBuild,
+		)
+	} else {
+		pkgVersion = fmt.Sprintf("%s%s%s",
+			version.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+			gp.Version,
+			gp.VersionSuffix,
+		)
+	}
+
+	pack := &pkg.DefaultPackage{
+		Name:     gp.Name,
+		Category: gp.Category,
+		Version:  pkgVersion,
+		Uri:      make([]string, 0),
+	}
+
+	return pack, nil
+}

cmd/install.go

@@ -15,54 +15,45 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 
 	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
-	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/spf13/cobra"
 )
 
 var installCmd = &cobra.Command{
-	Use:   "install <pkg1> <pkg2> ...",
-	Short: "Install a package",
+	Use:     "install <pkg1> <pkg2> ...",
+	Short:   "Install a package",
+	Aliases: []string{"i"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
 		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Long: `Install packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
-		var toInstall []pkg.Package
+		var toInstall pkg.Packages
 		var systemDB pkg.PackageDatabase
 
 		for _, a := range args {
-			gp, err := _gentoo.ParsePackageStr(a)
+			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
 				Fatal("Invalid package string ", a, ": ", err.Error())
 			}
-
-			if gp.Version == "" {
-				gp.Version = "0"
-				gp.Condition = _gentoo.PkgCondGreaterEqual
-			}
-
-			pack := &pkg.DefaultPackage{
-				Name: gp.Name,
-				Version: fmt.Sprintf("%s%s%s",
-					pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
-					gp.Version,
-					gp.VersionSuffix,
-				),
-				Category: gp.Category,
-				Uri:      make([]string, 0),
-			}
 			toInstall = append(toInstall, pack)
 		}
 
@@ -76,12 +67,44 @@ var installCmd = &cobra.Command{
 			repos = append(repos, r)
 		}
 
-		inst := installer.NewLuetInstaller(LuetCfg.GetGeneral().Concurrency)
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps := LuetCfg.Viper.GetBool("nodeps")
+		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+		})
 		inst.Repositories(repos)
 
 		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
 			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(helpers.GetSystemRepoDatabaseDirPath(), "luet.db"))
+				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
 		} else {
 			systemDB = pkg.NewInMemoryDatabase(true)
 		}
@@ -100,6 +123,14 @@ func init() {
 	}
 	installCmd.Flags().String("system-dbpath", path, "System db path")
 	installCmd.Flags().String("system-target", path, "System rootpath")
+	installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	installCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	installCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	installCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/reclaim.go

@@ -0,0 +1,88 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	installer "github.com/mudler/luet/pkg/installer"
+
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+var reclaimCmd = &cobra.Command{
+	Use:   "reclaim",
+	Short: "Reclaim packages to Luet database from available repositories",
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+	},
+	Long: `Add packages to the systemdb if files belonging to packages 
+in available repositories exists in the target root.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		var systemDB pkg.PackageDatabase
+
+		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		force := LuetCfg.Viper.GetBool("force")
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			Force:                       force,
+			PreserveSystemEssentialData: true,
+		})
+		inst.Repositories(repos)
+
+		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+			systemDB = pkg.NewBoltDatabase(
+				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+		} else {
+			systemDB = pkg.NewInMemoryDatabase(true)
+		}
+		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Reclaim(system)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	reclaimCmd.Flags().String("system-dbpath", path, "System db path")
+	reclaimCmd.Flags().String("system-target", path, "System rootpath")
+	reclaimCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+
+	RootCmd.AddCommand(reclaimCmd)
+}

cmd/repo/list.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	. "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
 	installer "github.com/mudler/luet/pkg/installer"
 
 	. "github.com/logrusorgru/aurora"
@@ -69,7 +68,7 @@ func NewRepoListCommand() *cobra.Command {
 						repoText = Yellow(repo.Urls[0]).String()
 					}
 
-					repobasedir := helpers.GetRepoDatabaseDirPath(repo.Name)
+					repobasedir := LuetCfg.GetSystem().GetRepoDatabaseDirPath(repo.Name)
 					if repo.Cached {
 
 						r := installer.NewSystemRepository(repo)

cmd/repo/update.go

@@ -35,6 +35,7 @@ $> luet repo update
 # Update only repo1 and repo2
 $> luet repo update repo1 repo2
 `,
+		Aliases: []string{"up"},
 		PreRun: func(cmd *cobra.Command, args []string) {
 		},
 		Run: func(cmd *cobra.Command, args []string) {
@@ -62,7 +63,7 @@ $> luet repo update repo1 repo2
 
 			} else {
 				for _, repo := range LuetCfg.SystemRepositories {
-					if repo.Cached {
+					if repo.Cached && repo.Enable {
 						r := installer.NewSystemRepository(repo)
 						Spinner(32)
 						_, err := r.Sync(force)

cmd/root.go

@@ -16,6 +16,7 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -23,52 +24,80 @@ import (
 	"strings"
 
 	"github.com/marcsauter/single"
+	extensions "github.com/mudler/cobra-extensions"
 	config "github.com/mudler/luet/pkg/config"
+	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	repo "github.com/mudler/luet/pkg/repository"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 var cfgFile string
 var Verbose bool
+var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.5"
+	LuetCLIVersion = "0.8.10"
 	LuetEnvPrefix  = "LUET"
 )
 
+// Build time and commit information.
+//
+// ⚠️ WARNING: should only be set by "-ldflags".
+var (
+	BuildTime   string
+	BuildCommit string
+)
+
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
 	Use:     "luet",
 	Short:   "Package manager for the XXth century!",
 	Long:    `Package manager which uses containers to build packages`,
-	Version: LuetCLIVersion,
+	Version: fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime),
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		err := LoadConfig(config.LuetCfg)
 		if err != nil {
 			Fatal("failed to load configuration:", err.Error())
 		}
+		// Initialize tmpdir prefix. TODO: Move this with LoadConfig
+		// directly on sub command to ensure the creation only when it's
+		// needed.
+		err = config.LuetCfg.GetSystem().InitTmpDir()
+		if err != nil {
+			Fatal("failed on init tmp basedir:", err.Error())
+		}
 	},
+	PersistentPostRun: func(cmd *cobra.Command, args []string) {
+		// Cleanup all tmp directories used by luet
+		err := config.LuetCfg.GetSystem().CleanupTmpDir()
+		if err != nil {
+			Warning("failed on cleanup tmpdir:", err.Error())
+		}
+	},
+	SilenceErrors: true,
 }
 
 func LoadConfig(c *config.LuetConfig) error {
 	// If a config file is found, read it in.
-	if err := c.Viper.ReadInConfig(); err != nil {
-		Warning(err)
-	}
+	c.Viper.ReadInConfig()
 
 	err := c.Viper.Unmarshal(&config.LuetCfg)
 	if err != nil {
 		return err
 	}
 
+	noSpinner := c.Viper.GetBool("no_spinner")
+
+	InitAurora()
+	if !noSpinner {
+		NewSpinner()
+	}
+
 	Debug("Using config file:", c.Viper.ConfigFileUsed())
 
-	NewSpinner()
-
-	if c.GetLogging().Path != "" {
+	if c.GetLogging().EnableLogFile && c.GetLogging().Path != "" {
 		// Init zap logger
 		err = ZapLogger()
 		if err != nil {
@@ -88,19 +117,27 @@ func LoadConfig(c *config.LuetConfig) error {
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
-	// XXX: This is mostly from scratch images.
+
 	if os.Getenv("LUET_NOLOCK") != "true" {
-		s := single.New("luet")
-		if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
-			Fatal("another instance of the app is already running, exiting")
-		} else if err != nil {
-			// Another error occurred, might be worth handling it as well
-			Fatal("failed to acquire exclusive app lock:", err.Error())
+		if len(os.Args) > 1 {
+			for _, lockedCmd := range LockedCommands {
+				if os.Args[1] == lockedCmd {
+					s := single.New("luet")
+					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
+						Fatal("another instance of the app is already running, exiting")
+					} else if err != nil {
+						// Another error occurred, might be worth handling it as well
+						Fatal("failed to acquire exclusive app lock:", err.Error())
+					}
+					defer s.TryUnlock()
+					break
+				}
+			}
 		}
-		defer s.TryUnlock()
 	}
+
 	if err := RootCmd.Execute(); err != nil {
-		Error(err)
+		fmt.Println(err)
 		os.Exit(-1)
 	}
 }
@@ -109,45 +146,83 @@ func init() {
 	cobra.OnInitialize(initConfig)
 	pflags := RootCmd.PersistentFlags()
 	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
-	pflags.BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
+	pflags.BoolP("debug", "d", false, "verbose output")
 	pflags.Bool("fatal", false, "Enables Warnings to exit")
+	pflags.Bool("enable-logfile", false, "Enable log to file")
+	pflags.Bool("no-spinner", false, "Disable spinner.")
+	pflags.Bool("color", config.LuetCfg.GetLogging().Color, "Enable/Disable color.")
+	pflags.Bool("emoji", config.LuetCfg.GetLogging().EnableEmoji, "Enable/Disable emoji.")
+	pflags.Bool("skip-config-protect", config.LuetCfg.ConfigProtectSkip,
+		"Disable config protect analysis.")
+	pflags.StringP("logfile", "l", config.LuetCfg.GetLogging().Path,
+		"Logfile path. Empty value disable log to file.")
 
-	u, err := user.Current()
+	// os/user doesn't work in from scratch environments.
+	// Check if i can retrieve user informations.
+	_, err := user.Current()
 	if err != nil {
-		Fatal("failed to retrieve user identity:", err.Error())
+		Warning("failed to retrieve user identity:", err.Error())
 	}
-	sameOwner := false
-	if u.Uid == "0" {
-		sameOwner = true
-	}
-	pflags.Bool("same-owner", sameOwner, "Maintain same owner on uncompress.")
+	pflags.Bool("same-owner", config.LuetCfg.GetGeneral().SameOwner, "Maintain same owner on uncompress.")
 	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
 
-	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
-	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("verbose"))
+	config.LuetCfg.Viper.BindPFlag("logging.color", pflags.Lookup("color"))
+	config.LuetCfg.Viper.BindPFlag("logging.enable_emoji", pflags.Lookup("emoji"))
+	config.LuetCfg.Viper.BindPFlag("logging.enable_logfile", pflags.Lookup("enable-logfile"))
+	config.LuetCfg.Viper.BindPFlag("logging.path", pflags.Lookup("logfile"))
+
 	config.LuetCfg.Viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
+	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("debug"))
 	config.LuetCfg.Viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
+	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
+	// Currently I maintain this only from cli.
+	config.LuetCfg.Viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))
+	config.LuetCfg.Viper.BindPFlag("config_protect_skip", pflags.Lookup("skip-config-protect"))
+
+	// Extensions must be binary with the "luet-" prefix to be able to be shown in the help.
+	// we also accept extensions in the relative path where luet is being started, "extensions/"
+	exts := extensions.Discover("luet", "extensions")
+	for _, ex := range exts {
+		cobraCmd := ex.CobraCommand()
+		RootCmd.AddCommand(cobraCmd)
+	}
+
 }
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
+	// Luet support these priorities on read configuration file:
+	// - command line option (if available)
+	// - $PWD/.luet.yaml
+	// - $HOME/.luet.yaml
+	// - /etc/luet/luet.yaml
+	//
+	// Note: currently a single viper instance support only one config name.
 
-	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
-	if err != nil {
-		Error(err)
-		os.Exit(1)
-	}
 	viper.SetEnvPrefix(LuetEnvPrefix)
 	viper.SetConfigType("yaml")
-	viper.SetConfigName(".luet") // name of config file (without extension)
-	if cfgFile != "" {           // enable ability to specify config file via flag
-		Info(">>> cfgFile: ", cfgFile)
+
+	if cfgFile != "" { // enable ability to specify config file via flag
 		viper.SetConfigFile(cfgFile)
 	} else {
-		viper.AddConfigPath(dir)
-		viper.AddConfigPath(".")
-		viper.AddConfigPath("$HOME")
-		viper.AddConfigPath("/etc/luet")
+		// Retrieve pwd directory
+		pwdDir, err := os.Getwd()
+		if err != nil {
+			Error(err)
+			os.Exit(1)
+		}
+		homeDir := helpers.GetHomeDir()
+
+		if helpers.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && helpers.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
+			viper.AddConfigPath(".")
+			if homeDir != "" {
+				viper.AddConfigPath(homeDir)
+			}
+			viper.SetConfigName(".luet")
+		} else {
+			viper.SetConfigName("luet")
+			viper.AddConfigPath("/etc/luet")
+		}
 	}
 
 	viper.AutomaticEnv() // read in environment variables that match

cmd/search.go

@@ -15,36 +15,73 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
 
+	"github.com/ghodss/yaml"
 	. "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
-
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
+type PackageResult struct {
+	Name       string `json:"name"`
+	Category   string `json:"category"`
+	Version    string `json:"version"`
+	Repository string `json:"repository"`
+	Hidden     bool   `json:"hidden"`
+}
+
+type Results struct {
+	Packages []PackageResult `json:"packages"`
+}
+
 var searchCmd = &cobra.Command{
-	Use:   "search <term>",
-	Short: "Search packages",
-	Long:  `Search for installed and available packages`,
+	Use:     "search <term>",
+	Short:   "Search packages",
+	Long:    `Search for installed and available packages`,
+	Aliases: []string{"s"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
 		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var systemDB pkg.PackageDatabase
-
+		var results Results
 		if len(args) != 1 {
 			Fatal("Wrong number of arguments (expected 1)")
 		}
-		installed := viper.GetBool("installed")
+
+		hidden, _ := cmd.Flags().GetBool("hidden")
+
+		installed := LuetCfg.Viper.GetBool("installed")
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		searchWithLabel, _ := cmd.Flags().GetBool("by-label")
+		searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex")
+		revdeps, _ := cmd.Flags().GetBool("revdeps")
+
+		out, _ := cmd.Flags().GetString("output")
+		if out != "terminal" {
+			LuetCfg.GetLogging().SetLogLevel("error")
+		}
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
 		if !installed {
 
@@ -57,39 +94,132 @@ var searchCmd = &cobra.Command{
 				repos = append(repos, r)
 			}
 
-			inst := installer.NewLuetInstaller(LuetCfg.GetGeneral().Concurrency)
+			inst := installer.NewLuetInstaller(
+				installer.LuetInstallerOptions{
+					Concurrency:   LuetCfg.GetGeneral().Concurrency,
+					SolverOptions: *LuetCfg.GetSolverOptions(),
+				},
+			)
 			inst.Repositories(repos)
 			synced, err := inst.SyncRepositories(false)
 			if err != nil {
 				Fatal("Error: " + err.Error())
 			}
 
-			Info("--- Search results: ---")
+			Info("--- Search results (" + args[0] + "): ---")
 
-			matches := synced.Search(args[0])
+			matches := []installer.PackageMatch{}
+			if searchWithLabel {
+				matches = synced.SearchLabel(args[0])
+			} else if searchWithLabelMatch {
+				matches = synced.SearchLabelMatch(args[0])
+			} else {
+				matches = synced.Search(args[0])
+			}
 			for _, m := range matches {
-				Info(":package:", m.Package.GetCategory(), m.Package.GetName(),
-					m.Package.GetVersion(), "repository:", m.Repo.GetName())
+				if !revdeps {
+					if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
+						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
+						results.Packages = append(results.Packages,
+							PackageResult{
+								Name:       m.Package.GetName(),
+								Version:    m.Package.GetVersion(),
+								Category:   m.Package.GetCategory(),
+								Repository: m.Repo.GetName(),
+								Hidden:     m.Package.IsHidden(),
+							})
+					}
+				} else {
+					visited := make(map[string]interface{})
+					for _, revdep := range m.Package.ExpandedRevdeps(m.Repo.GetTree().GetDatabase(), visited) {
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: m.Repo.GetName(),
+									Hidden:     revdep.IsHidden(),
+								})
+						}
+					}
+				}
 			}
 		} else {
 
 			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
 				systemDB = pkg.NewBoltDatabase(
-					filepath.Join(helpers.GetSystemRepoDatabaseDirPath(), "luet.db"))
+					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
 			} else {
 				systemDB = pkg.NewInMemoryDatabase(true)
 			}
 			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
-			var term = regexp.MustCompile(args[0])
 
-			for _, k := range system.Database.GetPackages() {
-				pack, err := system.Database.GetPackage(k)
-				if err == nil && term.MatchString(pack.GetName()) {
-					Info(":package:", pack.GetCategory(), pack.GetName(), pack.GetVersion())
+			var err error
+			iMatches := pkg.Packages{}
+			if searchWithLabel {
+				iMatches, err = system.Database.FindPackageLabel(args[0])
+			} else if searchWithLabelMatch {
+				iMatches, err = system.Database.FindPackageLabelMatch(args[0])
+			} else {
+				iMatches, err = system.Database.FindPackageMatch(args[0])
+			}
+
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
+			for _, pack := range iMatches {
+				if !revdeps {
+					if !pack.IsHidden() || pack.IsHidden() && hidden {
+						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+						results.Packages = append(results.Packages,
+							PackageResult{
+								Name:       pack.GetName(),
+								Version:    pack.GetVersion(),
+								Category:   pack.GetCategory(),
+								Repository: "system",
+								Hidden:     pack.IsHidden(),
+							})
+					}
+				} else {
+					visited := make(map[string]interface{})
+
+					for _, revdep := range pack.ExpandedRevdeps(system.Database, visited) {
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: "system",
+									Hidden:     revdep.IsHidden(),
+								})
+						}
+					}
 				}
 			}
 		}
 
+		y, err := yaml.Marshal(results)
+		if err != nil {
+			fmt.Printf("err: %v\n", err)
+			return
+		}
+		switch out {
+		case "yaml":
+			fmt.Println(string(y))
+		case "json":
+			j2, err := yaml.YAMLToJSON(y)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			fmt.Println(string(j2))
+		}
+
 	},
 }
 
@@ -101,5 +231,15 @@ func init() {
 	searchCmd.Flags().String("system-dbpath", path, "System db path")
 	searchCmd.Flags().String("system-target", path, "System rootpath")
 	searchCmd.Flags().Bool("installed", false, "Search between system packages")
+	searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	searchCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+	searchCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	searchCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	searchCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	searchCmd.Flags().Bool("by-label", false, "Search packages through label")
+	searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex")
+	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
+
 	RootCmd.AddCommand(searchCmd)
 }

cmd/tree.go

@@ -0,0 +1,38 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/tree"
+
+	"github.com/spf13/cobra"
+)
+
+var treeGroupCmd = &cobra.Command{
+	Use:   "tree [command] [OPTIONS]",
+	Short: "Tree operations",
+}
+
+func init() {
+	RootCmd.AddCommand(treeGroupCmd)
+
+	treeGroupCmd.AddCommand(
+		NewTreePkglistCommand(),
+		NewTreeValidateCommand(),
+		NewTreeBumpCommand(),
+	)
+}

cmd/tree/bump.go

@@ -0,0 +1,88 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"fmt"
+
+	. "github.com/mudler/luet/pkg/logger"
+	spectooling "github.com/mudler/luet/pkg/spectooling"
+	tree "github.com/mudler/luet/pkg/tree"
+	version "github.com/mudler/luet/pkg/versioner"
+
+	"github.com/spf13/cobra"
+)
+
+func NewTreeBumpCommand() *cobra.Command {
+
+	var ans = &cobra.Command{
+		Use:   "bump [OPTIONS]",
+		Short: "Bump a new package build version.",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			df, _ := cmd.Flags().GetString("definition-file")
+			if df == "" {
+				Fatal("Mandatory definition.yaml path missing.")
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			spec, _ := cmd.Flags().GetString("definition-file")
+			toStdout, _ := cmd.Flags().GetBool("to-stdout")
+			pkgVersion, _ := cmd.Flags().GetString("pkg-version")
+			pack, err := tree.ReadDefinitionFile(spec)
+			if err != nil {
+				Fatal(err.Error())
+			}
+
+			if pkgVersion != "" {
+				validator := &version.WrappedVersioner{}
+				err := validator.Validate(pkgVersion)
+				if err != nil {
+					Fatal("Invalid version string: " + err.Error())
+				}
+				pack.SetVersion(pkgVersion)
+			} else {
+				// Retrieve version build section with Gentoo parser
+				err = pack.BumpBuildVersion()
+				if err != nil {
+					Fatal("Error on increment build version: " + err.Error())
+				}
+			}
+			if toStdout {
+				data, err := spectooling.NewDefaultPackageSanitized(&pack).Yaml()
+				if err != nil {
+					Fatal("Error on yaml conversion: " + err.Error())
+				}
+				fmt.Println(string(data))
+			} else {
+
+				err = tree.WriteDefinitionFile(&pack, spec)
+				if err != nil {
+					Fatal("Error on write definition file: " + err.Error())
+				}
+
+				fmt.Printf("Bumped package %s/%s-%s.\n", pack.Category, pack.Name, pack.Version)
+			}
+		},
+	}
+
+	ans.Flags().StringP("pkg-version", "p", "", "Set a specific package version")
+	ans.Flags().StringP("definition-file", "f", "", "Path of the definition to bump.")
+	ans.Flags().BoolP("to-stdout", "o", false, "Bump package to output.")
+
+	return ans
+}

cmd/tree/pkglist.go

@@ -0,0 +1,284 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"fmt"
+	"sort"
+
+	//. "github.com/mudler/luet/pkg/config"
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/spf13/cobra"
+)
+
+type TreePackageResult struct {
+	Name     string `json:"name"`
+	Category string `json:"category"`
+	Version  string `json:"version"`
+	Path     string `json:"path"`
+}
+
+type TreeResults struct {
+	Packages []TreePackageResult `json:"packages"`
+}
+
+func pkgDetail(pkg pkg.Package) string {
+	ans := fmt.Sprintf(`
+  @@ Package: %s/%s-%s
+     Description: %s
+     License:     %s`,
+		pkg.GetCategory(), pkg.GetName(), pkg.GetVersion(),
+		pkg.GetDescription(), pkg.GetLicense())
+
+	for idx, u := range pkg.GetURI() {
+		if idx == 0 {
+			ans += fmt.Sprintf("     URLs:        %s", u)
+		} else {
+			ans += fmt.Sprintf("                  %s", u)
+		}
+	}
+
+	return ans
+}
+
+func NewTreePkglistCommand() *cobra.Command {
+	var excludes []string
+	var matches []string
+
+	var ans = &cobra.Command{
+		Use:   "pkglist [OPTIONS]",
+		Short: "List of the packages found in tree.",
+		Args:  cobra.NoArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			t, _ := cmd.Flags().GetStringArray("tree")
+			if len(t) == 0 {
+				Fatal("Mandatory tree param missing.")
+			}
+
+			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
+			if revdeps && deps {
+				Fatal("Both revdeps and deps option used. Choice only one.")
+			}
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var results TreeResults
+			var depSolver solver.PackageSolver
+
+			treePath, _ := cmd.Flags().GetStringArray("tree")
+			verbose, _ := cmd.Flags().GetBool("verbose")
+			buildtime, _ := cmd.Flags().GetBool("buildtime")
+			full, _ := cmd.Flags().GetBool("full")
+			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
+
+			out, _ := cmd.Flags().GetString("output")
+			if out != "terminal" {
+				LuetCfg.GetLogging().SetLogLevel("error")
+			}
+
+			var reciper tree.Builder
+			if buildtime {
+				reciper = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			} else {
+				reciper = tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
+			}
+
+			for _, t := range treePath {
+				err := reciper.Load(t)
+				if err != nil {
+					Fatal("Error on load tree ", err)
+				}
+			}
+
+			if deps {
+				emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+
+				depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
+					reciper.GetDatabase(),
+					emptyInstallationDb)
+
+			}
+
+			regExcludes, err := helpers.CreateRegexArray(excludes)
+			if err != nil {
+				Fatal(err.Error())
+			}
+			regMatches, err := helpers.CreateRegexArray(matches)
+			if err != nil {
+				Fatal(err.Error())
+			}
+
+			plist := make([]string, 0)
+			for _, p := range reciper.GetDatabase().World() {
+				pkgstr := ""
+				addPkg := true
+				if full {
+					pkgstr = pkgDetail(p)
+				} else if verbose {
+					pkgstr = p.HumanReadableString()
+				} else {
+					pkgstr = fmt.Sprintf("%s/%s", p.GetCategory(), p.GetName())
+				}
+
+				if len(matches) > 0 {
+					matched := false
+					for _, rgx := range regMatches {
+						if rgx.MatchString(pkgstr) {
+							matched = true
+							break
+						}
+					}
+					if !matched {
+						addPkg = false
+					}
+				}
+
+				if len(excludes) > 0 && addPkg {
+					for _, rgx := range regExcludes {
+						if rgx.MatchString(pkgstr) {
+							addPkg = false
+							break
+						}
+					}
+				}
+
+				if addPkg {
+					if revdeps {
+
+						visited := make(map[string]interface{})
+						for _, revdep := range p.ExpandedRevdeps(reciper.GetDatabase(), visited) {
+							if full {
+								pkgstr = pkgDetail(revdep)
+							} else if verbose {
+								pkgstr = revdep.HumanReadableString()
+							} else {
+								pkgstr = fmt.Sprintf("%s/%s", revdep.GetCategory(), revdep.GetName())
+							}
+							plist = append(plist, pkgstr)
+							results.Packages = append(results.Packages, TreePackageResult{
+								Name:     revdep.GetName(),
+								Version:  revdep.GetVersion(),
+								Category: revdep.GetCategory(),
+								Path:     revdep.GetPath(),
+							})
+						}
+					} else if deps {
+
+						Spinner(32)
+						solution, err := depSolver.Install(pkg.Packages{p})
+						if err != nil {
+							Fatal(err.Error())
+						}
+						ass := solution.SearchByName(p.GetPackageName())
+						solution, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+						if err != nil {
+							Fatal(err.Error())
+						}
+						SpinnerStop()
+
+						for _, pa := range solution {
+
+							if pa.Value {
+								// Exclude itself
+								if pa.Package.GetName() == p.GetName() && pa.Package.GetCategory() == p.GetCategory() {
+									continue
+								}
+
+								if full {
+									pkgstr = pkgDetail(pa.Package)
+								} else if verbose {
+									pkgstr = pa.Package.HumanReadableString()
+								} else {
+									pkgstr = fmt.Sprintf("%s/%s", pa.Package.GetCategory(), pa.Package.GetName())
+								}
+								plist = append(plist, pkgstr)
+								results.Packages = append(results.Packages, TreePackageResult{
+									Name:     pa.Package.GetName(),
+									Version:  pa.Package.GetVersion(),
+									Category: pa.Package.GetCategory(),
+									Path:     pa.Package.GetPath(),
+								})
+							}
+
+						}
+
+					} else {
+
+						plist = append(plist, pkgstr)
+						results.Packages = append(results.Packages, TreePackageResult{
+							Name:     p.GetName(),
+							Version:  p.GetVersion(),
+							Category: p.GetCategory(),
+							Path:     p.GetPath(),
+						})
+
+					}
+
+				}
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			default:
+				if !deps {
+					sort.Strings(plist)
+				}
+				for _, p := range plist {
+					fmt.Println(p)
+				}
+			}
+
+		},
+	}
+
+	ans.Flags().BoolP("buildtime", "b", false, "Build time match")
+	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+	ans.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	ans.Flags().Bool("deps", false, "Search package dependencies")
+
+	ans.Flags().BoolP("verbose", "v", false, "Add package version")
+	ans.Flags().BoolP("full", "f", false, "Show package detail")
+	ans.Flags().StringArrayP("tree", "t", []string{}, "Path of the tree to use.")
+	ans.Flags().StringSliceVarP(&matches, "matches", "m", []string{},
+		"Include only matched packages from list. (Use string as regex).")
+	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},
+		"Exclude matched packages from list. (Use string as regex).")
+
+	return ans
+}

cmd/tree/validate.go

@@ -0,0 +1,474 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"sync"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/spf13/cobra"
+)
+
+type ValidateOpts struct {
+	WithSolver    bool
+	OnlyRuntime   bool
+	OnlyBuildtime bool
+	RegExcludes   []*regexp.Regexp
+	RegMatches    []*regexp.Regexp
+	Excludes      []string
+	Matches       []string
+
+	// Runtime validate stuff
+	RuntimeCacheDeps *pkg.InMemoryDatabase
+	RuntimeReciper   *tree.InstallerRecipe
+
+	// Buildtime validate stuff
+	BuildtimeCacheDeps *pkg.InMemoryDatabase
+	BuildtimeReciper   *tree.CompilerRecipe
+
+	Mutex      sync.Mutex
+	BrokenPkgs int
+	BrokenDeps int
+
+	Errors []error
+}
+
+func (o *ValidateOpts) IncrBrokenPkgs() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenPkgs++
+}
+
+func (o *ValidateOpts) IncrBrokenDeps() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenDeps++
+}
+
+func (o *ValidateOpts) AddError(err error) {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.Errors = append(o.Errors, err)
+}
+
+func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, reciper tree.Builder, cacheDeps *pkg.InMemoryDatabase) error {
+	var errstr string
+	var ans error
+
+	var depSolver solver.PackageSolver
+
+	if opts.WithSolver {
+		emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+		depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple},pkg.NewInMemoryDatabase(false),
+			reciper.GetDatabase(),
+			emptyInstallationDb)
+	}
+
+	found, err := reciper.GetDatabase().FindPackages(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  ">=0",
+		},
+	)
+
+	if err != nil || len(found) < 1 {
+		if err != nil {
+			errstr = err.Error()
+		} else {
+			errstr = "No packages"
+		}
+		Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			errstr,
+		))
+
+		opts.IncrBrokenDeps()
+
+		return errors.New(
+			fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+				checkType,
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				errstr,
+			))
+	}
+
+	// Ensure that we use the right package from right recipier for deps
+	pReciper, err := reciper.GetDatabase().FindPackage(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  p.GetVersion(),
+		},
+	)
+	if err != nil {
+		errstr = fmt.Sprintf("[%9s] %s/%s-%s: Error on retrieve package - %s.",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			err.Error(),
+		)
+		Error(errstr)
+
+		return errors.New(errstr)
+	}
+	p = pReciper
+
+	pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
+		p.GetVersion())
+
+	validpkg := true
+
+	if len(opts.Matches) > 0 {
+		matched := false
+		for _, rgx := range opts.RegMatches {
+			if rgx.MatchString(pkgstr) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return nil
+		}
+	}
+
+	if len(opts.Excludes) > 0 {
+		excluded := false
+		for _, rgx := range opts.RegExcludes {
+			if rgx.MatchString(pkgstr) {
+				excluded = true
+				break
+			}
+		}
+
+		if excluded {
+			return nil
+		}
+	}
+
+	Info(fmt.Sprintf("[%9s] Checking package ", checkType)+
+		fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
+		"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
+
+	all := p.GetRequires()
+	all = append(all, p.GetConflicts()...)
+	for idx, r := range all {
+
+		var deps pkg.Packages
+		var err error
+		if r.IsSelector() {
+			deps, err = reciper.GetDatabase().FindPackages(
+				&pkg.DefaultPackage{
+					Name:     r.GetName(),
+					Category: r.GetCategory(),
+					Version:  r.GetVersion(),
+				},
+			)
+		} else {
+			deps = append(deps, r)
+		}
+
+		if err != nil || len(deps) < 1 {
+			if err != nil {
+				errstr = err.Error()
+			} else {
+				errstr = "No packages"
+			}
+			Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+				checkType,
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				r.GetCategory(), r.GetName(), r.GetVersion(),
+				errstr,
+			))
+
+			opts.IncrBrokenDeps()
+
+			ans = errors.New(
+				fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+					checkType,
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					r.GetCategory(), r.GetName(), r.GetVersion(),
+					errstr))
+
+			validpkg = false
+
+		} else {
+
+			Debug(fmt.Sprintf("[%9s] Find packages for dep", checkType),
+				fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+
+			if opts.WithSolver {
+
+				Info(fmt.Sprintf("[%9s]  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
+					checkType,
+					idx+1, len(all),
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					r.GetCategory(), r.GetName(), r.GetVersion(),
+				))
+
+				// Check if the solver is already been done for the deep
+				_, err := cacheDeps.Get(r.HashFingerprint(""))
+				if err == nil {
+					Debug(fmt.Sprintf("[%9s]  :direct_hit: Cache Hit for dep", checkType),
+						fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+					continue
+				}
+
+				Spinner(32)
+				solution, err := depSolver.Install(pkg.Packages{r})
+				ass := solution.SearchByName(r.GetPackageName())
+				if err == nil {
+					_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+				}
+				SpinnerStop()
+
+				if err != nil {
+
+					Error(fmt.Sprintf("[%9s] %s/%s-%s: solver broken for dep %s/%s-%s - %s",
+						checkType,
+						p.GetCategory(), p.GetName(), p.GetVersion(),
+						r.GetCategory(), r.GetName(), r.GetVersion(),
+						err.Error(),
+					))
+
+					ans = errors.New(
+						fmt.Sprintf("[%9s] %s/%s-%s: solver broken for Dep %s/%s-%s - %s",
+							checkType,
+							p.GetCategory(), p.GetName(), p.GetVersion(),
+							r.GetCategory(), r.GetName(), r.GetVersion(),
+							err.Error()))
+
+					opts.IncrBrokenDeps()
+					validpkg = false
+				}
+
+				// Register the key
+				cacheDeps.Set(r.HashFingerprint(""), "1")
+
+			}
+		}
+
+	}
+
+	if !validpkg {
+		opts.IncrBrokenPkgs()
+	}
+
+	return ans
+}
+
+func validateWorker(i int,
+	wg *sync.WaitGroup,
+	c <-chan pkg.Package,
+	opts *ValidateOpts) {
+
+	defer wg.Done()
+
+	for p := range c {
+
+		if opts.OnlyBuildtime {
+			// Check buildtime compiler/deps
+			err := validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+		} else if opts.OnlyRuntime {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+		} else {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+			// Check buildtime compiler/deps
+			err = validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+			}
+
+		}
+
+	}
+}
+
+func initOpts(opts *ValidateOpts, onlyRuntime, onlyBuildtime, withSolver bool, treePaths []string) {
+	var err error
+
+	opts.OnlyBuildtime = onlyBuildtime
+	opts.OnlyRuntime = onlyRuntime
+	opts.WithSolver = withSolver
+	opts.RuntimeReciper = nil
+	opts.BuildtimeReciper = nil
+	opts.BrokenPkgs = 0
+	opts.BrokenDeps = 0
+
+	if onlyBuildtime {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+	} else if onlyRuntime {
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	} else {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	}
+
+	opts.RuntimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+	opts.BuildtimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+
+	for _, treePath := range treePaths {
+		Info(fmt.Sprintf("Loading :deciduous_tree: %s...", treePath))
+		if opts.BuildtimeReciper != nil {
+			err = opts.BuildtimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
+		}
+		if opts.RuntimeReciper != nil {
+			err = opts.RuntimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
+		}
+	}
+
+	opts.RegExcludes, err = helpers.CreateRegexArray(opts.Excludes)
+	if err != nil {
+		Fatal(err.Error())
+	}
+	opts.RegMatches, err = helpers.CreateRegexArray(opts.Matches)
+	if err != nil {
+		Fatal(err.Error())
+	}
+
+}
+
+func NewTreeValidateCommand() *cobra.Command {
+	var excludes []string
+	var matches []string
+	var treePaths []string
+	var opts ValidateOpts
+
+	var ans = &cobra.Command{
+		Use:   "validate [OPTIONS]",
+		Short: "Validate a tree or a list of packages",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
+			if len(treePaths) < 1 {
+				Fatal("Mandatory tree param missing.")
+			}
+			if onlyRuntime && onlyBuildtime {
+				Fatal("Both --only-runtime and --only-buildtime options are not possibile.")
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var reciper tree.Builder
+
+			concurrency := LuetCfg.GetGeneral().Concurrency
+
+			withSolver, _ := cmd.Flags().GetBool("with-solver")
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
+			opts.Excludes = excludes
+			opts.Matches = matches
+			initOpts(&opts, onlyRuntime, onlyBuildtime, withSolver, treePaths)
+
+			// We need at least one valid reciper for get list of the packages.
+			if onlyBuildtime {
+				reciper = opts.BuildtimeReciper
+			} else {
+				reciper = opts.RuntimeReciper
+			}
+
+			all := make(chan pkg.Package)
+
+			var wg = new(sync.WaitGroup)
+
+			for i := 0; i < concurrency; i++ {
+				wg.Add(1)
+				go validateWorker(i, wg, all, &opts)
+			}
+			for _, p := range reciper.GetDatabase().World() {
+				all <- p
+			}
+			close(all)
+
+			// Wait separately and once done close the channel
+			go func() {
+				wg.Wait()
+			}()
+
+			stringerrs := []string{}
+			for _, e := range opts.Errors {
+				stringerrs = append(stringerrs, e.Error())
+			}
+			sort.Strings(stringerrs)
+			for _, e := range stringerrs {
+				fmt.Println(e)
+			}
+
+			// fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
+			if len(stringerrs) != 0 {
+				Error(fmt.Sprintf("Found %d broken packages and %d broken deps.",
+					opts.BrokenPkgs, opts.BrokenDeps))
+				Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
+			} else {
+				Info("All good! :white_check_mark:")
+				os.Exit(0)
+			}
+		},
+	}
+
+	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
+	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")
+	ans.Flags().BoolP("with-solver", "s", false,
+		"Enable check of requires also with solver.")
+	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{},
+		"Path of the tree to use.")
+	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},
+		"Exclude matched packages from analysis. (Use string as regex).")
+	ans.Flags().StringSliceVarP(&matches, "matches", "m", []string{},
+		"Analyze only matched packages. (Use string as regex).")
+
+	return ans
+}

cmd/uninstall.go

@@ -15,56 +15,79 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 
-	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/spf13/cobra"
 )
 
 var uninstallCmd = &cobra.Command{
-	Use:   "uninstall <pkg> <pkg2> ...",
-	Short: "Uninstall a package or a list of packages",
-	Long:  `Uninstall packages`,
+	Use:     "uninstall <pkg> <pkg2> ...",
+	Short:   "Uninstall a package or a list of packages",
+	Long:    `Uninstall packages`,
+	Aliases: []string{"rm", "un"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
 		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var systemDB pkg.PackageDatabase
 
 		for _, a := range args {
-			gp, err := _gentoo.ParsePackageStr(a)
+
+			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
 				Fatal("Invalid package string ", a, ": ", err.Error())
 			}
-			if gp.Version == "" {
-				gp.Version = "0"
-				gp.Condition = _gentoo.PkgCondGreaterEqual
-			}
-			pack := &pkg.DefaultPackage{
-				Name: gp.Name,
-				Version: fmt.Sprintf("%s%s%s",
-					pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
-					gp.Version,
-					gp.VersionSuffix,
-				),
-				Category: gp.Category,
-				Uri:      make([]string, 0),
-			}
 
-			inst := installer.NewLuetInstaller(LuetCfg.GetGeneral().Concurrency)
+			stype := LuetCfg.Viper.GetString("solver.type")
+			discount := LuetCfg.Viper.GetFloat64("solver.discount")
+			rate := LuetCfg.Viper.GetFloat64("solver.rate")
+			attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+			force := LuetCfg.Viper.GetBool("force")
+			nodeps, _ := cmd.Flags().GetBool("nodeps")
+			full, _ := cmd.Flags().GetBool("full")
+			checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
+			fullClean, _ := cmd.Flags().GetBool("full-clean")
+			concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+
+			LuetCfg.GetSolverOptions().Type = stype
+			LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+			LuetCfg.GetSolverOptions().Discount = float32(discount)
+			LuetCfg.GetSolverOptions().MaxAttempts = attempts
+			if concurrent {
+				LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+			} else {
+				LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+			}
+			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+				Concurrency:        LuetCfg.GetGeneral().Concurrency,
+				SolverOptions:      *LuetCfg.GetSolverOptions(),
+				NoDeps:             nodeps,
+				Force:              force,
+				FullUninstall:      full,
+				FullCleanUninstall: fullClean,
+				CheckConflicts:     checkconflicts,
+			})
 
 			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
 				systemDB = pkg.NewBoltDatabase(
-					filepath.Join(helpers.GetSystemRepoDatabaseDirPath(), "luet.db"))
+					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
 			} else {
 				systemDB = pkg.NewInMemoryDatabase(true)
 			}
@@ -84,5 +107,16 @@ func init() {
 	}
 	uninstallCmd.Flags().String("system-dbpath", path, "System db path")
 	uninstallCmd.Flags().String("system-target", path, "System rootpath")
+	uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	uninstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	uninstallCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
+	uninstallCmd.Flags().Bool("force", false, "Force uninstall")
+	uninstallCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
+	uninstallCmd.Flags().Bool("conflictscheck", true, "Check if the package marked for deletion is required by other packages")
+	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
+	uninstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -19,20 +19,26 @@ import (
 	"path/filepath"
 
 	. "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/spf13/cobra"
 )
 
 var upgradeCmd = &cobra.Command{
-	Use:   "upgrade",
-	Short: "Upgrades the system",
+	Use:     "upgrade",
+	Short:   "Upgrades the system",
+	Aliases: []string{"u"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", installCmd.Flags().Lookup("system-dbpath"))
 		LuetCfg.Viper.BindPFlag("system.rootfs", installCmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Long: `Upgrades packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
@@ -48,7 +54,40 @@ var upgradeCmd = &cobra.Command{
 			repos = append(repos, r)
 		}
 
-		inst := installer.NewLuetInstaller(LuetCfg.GetGeneral().Concurrency)
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps, _ := cmd.Flags().GetBool("nodeps")
+		full, _ := cmd.Flags().GetBool("full")
+		universe, _ := cmd.Flags().GetBool("universe")
+		clean, _ := cmd.Flags().GetBool("clean")
+		sync, _ := cmd.Flags().GetBool("sync")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
+		Debug("Solver", LuetCfg.GetSolverOptions().String())
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:              *LuetCfg.GetSolverOptions(),
+			Force:                      force,
+			FullUninstall:              full,
+			NoDeps:                     nodeps,
+			SolverUpgrade:              universe,
+			RemoveUnavailableOnUpgrade: clean,
+			UpgradeNewRevisions:        sync,
+		})
 		inst.Repositories(repos)
 		_, err := inst.SyncRepositories(false)
 		if err != nil {
@@ -57,7 +96,7 @@ var upgradeCmd = &cobra.Command{
 
 		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
 			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(helpers.GetSystemRepoDatabaseDirPath(), "luet.db"))
+				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
 		} else {
 			systemDB = pkg.NewInMemoryDatabase(true)
 		}
@@ -76,6 +115,17 @@ func init() {
 	}
 	upgradeCmd.Flags().String("system-dbpath", path, "System db path")
 	upgradeCmd.Flags().String("system-target", path, "System rootpath")
+	upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
+	upgradeCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
+	upgradeCmd.Flags().Bool("full", true, "Attempts to remove as much packages as possible which aren't required (slow)")
+	upgradeCmd.Flags().Bool("universe", false, "Use ONLY the SAT solver to compute upgrades (experimental)")
+	upgradeCmd.Flags().Bool("clean", false, "Try to drop removed packages (experimental, only when --universe is enabled)")
+	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
+	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

contrib/config/config.protect.d/01_etc.yml.example

@@ -0,0 +1,3 @@
+name: "etc_conf"
+dirs:
+  - "/etc/"

contrib/config/luet.yaml

@@ -4,8 +4,11 @@
 # Logging configuration section:
 # ---------------------------------------------
 # logging:
+#   # Enable loggging to file (if path is not empty)
+#   enable_logfile: false
+#
 #   Leave empty to skip logging to file.
-#   path: ""
+#   path: "/var/log/luet.log"
 #
 #   Set logging level: error|warning|info|debug
 #   level: "info"
@@ -13,6 +16,12 @@
 #   Enable JSON log format instead of console mode.
 #   json_format: false.
 #
+#   Disable/Enable color
+#   color: true
+#
+#   Enable/Disable emoji
+#   enable_emoji: true
+#
 # ---------------------------------------------
 # General configuration section:
 # ---------------------------------------------
@@ -56,6 +65,10 @@
 #   The path is append to rootfs option path.
 #   database_path: "/var/cache/luet"
 #
+#   Define the tmpdir base directory where luet store temporary files.
+#   Default $TMPDIR/tmpluet
+#   tmpdir_base: "/tmp/tmpluet"
+#
 # ---------------------------------------------
 # Repositories configurations directories.
 # ---------------------------------------------
@@ -66,7 +79,20 @@
 #   - /etc/luet/repos.conf.d
 #
 #
-# ---------------------------------------------
+# ------------------------------------------------
+# Config protect configuration files directories.
+# -----------------------------------------------
+# Define the list of directories where load
+# configuration files with the list of config
+# protect paths.
+# config_protect_confdir:
+#   - /etc/luet/config.protect.d
+#
+# Permit to ignore rules defined on
+# config protect confdir and packages
+# annotation.
+# config_protect_skip: false
+#
 # System repositories
 # ---------------------------------------------
 # In alternative to define repositories files
@@ -96,8 +122,11 @@
 #     packages. By default caching is disable.
 #     cached: false
 #
-#     Path where store tree of the specifications. Default path is $database_path/repos/$repo_name
-#     tree_path: "/var/cache/luet/repos/local"
+#     Path where store tree of the specifications. Default path is $database_path/repos/$repo_name/treefs
+#     tree_path: "/var/cache/luet/repos/local/treefs"
+#
+#     Path where store repository metadata. Default path is $database_path/repos/$repo_name/meta
+#     meta_path: "/var/cache/luet/repos/local/meta"
 #
 #     Define the list of the URL where retrieve tree and packages.
 #     urls:
@@ -108,3 +137,21 @@
 #        basic: "mybasicauth"
 #        Define token authentication header
 #        token: "mytoken"
+# ---------------------------------------------
+# Solver parameter configuration:
+# ---------------------------------------------
+# solver:
+#
+#   Solver strategy to solve possible conflicts during depedency
+#   solving. Defaults to empty (none). Available: qlearning
+#   type: ""
+#
+#   Solver agent learning rate. 0.1 to 1.0
+#   rate: 0.7
+#
+#   Learning discount factor.
+#   discount: 1.0
+#
+#   Number of overall attempts that the solver has available before bailing out.
+#   max_attempts: 9000
+#

go.mod

@@ -4,48 +4,48 @@ go 1.12
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56 // indirect
-	github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7
+	github.com/MottainaiCI/simplestreams-builder v0.1.0 // indirect
+	github.com/Sabayon/pkgs-checker v0.7.2
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/briandowns/spinner v1.7.0
 	github.com/cavaliercoder/grab v2.0.0+incompatible
-	github.com/creack/pty v1.1.9 // indirect
-	github.com/crillab/gophersat v1.1.7
-	github.com/docker/docker v0.7.3-0.20180827131323-0c5f8d2b9b23
+	github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3
+	github.com/docker/docker v17.12.0-ce-rc1.0.20200417035958-130b0bc6032c+incompatible
+	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
+	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-yaml/yaml v2.1.0+incompatible // indirect
 	github.com/hashicorp/go-version v1.2.0
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
 	github.com/klauspost/pgzip v1.2.1
+	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/kr/pretty v0.2.0 // indirect
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
 	github.com/mattn/go-isatty v0.0.10 // indirect
-	github.com/mudler/docker-companion v0.4.6-0.20191110154655-b8b364100616
-	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.0
-	github.com/otiai10/copy v1.0.2
+	github.com/moby/sys/mount v0.1.1-0.20200320164225-6154f11e6840 // indirect
+	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
+	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
+	github.com/onsi/ginkgo v1.12.1
+	github.com/onsi/gomega v1.10.0
+	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
 	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
-	github.com/pkg/errors v0.8.1
-	github.com/rogpeppe/go-internal v1.5.1 // indirect
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.5.0
+	github.com/pkg/errors v0.9.1
+	github.com/spf13/cobra v1.0.0
+	github.com/spf13/viper v1.6.3
 	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
-	go.etcd.io/bbolt v1.3.3
+	go.etcd.io/bbolt v1.3.4
 	go.uber.org/atomic v1.5.1 // indirect
 	go.uber.org/multierr v1.4.0 // indirect
 	go.uber.org/zap v1.13.0
-	golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
 	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
-	golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c // indirect
-	golang.org/x/tools v0.0.0-20200102200121-6de373a2766c // indirect
-	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
-	gopkg.in/yaml.v2 v2.2.7
-	mvdan.cc/sh v2.6.4+incompatible // indirect
+	gopkg.in/yaml.v2 v2.2.8
+	gotest.tools/v3 v3.0.2 // indirect
+	helm.sh/helm/v3 v3.3.4
 	mvdan.cc/sh/v3 v3.0.0-beta1
 )
+
+replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200605210607-749178b8f80d+incompatible

pkg/box/exec.go

@@ -0,0 +1,185 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package box
+
+import (
+	b64 "encoding/base64"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/errors"
+
+	helpers "github.com/mudler/luet/pkg/helpers"
+)
+
+type Box interface {
+	Run() error
+	Exec() error
+}
+
+type DefaultBox struct {
+	Name                  string
+	Root                  string
+	Env                   []string
+	Cmd                   string
+	Args                  []string
+	HostMounts            []string
+	Stdin, Stdout, Stderr bool
+}
+
+func NewBox(cmd string, args, hostmounts, env []string, rootfs string, stdin, stdout, stderr bool) Box {
+	return &DefaultBox{
+		Stdin:      stdin,
+		Stdout:     stdout,
+		Stderr:     stderr,
+		Cmd:        cmd,
+		Args:       args,
+		Root:       rootfs,
+		HostMounts: hostmounts,
+		Env:        env,
+	}
+}
+
+func (b *DefaultBox) Exec() error {
+
+	if err := mountProc(b.Root); err != nil {
+		return errors.Wrap(err, "Failed mounting proc on rootfs")
+	}
+	if err := mountDev(b.Root); err != nil {
+		return errors.Wrap(err, "Failed mounting dev on rootfs")
+	}
+
+	for _, hostMount := range b.HostMounts {
+		target := hostMount
+		if strings.Contains(hostMount, ":") {
+			dest := strings.Split(hostMount, ":")
+			if len(dest) != 2 {
+				return errors.New("Invalid arguments for mount, it can be: fullpath, or source:target")
+			}
+			hostMount = dest[0]
+			target = dest[1]
+		}
+		if err := mountBind(hostMount, b.Root, target); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed mounting %s on rootfs", hostMount))
+		}
+	}
+
+	if err := PivotRoot(b.Root); err != nil {
+		return errors.Wrap(err, "Failed switching pivot on rootfs")
+	}
+	cmd := exec.Command(b.Cmd, b.Args...)
+
+	if b.Stdin {
+		cmd.Stdin = os.Stdin
+	}
+
+	if b.Stderr {
+		cmd.Stderr = os.Stderr
+	}
+
+	if b.Stdout {
+		cmd.Stdout = os.Stdout
+	}
+
+	cmd.Env = b.Env
+
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Error running the %s command in box.Exec", b.Cmd))
+	}
+	return nil
+}
+
+func (b *DefaultBox) Run() error {
+
+	if !helpers.Exists(b.Root) {
+		return errors.New(b.Root + " does not exist")
+	}
+
+	// This matches with exec CLI command in luet
+	// TODO: Pass by env var as well
+	execCmd := []string{"exec", "--rootfs", b.Root, "--entrypoint", b.Cmd}
+
+	if b.Stdin {
+		execCmd = append(execCmd, "--stdin")
+	}
+
+	if b.Stderr {
+		execCmd = append(execCmd, "--stderr")
+	}
+
+	if b.Stdout {
+		execCmd = append(execCmd, "--stdout")
+	}
+	// Encode the command in base64 to avoid bad input from the args given
+	execCmd = append(execCmd, "--decode")
+
+	for _, m := range b.HostMounts {
+		execCmd = append(execCmd, "--mount")
+		execCmd = append(execCmd, m)
+	}
+
+	for _, e := range b.Env {
+		execCmd = append(execCmd, "--env")
+		execCmd = append(execCmd, e)
+	}
+
+	for _, a := range b.Args {
+		execCmd = append(execCmd, b64.StdEncoding.EncodeToString([]byte(a)))
+	}
+
+	cmd := exec.Command("/proc/self/exe", execCmd...)
+	if b.Stdin {
+		cmd.Stdin = os.Stdin
+	}
+
+	if b.Stderr {
+		cmd.Stderr = os.Stderr
+	}
+
+	if b.Stdout {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Cloneflags: syscall.CLONE_NEWNS |
+			syscall.CLONE_NEWUTS |
+			syscall.CLONE_NEWIPC |
+			syscall.CLONE_NEWPID |
+			syscall.CLONE_NEWNET |
+			syscall.CLONE_NEWUSER,
+		UidMappings: []syscall.SysProcIDMap{
+			{
+				ContainerID: 0,
+				HostID:      os.Getuid(),
+				Size:        1,
+			},
+		},
+		GidMappings: []syscall.SysProcIDMap{
+			{
+				ContainerID: 0,
+				HostID:      os.Getgid(),
+				Size:        1,
+			},
+		},
+	}
+
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "Failed running Box command in box.Run")
+	}
+	return nil
+}

pkg/box/rootfs.go

@@ -0,0 +1,94 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package box
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+func PivotRoot(newroot string) error {
+	putold := filepath.Join(newroot, "/.pivot_root")
+
+	// bind mount newroot to itself - this is a slight hack needed to satisfy the
+	// pivot_root requirement that newroot and putold must not be on the same
+	// filesystem as the current root
+	if err := syscall.Mount(newroot, newroot, "", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
+		return err
+	}
+
+	// create putold directory
+	if err := os.MkdirAll(putold, 0700); err != nil {
+		return err
+	}
+
+	// call pivot_root
+	if err := syscall.PivotRoot(newroot, putold); err != nil {
+		return err
+	}
+
+	// ensure current working directory is set to new root
+	if err := os.Chdir("/"); err != nil {
+		return err
+	}
+
+	// umount putold, which now lives at /.pivot_root
+	putold = "/.pivot_root"
+	if err := syscall.Unmount(putold, syscall.MNT_DETACH); err != nil {
+		return err
+	}
+
+	// remove putold
+	if err := os.RemoveAll(putold); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountProc(newroot string) error {
+	source := "proc"
+	target := filepath.Join(newroot, "/proc")
+	fstype := "proc"
+	flags := 0
+	data := ""
+
+	os.MkdirAll(target, 0755)
+	if err := syscall.Mount(source, target, fstype, uintptr(flags), data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountBind(hostfolder, newroot, dst string) error {
+	source := hostfolder
+	target := filepath.Join(newroot, dst)
+	fstype := "bind"
+	data := ""
+
+	os.MkdirAll(target, 0755)
+	if err := syscall.Mount(source, target, fstype, syscall.MS_BIND|syscall.MS_REC, data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountDev(newroot string) error {
+	return mountBind("/dev", newroot, "/dev")
+}

pkg/compiler/artifact.go

@@ -18,6 +18,8 @@ package compiler
 import (
 	"archive/tar"
 	"bufio"
+	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -25,6 +27,7 @@ import (
 	"path/filepath"
 	"regexp"
 
+	system "github.com/docker/docker/pkg/system"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
@@ -34,6 +37,7 @@ import (
 	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v2"
@@ -60,6 +64,7 @@ func (i ArtifactIndex) CleanPath() ArtifactIndex {
 			Dependencies:    art.Dependencies,
 			CompressionType: art.CompressionType,
 			Checksums:       art.Checksums,
+			Files:           art.Files,
 		})
 	}
 	return newIndex
@@ -77,6 +82,7 @@ type PackageArtifact struct {
 	Checksums       Checksums                 `json:"checksums"`
 	SourceAssertion solver.PackagesAssertions `json:"-"`
 	CompressionType CompressionImplementation `json:"compressiontype"`
+	Files           []string                  `json:"files"`
 }
 
 func NewPackageArtifact(path string) Artifact {
@@ -116,9 +122,19 @@ func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) {
 func (a *PackageArtifact) GetChecksums() Checksums {
 	return a.Checksums
 }
+
 func (a *PackageArtifact) SetChecksums(c Checksums) {
 	a.Checksums = c
 }
+
+func (a *PackageArtifact) SetFiles(f []string) {
+	a.Files = f
+}
+
+func (a *PackageArtifact) GetFiles() []string {
+	return a.Files
+}
+
 func (a *PackageArtifact) Hash() error {
 	return a.Checksums.Generate(a)
 }
@@ -265,8 +281,97 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	return errors.New("Compression type must be supplied")
 }
 
+func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	// If the destination path already exists I rename target file name with postfix.
+	var destPath string
+
+	// Read data. TODO: We need change archive callback to permit to return a Reader
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	// If file is not present on archive but is defined on mods
+	// I receive the callback. Prevent nil exception.
+	if header != nil {
+		switch header.Typeflag {
+		case tar.TypeReg:
+			destPath = filepath.Join(dst, path)
+		default:
+			// Nothing to do. I return original reader
+			return header, buffer.Bytes(), nil
+		}
+
+		// Check if exists
+		if helpers.Exists(destPath) {
+			for i := 1; i < 1000; i++ {
+				name := filepath.Join(filepath.Join(filepath.Dir(path),
+					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
+
+				if helpers.Exists(name) {
+					continue
+				}
+				Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
+					filepath.Join(dst, name)))
+				return &tar.Header{
+					Mode:       header.Mode,
+					Typeflag:   header.Typeflag,
+					PAXRecords: header.PAXRecords,
+					Name:       name,
+				}, buffer.Bytes(), nil
+			}
+		}
+	}
+
+	return header, buffer.Bytes(), nil
+}
+
+func (a *PackageArtifact) GetProtectFiles() []string {
+	ans := []string{}
+
+	if !LuetCfg.ConfigProtectSkip &&
+		LuetCfg.GetConfigProtectConfFiles() != nil &&
+		len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+
+		for _, file := range a.Files {
+			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
+				for _, dir := range conf.Directories {
+					// Note file is without / at begin.
+					if strings.HasPrefix("/"+file, filepath.Clean(dir)) {
+						// docker archive modifier works with path without / at begin.
+						ans = append(ans, file)
+						goto nextFile
+					}
+				}
+			}
+
+			if a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+				dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+				if ok {
+					if strings.HasPrefix("/"+file, filepath.Clean(dir)) {
+						ans = append(ans, file)
+						goto nextFile
+					}
+				}
+			}
+
+		nextFile:
+		}
+	}
+
+	return ans
+}
+
 // Unpack Untar and decompress (TODO) to the given path
 func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+
+	// Create
+	protectedFiles := a.GetProtectFiles()
+
+	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
+
 	switch a.CompressionType {
 	case GZip:
 		// Create the uncompressed archive
@@ -295,19 +400,21 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
 		}
 
-		err = helpers.Untar(a.GetPath()+".uncompressed", dst,
-			LuetCfg.GetGeneral().SameOwner)
+		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
 		}
 		return nil
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.Untar(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner)
+		return helpers.UntarProtect(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner,
+			protectedFiles, tarModifier)
 	}
 	return errors.New("Compression type must be supplied")
 }
 
+// FileList generates the list of file of a package from the local archive
 func (a *PackageArtifact) FileList() ([]string, error) {
 	var tr *tar.Reader
 	switch a.CompressionType {
@@ -373,6 +480,27 @@ type CopyJob struct {
 	Artifact string
 }
 
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
@@ -386,10 +514,13 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 		// 	continue
 		// }
 
-		if !helpers.Exists(job.Dst) {
+		_, err := os.Lstat(job.Dst)
+		if err != nil {
+			Debug("Copying ", job.Src)
 			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
+			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
@@ -397,14 +528,14 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
 func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, t CompressionImplementation) (Artifact, error) {
 
-	archive, err := ioutil.TempDir(os.TempDir(), "archive")
+	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for archive")
 	}
 	defer os.RemoveAll(archive) // clean up
 
 	if strings.HasSuffix(src, ".tar") {
-		rootfs, err := ioutil.TempDir(os.TempDir(), "rootfs")
+		rootfs, err := LuetCfg.GetSystem().TempDir("rootfs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 		}
@@ -446,19 +577,33 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 					}
 				}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	} else {
 		// Otherwise just grab all
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 			for _, a := range l.Diffs.Additions {
+				Debug("File ", a.Name, " added")
 				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	}
 
 	close(toCopy)
 	wg.Wait()
+
 	a := NewPackageArtifact(dst)
 	a.SetCompressionType(t)
 	err = a.Compress(archive, concurrency)

pkg/compiler/artifact_test.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/solver"
 
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
@@ -41,7 +42,7 @@ var _ = Describe("Artifact", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 

pkg/compiler/backend/diff.go

@@ -0,0 +1,199 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/pkg/errors"
+)
+
+// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
+// example of json return: [
+//   {
+//     "Image1": "luet/base",
+//     "Image2": "alpine",
+//     "DiffType": "File",
+//     "Diff": {
+//       "Adds": null,
+//       "Dels": [
+//         {
+//           "Name": "/luetbuild",
+//           "Size": 5830706
+//         },
+//         {
+//           "Name": "/luetbuild/Dockerfile",
+//           "Size": 50
+//         },
+//         {
+//           "Name": "/luetbuild/output1",
+//           "Size": 5830656
+//         }
+//       ],
+//       "Mods": null
+//     }
+//   }
+// ]
+func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]compiler.ArtifactLayer, error) {
+
+	res := compiler.ArtifactLayer{FromImage: srcImage, ToImage: dstImage}
+
+	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(srcRootFS) // clean up
+
+	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(dstRootFS) // clean up
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(srcImage, "/") {
+		srcImageTar, err := ioutil.TempFile(tmpdiffs, "srctar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(srcImageTar.Name()) // clean up
+		srcImageExport := compiler.CompilerBackendOptions{
+			ImageName:   srcImage,
+			Destination: srcImageTar.Name(),
+		}
+		err = b.ExportImage(srcImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting src image "+srcImage)
+		}
+		srcImage = srcImageTar.Name()
+	}
+
+	srcImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  srcImage,
+		Destination: srcRootFS,
+	}
+	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+srcImage)
+	}
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(dstImage, "/") {
+		dstImageTar, err := ioutil.TempFile(tmpdiffs, "dsttar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(dstImageTar.Name()) // clean up
+		dstImageExport := compiler.CompilerBackendOptions{
+			ImageName:   dstImage,
+			Destination: dstImageTar.Name(),
+		}
+		err = b.ExportImage(dstImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting dst image "+dstImage)
+		}
+		dstImage = dstImageTar.Name()
+	}
+
+	dstImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  dstImage,
+		Destination: dstRootFS,
+	}
+	err = b.ExtractRootfs(dstImageExtract, false)
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+dstImage)
+	}
+
+	// Get Additions/Changes. dst -> src
+	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, dstRootFS, "", -1)
+		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
+		if err == nil {
+			var sizeA, sizeB int64
+			sizeA = fileInfo.Size()
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+
+			if sizeA != sizeB {
+				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
+				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
+					Name: filepath.Join("/", realpath),
+					Size: int(sizeB),
+				})
+			} else {
+				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
+			}
+		} else {
+			var sizeB int64
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+				Size: int(sizeB),
+			})
+
+			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+	}
+
+	// Get deletions. src -> dst
+	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, srcRootFS, "", -1)
+		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
+			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
+			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+			})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+	}
+
+	return []compiler.ArtifactLayer{res}, nil
+}

pkg/compiler/backend/diff_test.go

@@ -0,0 +1,73 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend_test
+
+import (
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Docker image diffs", func() {
+	var b compiler.CompilerBackend
+
+	BeforeEach(func() {
+		b = NewSimpleDockerBackend()
+	})
+
+	Context("Generate diffs from docker images", func() {
+		It("Detect no changes", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "alpine:latest",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, "alpine:latest", "alpine:latest")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+
+		It("Detects additions and changed files", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/micro",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			err = b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/extra",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, "quay.io/mocaccino/micro", "quay.io/mocaccino/extra")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+
+			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+	})
+})

pkg/compiler/backend/simpledocker.go

@@ -89,6 +89,19 @@ func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
+func (*SimpleDocker) ImageExists(imagename string) bool {
+	buildarg := []string{"inspect", "--type=image", imagename}
+	Debug(":whale: Checking existance of docker image: " + imagename)
+	cmd := exec.Command("docker", buildarg...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		Warning("Image not present")
+		Debug(string(out))
+		return false
+	}
+	return true
+}
+
 func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
@@ -101,6 +114,18 @@ func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
+func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
+	name := opts.ImageName
+	pusharg := []string{"push", name}
+	out, err := exec.Command("docker", pusharg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed pushing image: "+string(out))
+	}
+	Info(":whale: Pushed image:", name)
+	//Info(string(out))
+	return nil
+}
+
 func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
@@ -196,58 +221,11 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 	return nil
 }
 
-// 	container-diff diff daemon://luet/base alpine --type=file -j
-// [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	tmpdiffs, err := ioutil.TempDir(os.TempDir(), "tmpdiffs")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
+// Changes retrieves changes between image layers
+func (d *SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+	diffs, err := GenerateChanges(d, fromImage, toImage)
 
-	diffargs := []string{"diff", fromImage, toImage, "-v", "error", "-q", "--type=file", "-j", "-n", "-c", tmpdiffs}
-	out, err := exec.Command("container-diff", diffargs...).Output()
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
-	}
-
-	if config.LuetCfg.GetGeneral().ShowBuildOutput {
-		Info(string(out))
-	}
-
-	var diffs []compiler.ArtifactLayer
-
-	err = json.Unmarshal(out, &diffs)
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
-	}
-
-	if config.LuetCfg.GetLogging().Level == "debug" {
+	if config.LuetCfg.GetGeneral().Debug {
 		summary := compiler.ComputeArtifactLayerSummary(diffs)
 		for _, l := range summary.Layers {
 			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
@@ -258,5 +236,5 @@ func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLaye
 		}
 	}
 
-	return diffs, nil
+	return diffs, err
 }

pkg/compiler/backend/simpledocker_test.go

@@ -18,6 +18,7 @@ package backend_test
 import (
 	. "github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/solver"
 
 	"io/ioutil"
 	"os"
@@ -40,7 +41,7 @@ var _ = Describe("Docker backend", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 

pkg/compiler/backend/simpleimg.go

@@ -97,6 +97,13 @@ func (*SimpleImg) CopyImage(src, dst string) error {
 	return nil
 }
 
+func (*SimpleImg) ImageExists(imagename string) bool {
+	// NOOP: not implemented
+	// TODO: Since img doesn't have an inspect command,
+	// we need to parse the ls output manually
+	return false
+}
+
 func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
@@ -142,6 +149,18 @@ func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms
 
 // TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
 // Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	return NewSimpleDockerBackend().Changes(fromImage, toImage)
+func (i *SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+	return GenerateChanges(i, fromImage, toImage)
+}
+
+func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {
+	name := opts.ImageName
+	pusharg := []string{"push", name}
+	out, err := exec.Command("img", pusharg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed pushing image: "+string(out))
+	}
+	Info(":tea: Pushed image:", name)
+	//Info(string(out))
+	return nil
 }

pkg/compiler/compiler.go

@@ -20,9 +20,13 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+
+	"github.com/ghodss/yaml"
+
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -33,6 +37,7 @@ import (
 )
 
 const BuildFile = "build.yaml"
+const DefinitionFile = "definition.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
@@ -42,9 +47,11 @@ type LuetCompiler struct {
 	PullFirst, KeepImg, Clean bool
 	Concurrency               int
 	CompressionType           CompressionImplementation
+	Options                   CompilerOptions
+	SolverOptions             solver.Options
 }
 
-func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions) Compiler {
+func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions, solvopts solver.Options) Compiler {
 	// The CompilerRecipe will gives us a tree with only build deps listed.
 	return &LuetCompiler{
 		Backend: backend,
@@ -58,6 +65,8 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *Compi
 		KeepImg:         opt.KeepImg,
 		Concurrency:     opt.Concurrency,
 		Clean:           opt.Clean,
+		Options:         *opt,
+		SolverOptions:   solvopts,
 	}
 }
 
@@ -226,14 +235,38 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 	return nil
 }
 
-func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec) (Artifact, error) {
+func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec, generateArtifact bool) (Artifact, error) {
+
+	pkgTag := ":package:  " + p.GetPackage().GetName()
+
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	// TODO: As the salt contains the packageImage ( in registry/organization/imagename:tag format)
+	// the images hashes are broken with registry mirrors.
+	// We should use the image tag, or pass by the package assertion hash which is unique
+	// and identifies the deptree of the package.
+
+	fp := p.GetPackage().HashFingerprint(packageImage)
+
+	if buildertaggedImage == "" {
+		buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder"
+		Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image)
+	}
+
+	// TODO:  Cleanup, not actually hit
+	if packageImage == "" {
+		packageImage = cs.ImageRepository + "-" + fp
+	}
+
 	if !cs.Clean {
-		if art, err := LoadArtifactFromYaml(p); err == nil {
+		exists := cs.Backend.ImageExists(buildertaggedImage) && cs.Backend.ImageExists(packageImage)
+		if art, err := LoadArtifactFromYaml(p); err == nil && (cs.Options.SkipIfMetadataExists || exists) {
 			Debug("Artifact reloaded. Skipping build")
 			return art, err
 		}
 	}
-	pkgTag := ":package:  " + p.GetPackage().GetName()
 
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
 	// and we build all the images first.
@@ -254,17 +287,13 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		return nil, errors.Wrap(err, "Could not copy package sources")
 
 	}
-	if buildertaggedImage == "" {
-		buildertaggedImage = cs.ImageRepository + "-" + p.GetPackage().GetFingerPrint() + "-builder"
-	}
-	if packageImage == "" {
-		packageImage = cs.ImageRepository + "-" + p.GetPackage().GetFingerPrint()
-	}
 
-	if cs.PullFirst {
-		//Best effort pull
-		cs.Backend.DownloadImage(CompilerBackendOptions{ImageName: buildertaggedImage})
-		cs.Backend.DownloadImage(CompilerBackendOptions{ImageName: packageImage})
+	// Copy file into the build context, the compilespec might have requested to do so.
+	if len(p.GetRetrieve()) > 0 {
+		err := p.CopyRetrieves(buildDir)
+		if err != nil {
+			Warning("Failed copying retrieves", err.Error())
+		}
 	}
 
 	Info(pkgTag, "Generating :whale: definition for builder image from", image)
@@ -278,16 +307,32 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 	}
 
-	err = cs.Backend.BuildImage(builderOpts)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
+	buildBuilderImage := true
+	if cs.Options.PullFirst {
+		if err := cs.Backend.DownloadImage(builderOpts); err == nil {
+			buildBuilderImage = false
+		}
 	}
 
-	err = cs.Backend.ExportImage(builderOpts)
-	if err != nil {
+	if buildBuilderImage {
+		if err = cs.Backend.BuildImage(builderOpts); err != nil {
+			return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
+
+	if err = cs.Backend.ExportImage(builderOpts); err != nil {
 		return nil, errors.Wrap(err, "Could not export image")
 	}
 
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(builderOpts.Destination)
+	}
+
+	if cs.Options.Push && buildBuilderImage {
+		if err = cs.Backend.Push(builderOpts); err != nil {
+			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
 	// Then we write the step image, which uses the builder one
 	p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
 	runnerOpts := CompilerBackendOptions{
@@ -297,31 +342,44 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 	}
 
-	// if !keepPackageImg {
-	// 	err = cs.Backend.ImageDefinitionToTar(runnerOpts)
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "Could not export image to tar")
-	// 	}
-	// } else {
-	if err := cs.Backend.BuildImage(runnerOpts); err != nil {
-		return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
+	buildPackageImage := true
+	if cs.Options.PullFirst {
+		//Best effort pull
+		if err := cs.Backend.DownloadImage(runnerOpts); err == nil {
+			buildPackageImage = false
+		}
 	}
+
+	if buildPackageImage {
+		if err := cs.Backend.BuildImage(runnerOpts); err != nil {
+			return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
+		}
+	}
+
 	if err := cs.Backend.ExportImage(runnerOpts); err != nil {
 		return nil, errors.Wrap(err, "Failed exporting image")
 	}
-	//	}
 
-	var diffs []ArtifactLayer
-	var artifact Artifact
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(runnerOpts.Destination)
+	}
 
-	if !p.ImageUnpack() {
-		// we have to get diffs only if spec is not unpacked
-		diffs, err = cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
+	if cs.Options.Push && buildPackageImage {
+		err = cs.Backend.Push(runnerOpts)
 		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate changes from layers")
+			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
 		}
 	}
 
+	var artifact Artifact
+	unpack := p.ImageUnpack()
+
+	// If package_dir was specified in the spec, we want to treat the content of the directory
+	// as the root of our archive.  ImageUnpack is implied to be true. override it
+	if p.GetPackageDir() != "" {
+		unpack = true
+	}
+
 	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
@@ -351,7 +409,15 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 	}
 
-	if p.ImageUnpack() {
+	if !generateArtifact {
+		return &PackageArtifact{}, nil
+	}
+
+	if unpack {
+		if p.GetPackageDir() != "" {
+			Info(":tophat: Packing from output dir", p.GetPackageDir())
+			rootfs = filepath.Join(rootfs, p.GetPackageDir())
+		}
 
 		if len(p.GetIncludes()) > 0 {
 			// strip from includes
@@ -359,6 +425,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 		artifact.SetCompressionType(cs.CompressionType)
+
 		err = artifact.Compress(rootfs, concurrency)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating package archive")
@@ -367,7 +434,10 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		artifact.SetCompileSpec(p)
 	} else {
 		Info(pkgTag, "Generating delta")
-
+		diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not generate changes from layers")
+		}
 		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not generate deltas")
@@ -376,105 +446,91 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		artifact.SetCompileSpec(p)
 	}
 
+	filelist, err := artifact.FileList()
+	if err != nil {
+		return artifact, errors.Wrap(err, "Failed getting package list")
+	}
+
+	artifact.SetFiles(filelist)
+
+	artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
+
 	err = artifact.WriteYaml(p.GetOutputPath())
 	if err != nil {
-		return artifact, err
+		return artifact, errors.Wrap(err, "Failed while writing metadata file")
 	}
 	Info(pkgTag, "   :white_check_mark: Done")
 
 	return artifact, nil
 }
 
-func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions, keepImg bool, concurrency int) (Artifact, error) {
-	if !cs.Clean {
-		if art, err := LoadArtifactFromYaml(p); err == nil {
-			Debug("Artifact reloaded. Skipping build")
-			return art, err
-		}
-	}
-	pkgTag := ":package:  " + p.GetPackage().GetName()
+func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) {
+	compilerSpecs := NewLuetCompilationspecs()
 
-	Info(pkgTag, "   🍩 Build starts 🔨 🔨 🔨 ")
+	w := db.World()
 
-	builderOpts := CompilerBackendOptions{
-		ImageName:   p.GetImage(),
-		Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
-	}
-	err := cs.Backend.DownloadImage(builderOpts)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not download image")
-	}
-
-	if tag != "" {
-		err = cs.Backend.CopyImage(p.GetImage(), tag)
+	for _, p := range w {
+		spec, err := cs.FromPackage(p)
 		if err != nil {
-			return nil, errors.Wrap(err, "Could not download image")
+			return nil, err
 		}
-	}
-	err = cs.Backend.ExportImage(builderOpts)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not export image")
+		if dst != "" {
+			spec.SetOutputPath(dst)
+		}
+		compilerSpecs.Add(spec)
 	}
 
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not create tempdir")
+	switch minimum {
+	case true:
+		return cs.ComputeMinimumCompilableSet(compilerSpecs.Unique().All()...)
+	default:
+		return compilerSpecs.Unique().All(), nil
 	}
-	defer os.RemoveAll(rootfs) // clean up
+}
 
-	// TODO: Compression and such
-	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
-		ImageName:  p.GetImage(),
-		SourcePath: builderOpts.Destination, Destination: rootfs}, keepPermissions)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not extract rootfs")
-	}
-	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
-	artifact.SetCompileSpec(p)
-	artifact.SetCompressionType(cs.CompressionType)
-
-	err = artifact.Compress(rootfs, concurrency)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating package archive")
-	}
-
-	if !keepImg {
-		// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
-		// TODO: Handle caching and optionally do not remove things
-		err = cs.Backend.RemoveImage(builderOpts)
+// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
+func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error) {
+	// Generate a set with all the deps of the provided specs
+	// we will use that set to remove the deps from the list of provided compilation specs
+	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
+	result := []CompilationSpec{}
+	for _, spec := range p {
+		ass, err := cs.ComputeDepTree(spec)
 		if err != nil {
-			Warning("Could not remove image ", builderOpts.ImageName)
-			//	return nil, errors.Wrap(err, "Could not remove image")
+			return result, errors.Wrap(err, "computin specs deptree")
+		}
+
+		allDependencies = append(allDependencies, ass.Drop(spec.GetPackage())...)
+	}
+
+	for _, spec := range p {
+		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
+			result = append(result, spec)
 		}
 	}
-
-	Info(pkgTag, "   :white_check_mark: Done")
-
-	err = artifact.WriteYaml(p.GetOutputPath())
-	if err != nil {
-		return artifact, err
-	}
-	return artifact, nil
+	return result, nil
 }
 
 func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
 
-	s := solver.NewSolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false))
+	s := solver.NewResolver(cs.SolverOptions, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
 
-	solution, err := s.Install([]pkg.Package{p.GetPackage()})
+	solution, err := s.Install(pkg.Packages{p.GetPackage()})
 	if err != nil {
-		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
 	}
 
-	dependencies := solution.Order(cs.Database, p.GetPackage().GetFingerPrint())
+	dependencies, err := solution.Order(cs.Database, p.GetPackage().GetFingerPrint())
+	if err != nil {
+		return nil, errors.Wrap(err, "While order a solution for "+p.GetPackage().HumanReadableString())
+	}
 
 	assertions := solver.PackagesAssertions{}
 	for _, assertion := range dependencies { //highly dependent on the order
 		if assertion.Value {
 			nthsolution := dependencies.Cut(assertion.Package)
-
 			assertion.Hash = solver.PackageHash{
-				BuildHash:   nthsolution.Drop(assertion.Package).AssertionHash(),
+				BuildHash:   nthsolution.HashFrom(assertion.Package),
 				PackageHash: nthsolution.AssertionHash(),
 			}
 			assertions = append(assertions, assertion)
@@ -495,21 +551,21 @@ func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifa
 }
 
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
-	Info(":package: Compiling", p.GetPackage().GetName(), "version", p.GetPackage().GetVersion(), ".... :coffee:")
+	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
 	if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" {
 		Error("Package with no deps and no seed image supplied, bailing out")
-		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() + "with no deps and no seed image supplied, bailing out")
+		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() +
+			" with no deps and no seed image supplied, bailing out")
 	}
 
+	targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
+	targetPackageHash := cs.ImageRepository + ":" + targetAssertion.Hash.PackageHash
+
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		if p.ImageUnpack() { // If it is just an entire image, create a package from it
-			return cs.packageFromImage(p, "", keepPermissions, cs.KeepImg, concurrency)
-		}
-
-		return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, cs.KeepImg, p)
+		return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@@ -523,76 +579,73 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	depsN := 0
 	currentN := 0
 
-	Info(":deciduous_tree: Build dependencies for " + p.GetPackage().GetName())
-	for _, assertion := range dependencies { //highly dependent on the order
-		depsN++
-		Info(" :arrow_right_hook:", assertion.Package.GetName(), ":leaves:", assertion.Package.GetVersion(), "(", assertion.Package.GetCategory(), ")")
-
-	}
-
-	for _, assertion := range dependencies { //highly dependent on the order
-		currentN++
-		pkgTag := fmt.Sprintf(":package:  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().GetName(), assertion.Package.GetName())
-		Info(pkgTag, "   :zap:  Building dependency")
-		compileSpec, err := cs.FromPackage(assertion.Package)
-		if err != nil {
-			return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+	packageDeps := !cs.Options.PackageTargetOnly
+	if !cs.Options.NoDeps {
+		Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
+		for _, assertion := range dependencies { //highly dependent on the order
+			depsN++
+			Info(" :arrow_right_hook:", assertion.Package.HumanReadableString(), ":leaves:")
 		}
-		compileSpec.SetOutputPath(p.GetOutputPath())
 
-		buildImageHash := cs.ImageRepository + ":" + assertion.Hash.BuildHash
-		currentPackageImageHash := cs.ImageRepository + ":" + assertion.Hash.PackageHash
-		Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from", buildImageHash)
-		Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
+		for _, assertion := range dependencies { //highly dependent on the order
+			currentN++
+			pkgTag := fmt.Sprintf(":package:  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
+			Info(pkgTag, "   :zap:  Building dependency")
+			compileSpec, err := cs.FromPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+			}
+			compileSpec.SetOutputPath(p.GetOutputPath())
 
-		lastHash = currentPackageImageHash
-		if compileSpec.GetImage() != "" {
-			// TODO: Refactor this
-			if compileSpec.ImageUnpack() { // If it is just an entire image, create a package from it
-				if compileSpec.GetImage() == "" {
-					return nil, errors.New("No image defined for package: " + assertion.Package.GetName())
-				}
-				Info(pkgTag, ":whale: Sourcing package from image", compileSpec.GetImage())
-				artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions, cs.KeepImg, concurrency)
+			buildImageHash := cs.ImageRepository + ":" + assertion.Hash.BuildHash
+			currentPackageImageHash := cs.ImageRepository + ":" + assertion.Hash.PackageHash
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from", buildImageHash)
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
+
+			lastHash = currentPackageImageHash
+			if compileSpec.GetImage() != "" {
+				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
+				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 				if err != nil {
-					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
+					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				}
 				departifacts = append(departifacts, artifact)
+				Info(pkgTag, ":white_check_mark: Done")
 				continue
 			}
 
-			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().GetFingerPrint()+" from image")
-			artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
+			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 			if err != nil {
-				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
+				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
+				//	deperrs = append(deperrs, err)
+				//		break // stop at first error
 			}
 			departifacts = append(departifacts, artifact)
-			Info(pkgTag, ":white_check_mark: Done")
-			continue
+			Info(pkgTag, ":collision: Done")
 		}
 
-		Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().GetFingerPrint()+" from tree")
-		artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+	} else if len(dependencies) > 0 {
+		lastHash = cs.ImageRepository + ":" + dependencies[len(dependencies)-1].Hash.PackageHash
+	}
+
+	if !cs.Options.OnlyDeps {
+		Info(":package:", p.GetPackage().HumanReadableString(), ":cyclone:  Building package target from:", lastHash)
+		artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 		if err != nil {
-			return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
-			//	deperrs = append(deperrs, err)
-			//		break // stop at first error
+			return artifact, err
 		}
-		departifacts = append(departifacts, artifact)
-		Info(pkgTag, ":collision: Done")
-	}
+		artifact.SetDependencies(departifacts)
+		artifact.SetSourceAssertion(p.GetSourceAssertion())
 
-	Info(":package:", p.GetPackage().GetName(), ":cyclone:  Building package target from:", lastHash)
-	artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, cs.KeepImg, p)
-	if err != nil {
 		return artifact, err
+	} else {
+		return departifacts[len(departifacts)-1], nil
 	}
-	artifact.SetDependencies(departifacts)
-	artifact.SetSourceAssertion(p.GetSourceAssertion())
-
-	return artifact, err
 }
 
+type templatedata map[string]interface{}
+
 func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 
 	pack, err := cs.Database.FindPackageCandidate(p)
@@ -604,12 +657,28 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 	if !helpers.Exists(buildFile) {
 		return nil, errors.New("No build file present for " + p.GetFingerPrint())
 	}
-
-	dat, err := ioutil.ReadFile(buildFile)
+	defFile := pack.Rel(DefinitionFile)
+	if !helpers.Exists(defFile) {
+		return nil, errors.New("No build file present for " + p.GetFingerPrint())
+	}
+	def, err := ioutil.ReadFile(defFile)
 	if err != nil {
 		return nil, err
 	}
-	return NewLuetCompilationSpec(dat, pack)
+
+	build, err := ioutil.ReadFile(buildFile)
+	if err != nil {
+		return nil, err
+	}
+	var values templatedata
+	if err = yaml.Unmarshal(def, &values); err != nil {
+		return nil, err
+	}
+	out, err := helpers.RenderHelm(string(build), values)
+	if err != nil {
+		return nil, err
+	}
+	return NewLuetCompilationSpec([]byte(out), pack)
 }
 
 func (cs *LuetCompiler) GetBackend() CompilerBackend {

pkg/compiler/compiler_test.go

@@ -23,6 +23,7 @@ import (
 	sd "github.com/mudler/luet/pkg/compiler/backend"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -38,7 +39,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -82,7 +83,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -108,6 +109,27 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Templated packages", func() {
+		It("Renders", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/templates")
+			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+			pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec, err := compiler.FromPackage(pkg)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetImage()).To(Equal("b:bar"))
+		})
+	})
+
 	Context("Reconstruct image tree", func() {
 		It("Compiles it", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -120,7 +142,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -177,7 +199,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -219,7 +241,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -254,7 +276,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -292,7 +314,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -333,7 +355,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -377,7 +399,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -419,7 +441,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -457,7 +479,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
 			Expect(err).ToNot(HaveOccurred())
@@ -496,7 +518,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
@@ -548,7 +570,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -591,7 +613,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -615,6 +637,61 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Packages which conents are a package folder", func() {
+		It("Compiles it in parallel", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/package_dir")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{
+				Name:     "dironly",
+				Category: "test",
+				Version:  "1.0",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{
+				Name:     "dironly_filter",
+				Category: "test",
+				Version:  "1.0",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			tmpdir2, err := ioutil.TempDir("", "tree2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir2) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir2)
+
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(2))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(0))
+
+			Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test2"))).To(BeTrue())
+
+			Expect(helpers.Untar(spec2.Rel("dironly_filter-test-1.0.package.tar"), tmpdir2, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec2.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
+		})
+	})
+
 	Context("Compression", func() {
 		It("Builds packages in gzip", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -624,7 +701,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -650,4 +727,62 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
 	})
+
+	Context("Compilation of whole tree", func() {
+		It("doesn't include dependencies that would be compiled anyway", func() {
+			// As some specs are dependent from each other, don't pull it in if they would
+			// be eventually
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/includeimage")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(specs)).To(Equal(1))
+
+			Expect(specs[0].GetPackage().GetFingerPrint()).To(Equal("b-test-1.0"))
+		})
+	})
+
+	Context("File list", func() {
+		It("is generated after the compilation process and annotated in the metadata", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/packagelayers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			compiler.SetCompressionType(GZip)
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(artifacts[0].GetFiles()).To(ContainElement("bin/busybox"))
+
+			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
+
+			art, err := LoadArtifactFromYaml(spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			files := art.GetFiles()
+			Expect(files).To(ContainElement("bin/busybox"))
+		})
+	})
 })

pkg/compiler/interface.go

@@ -18,6 +18,7 @@ package compiler
 import (
 	"runtime"
 
+	"github.com/mudler/luet/pkg/config"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 )
@@ -27,9 +28,10 @@ type Compiler interface {
 	CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
+	ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error)
 	SetConcurrency(i int)
 	FromPackage(pkg.Package) (CompilationSpec, error)
-
+	FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error)
 	SetBackend(CompilerBackend)
 	GetBackend() CompilerBackend
 	SetCompressionType(t CompressionImplementation)
@@ -43,21 +45,32 @@ type CompilerBackendOptions struct {
 }
 
 type CompilerOptions struct {
-	ImageRepository    string
-	PullFirst, KeepImg bool
-	Concurrency        int
-	CompressionType    CompressionImplementation
-	Clean              bool
+	ImageRepository          string
+	PullFirst, KeepImg, Push bool
+	Concurrency              int
+	CompressionType          CompressionImplementation
+	Clean                    bool
+	KeepImageExport          bool
+
+	OnlyDeps             bool
+	NoDeps               bool
+	SolverOptions        config.LuetSolverOptions
+	SkipIfMetadataExists bool
+
+	PackageTargetOnly bool
 }
 
 func NewDefaultCompilerOptions() *CompilerOptions {
 	return &CompilerOptions{
 		ImageRepository: "luet/cache",
-		PullFirst:       true,
+		PullFirst:       false,
+		Push:            false,
 		CompressionType: None,
 		KeepImg:         true,
 		Concurrency:     runtime.NumCPU(),
 		Clean:           true,
+		OnlyDeps:        false,
+		NoDeps:          false,
 	}
 }
 
@@ -71,6 +84,10 @@ type CompilerBackend interface {
 
 	CopyImage(string, string) error
 	DownloadImage(opts CompilerBackendOptions) error
+
+	Push(opts CompilerBackendOptions) error
+
+	ImageExists(string) bool
 }
 
 type Artifact interface {
@@ -91,6 +108,9 @@ type Artifact interface {
 	Hash() error
 	Verify() error
 
+	SetFiles(f []string)
+	GetFiles() []string
+
 	GetChecksums() Checksums
 	SetChecksums(c Checksums)
 }
@@ -151,6 +171,12 @@ type CompilationSpec interface {
 
 	GetSourceAssertion() solver.PackagesAssertions
 	SetSourceAssertion(as solver.PackagesAssertions)
+
+	GetRetrieve() []string
+	CopyRetrieves(dest string) error
+
+	SetPackageDir(string)
+	GetPackageDir() string
 }
 
 type CompilationSpecs interface {

pkg/compiler/spec.go

@@ -21,6 +21,7 @@ import (
 
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
+	"github.com/otiai10/copy"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -94,6 +95,9 @@ type LuetCompilationSpec struct {
 	Seed            string                    `json:"seed"`
 	Package         *pkg.DefaultPackage       `json:"package"`
 	SourceAssertion solver.PackagesAssertions `json:"-"`
+	PackageDir      string                    `json:"package_dir" yaml:"package_dir"`
+
+	Retrieve []string `json:"retrieve"`
 
 	OutputPath string   `json:"-"` // Where the build processfiles go
 	Unpack     bool     `json:"unpack"`
@@ -120,6 +124,14 @@ func (cs *LuetCompilationSpec) GetPackage() pkg.Package {
 	return cs.Package
 }
 
+func (cs *LuetCompilationSpec) GetPackageDir() string {
+	return cs.PackageDir
+}
+
+func (cs *LuetCompilationSpec) SetPackageDir(s string) {
+	cs.PackageDir = s
+}
+
 func (cs *LuetCompilationSpec) BuildSteps() []string {
 	return cs.Steps
 }
@@ -136,6 +148,10 @@ func (cs *LuetCompilationSpec) GetIncludes() []string {
 	return cs.Includes
 }
 
+func (cs *LuetCompilationSpec) GetRetrieve() []string {
+	return cs.Retrieve
+}
+
 func (cs *LuetCompilationSpec) GetSeedImage() string {
 	return cs.Seed
 }
@@ -164,6 +180,24 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 	cs.Seed = s
 }
 
+func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
+	var err error
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			matches, err := filepath.Glob(cs.Rel(s))
+
+			if err != nil {
+				continue
+			}
+
+			for _, m := range matches {
+				err = copy.Copy(m, filepath.Join(dest, filepath.Base(m)))
+			}
+		}
+	}
+	return err
+}
+
 // TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
 func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
 	spec := `
@@ -174,6 +208,19 @@ ENV PACKAGE_NAME=` + cs.Package.GetName() + `
 ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
 ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
 
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			//var file string
+			// if helpers.IsValidUrl(s) {
+			// 	file = s
+			// } else {
+			// 	file = cs.Rel(s)
+			// }
+			spec = spec + `
+ADD ` + s + ` /luetbuild/`
+		}
+	}
+
 	for _, s := range cs.Env {
 		spec = spec + `
 ENV ` + s

pkg/compiler/spec_test.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -61,7 +62,7 @@ var _ = Describe("Spec", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -105,4 +106,74 @@ RUN echo bar > /test2`))
 		})
 
 	})
+
+	It("Renders retrieve and env fields", func() {
+		generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
+
+		err := generalRecipe.Load("../../tests/fixtures/retrieve")
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+		spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
+		Expect(err).ToNot(HaveOccurred())
+
+		lspec, ok := spec.(*LuetCompilationSpec)
+		Expect(ok).To(BeTrue())
+
+		Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
+		Expect(lspec.Image).To(Equal("luet/base"))
+		Expect(lspec.Seed).To(Equal("alpine"))
+		tmpdir, err := ioutil.TempDir("", "tree")
+		Expect(err).ToNot(HaveOccurred())
+		defer os.RemoveAll(tmpdir) // clean up
+
+		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1`))
+
+		lspec.SetOutputPath("/foo/bar")
+
+		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1`))
+
+		err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(dockerfile).To(Equal(`
+FROM luet/base
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ENV test=1
+RUN echo foo > /test
+RUN echo bar > /test2`))
+
+	})
 })

pkg/config/config.go

@@ -19,19 +19,38 @@ package config
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"os"
 	"os/user"
+	"path/filepath"
 	"runtime"
+	"strings"
 	"time"
 
+	"github.com/mudler/luet/pkg/helpers"
+	solver "github.com/mudler/luet/pkg/solver"
+
 	v "github.com/spf13/viper"
 )
 
 var LuetCfg = NewLuetConfig(v.GetViper())
+var AvailableResolvers = strings.Join([]string{solver.QLearningResolverType}, " ")
 
 type LuetLoggingConfig struct {
-	Path       string `mapstructure:"path"`
-	JsonFormat bool   `mapstructure:"json_format"`
-	Level      string `mapstructure:"level"`
+	// Path of the logfile
+	Path string `mapstructure:"path"`
+	// Enable/Disable logging to file
+	EnableLogFile bool `mapstructure:"enable_logfile"`
+	// Enable JSON format logging in file
+	JsonFormat bool `mapstructure:"json_format"`
+
+	// Log level
+	Level string `mapstructure:"level"`
+
+	// Enable emoji
+	EnableEmoji bool `mapstructure:"enable_emoji"`
+	// Enable/Disable color in logging
+	Color bool `mapstructure:"color"`
 }
 
 type LuetGeneralConfig struct {
@@ -44,11 +63,76 @@ type LuetGeneralConfig struct {
 	FatalWarns      bool `mapstructure:"fatal_warnings"`
 }
 
+type LuetSolverOptions struct {
+	Type           string            `mapstructure:"type"`
+	LearnRate      float32           `mapstructure:"rate"`
+	Discount       float32           `mapstructure:"discount"`
+	MaxAttempts    int               `mapstructure:"max_attempts"`
+	Implementation solver.SolverType `mapstructure:"implementation"`
+}
+
+func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		if opts.LearnRate != 0.0 {
+			return solver.NewQLearningResolver(opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+
+		}
+		return solver.SimpleQLearningSolver()
+	}
+
+	return &solver.DummyPackageResolver{}
+}
+
+func (opts *LuetSolverOptions) CompactString() string {
+	return fmt.Sprintf("type: %s rate: %f, discount: %f, attempts: %d, initialobserved: %d",
+		opts.Type, opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+}
+
 type LuetSystemConfig struct {
 	DatabaseEngine string `yaml:"database_engine" mapstructure:"database_engine"`
 	DatabasePath   string `yaml:"database_path" mapstructure:"database_path"`
 	Rootfs         string `yaml:"rootfs" mapstructure:"rootfs"`
 	PkgsCachePath  string `yaml:"pkgs_cache_path" mapstructure:"pkgs_cache_path"`
+	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
+}
+
+func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
+	dbpath = filepath.Join(dbpath, "repos/"+name)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+	dbpath := filepath.Join(sc.Rootfs,
+		sc.DatabasePath)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+	var cachepath string
+	if sc.PkgsCachePath != "" {
+		cachepath = sc.PkgsCachePath
+	} else {
+		// Create dynamic cache for test suites
+		cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
+	}
+
+	if filepath.IsAbs(cachepath) {
+		ans = cachepath
+	} else {
+		ans = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
+	}
+
+	return
 }
 
 type LuetRepository struct {
@@ -62,6 +146,7 @@ type LuetRepository struct {
 	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
 	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
 	TreePath       string            `json:"tree_path,omitempty" yaml:"tree_path,omitempty" mapstructure:"tree_path"`
+	MetaPath       string            `json:"meta_path,omitempty" yaml:"meta_path,omitempty" mapstructure:"meta_path"`
 
 	// Serialized options not used in repository configuration
 
@@ -84,6 +169,7 @@ func NewLuetRepository(name, t, descr string, urls []string, priority int, enabl
 		Cached:         cached,
 		Authentication: make(map[string]string, 0),
 		TreePath:       "",
+		MetaPath:       "",
 	}
 }
 
@@ -95,6 +181,7 @@ func NewEmptyLuetRepository() *LuetRepository {
 		Type:           "",
 		Priority:       9999,
 		TreePath:       "",
+		MetaPath:       "",
 		Enable:         false,
 		Cached:         false,
 		Authentication: make(map[string]string, 0),
@@ -112,10 +199,15 @@ type LuetConfig struct {
 	Logging LuetLoggingConfig `mapstructure:"logging"`
 	General LuetGeneralConfig `mapstructure:"general"`
 	System  LuetSystemConfig  `mapstructure:"system"`
+	Solver  LuetSolverOptions `mapstructure:"solver"`
 
-	RepositoriesConfDir []string         `mapstructure:"repos_confdir"`
-	CacheRepositories   []LuetRepository `mapstructure:"repetitors"`
-	SystemRepositories  []LuetRepository `mapstructure:"repositories"`
+	RepositoriesConfDir  []string         `mapstructure:"repos_confdir"`
+	ConfigProtectConfDir []string         `mapstructure:"config_protect_confdir"`
+	ConfigProtectSkip    bool             `mapstructure:"config_protect_skip"`
+	CacheRepositories    []LuetRepository `mapstructure:"repetitors"`
+	SystemRepositories   []LuetRepository `mapstructure:"repositories"`
+
+	ConfigProtectConfFiles []ConfigProtectConfFile
 }
 
 func NewLuetConfig(viper *v.Viper) *LuetConfig {
@@ -124,13 +216,16 @@ func NewLuetConfig(viper *v.Viper) *LuetConfig {
 	}
 
 	GenDefault(viper)
-	return &LuetConfig{Viper: viper}
+	return &LuetConfig{Viper: viper, ConfigProtectConfFiles: nil}
 }
 
 func GenDefault(viper *v.Viper) {
 	viper.SetDefault("logging.level", "info")
-	viper.SetDefault("logging.path", "")
+	viper.SetDefault("logging.enable_logfile", false)
+	viper.SetDefault("logging.path", "/var/log/luet.log")
 	viper.SetDefault("logging.json_format", false)
+	viper.SetDefault("logging.enable_emoji", true)
+	viper.SetDefault("logging.color", true)
 
 	viper.SetDefault("general.concurrency", runtime.NumCPU())
 	viper.SetDefault("general.debug", false)
@@ -139,8 +234,9 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("general.spinner_charset", 22)
 	viper.SetDefault("general.fatal_warnings", false)
 
-	u, _ := user.Current()
-	if u.Uid == "0" {
+	u, err := user.Current()
+	// os/user doesn't work in from scratch environments
+	if err != nil || (u != nil && u.Uid == "0") {
 		viper.SetDefault("general.same_owner", true)
 	} else {
 		viper.SetDefault("general.same_owner", false)
@@ -149,11 +245,19 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("system.database_engine", "boltdb")
 	viper.SetDefault("system.database_path", "/var/cache/luet")
 	viper.SetDefault("system.rootfs", "/")
+	viper.SetDefault("system.tmpdir_base", filepath.Join(os.TempDir(), "tmpluet"))
 	viper.SetDefault("system.pkgs_cache_path", "packages")
 
 	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
+	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
+	viper.SetDefault("config_protect_skip", false)
 	viper.SetDefault("cache_repositories", []string{})
 	viper.SetDefault("system_repositories", []string{})
+
+	viper.SetDefault("solver.type", "")
+	viper.SetDefault("solver.rate", 0.7)
+	viper.SetDefault("solver.discount", 1.0)
+	viper.SetDefault("solver.max_attempts", 9000)
 }
 
 func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
@@ -172,6 +276,22 @@ func (c *LuetConfig) GetSystem() *LuetSystemConfig {
 	return &c.System
 }
 
+func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
+	return &c.Solver
+}
+
+func (c *LuetConfig) GetConfigProtectConfFiles() []ConfigProtectConfFile {
+	return c.ConfigProtectConfFiles
+}
+
+func (c *LuetConfig) AddConfigProtectConfFile(file *ConfigProtectConfFile) {
+	if c.ConfigProtectConfFiles == nil {
+		c.ConfigProtectConfFiles = []ConfigProtectConfFile{*file}
+	} else {
+		c.ConfigProtectConfFiles = append(c.ConfigProtectConfFiles, *file)
+	}
+}
+
 func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
 	var ans *LuetRepository = nil
 
@@ -188,6 +308,18 @@ func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
 	return ans, nil
 }
 
+func (c *LuetSolverOptions) String() string {
+	ans := fmt.Sprintf(`
+solver:
+  type: %s
+  rate: %f
+  discount: %f
+  max_attempts: %d`, c.Type, c.LearnRate, c.Discount,
+		c.MaxAttempts)
+
+	return ans
+}
+
 func (c *LuetGeneralConfig) String() string {
 	ans := fmt.Sprintf(`
 general:
@@ -212,12 +344,20 @@ func (c *LuetGeneralConfig) GetSpinnerMs() time.Duration {
 	return duration
 }
 
+func (c *LuetLoggingConfig) SetLogLevel(s string) {
+	c.Level = s
+}
+
 func (c *LuetLoggingConfig) String() string {
 	ans := fmt.Sprintf(`
 logging:
+  enable_logfile: %t
   path: %s
   json_format: %t
-  level: %s`, c.Path, c.JsonFormat, c.Level)
+  color: %t
+  enable_emoji: %t
+  level: %s`, c.EnableLogFile, c.Path, c.JsonFormat,
+		c.Color, c.EnableEmoji, c.Level)
 
 	return ans
 }
@@ -228,8 +368,37 @@ system:
   database_engine: %s
   database_path: %s
   pkgs_cache_path: %s
+  tmpdir_base: %s
   rootfs: %s`,
-		c.DatabaseEngine, c.DatabasePath, c.PkgsCachePath, c.Rootfs)
+		c.DatabaseEngine, c.DatabasePath, c.PkgsCachePath,
+		c.TmpDirBase, c.Rootfs)
 
 	return ans
 }
+
+func (c *LuetSystemConfig) InitTmpDir() error {
+	if !helpers.Exists(c.TmpDirBase) {
+		return os.MkdirAll(c.TmpDirBase, os.ModePerm)
+	}
+	return nil
+}
+
+func (c *LuetSystemConfig) CleanupTmpDir() error {
+	return os.RemoveAll(c.TmpDirBase)
+}
+
+func (c *LuetSystemConfig) TempDir(pattern string) (string, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return "", err
+	}
+	return ioutil.TempDir(c.TmpDirBase, pattern)
+}
+
+func (c *LuetSystemConfig) TempFile(pattern string) (*os.File, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.TempFile(c.TmpDirBase, pattern)
+}

pkg/config/config_protect.go

@@ -0,0 +1,41 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"fmt"
+)
+
+type ConfigProtectConfFile struct {
+	Filename string
+
+	Name        string   `mapstructure:"name" yaml:"name" json:"name"`
+	Directories []string `mapstructure:"dirs" yaml:"dirs" json:"dirs"`
+}
+
+func NewConfigProtectConfFile(filename string) *ConfigProtectConfFile {
+	return &ConfigProtectConfFile{
+		Filename:    filename,
+		Name:        "",
+		Directories: []string{},
+	}
+}
+
+func (c *ConfigProtectConfFile) String() string {
+	return fmt.Sprintf("[%s] filename: %s, dirs: %s", c.Name, c.Filename,
+		c.Directories)
+}

pkg/config/config_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Config Suite")
+}

pkg/config/config_test.go

@@ -0,0 +1,66 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	config "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+
+	Context("Simple temporary directory creation", func() {
+
+		It("Create Temporary directory", func() {
+			// PRE: tmpdir_base contains default value.
+
+			tmpDir, err := config.LuetCfg.GetSystem().TempDir("test1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.HasPrefix(tmpDir, filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
+			Expect(helpers.Exists(tmpDir)).To(BeTrue())
+
+			defer os.RemoveAll(tmpDir)
+		})
+
+		It("Create Temporary file", func() {
+			// PRE: tmpdir_base contains default value.
+
+			tmpFile, err := config.LuetCfg.GetSystem().TempFile("testfile1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.HasPrefix(tmpFile.Name(), filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
+			Expect(helpers.Exists(tmpFile.Name())).To(BeTrue())
+
+			defer os.Remove(tmpFile.Name())
+		})
+
+		It("Config1", func() {
+			cfg := config.LuetCfg
+
+			cfg.GetLogging().Color = false
+			Expect(cfg.GetLogging().Color).To(BeFalse())
+		})
+
+	})
+
+})

pkg/helpers/archive.go

@@ -17,12 +17,11 @@ package helpers
 
 import (
 	"archive/tar"
+	"bytes"
 	"io"
 	"os"
 	"path/filepath"
 
-	. "github.com/mudler/luet/pkg/config"
-
 	"github.com/docker/docker/pkg/archive"
 )
 
@@ -51,6 +50,146 @@ func Tar(src, dest string) error {
 	return err
 }
 
+type TarModifierWrapperFunc func(path, dst string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
+type TarModifierWrapper struct {
+	DestinationPath string
+	Modifier        TarModifierWrapperFunc
+}
+
+func NewTarModifierWrapper(dst string, modifier TarModifierWrapperFunc) *TarModifierWrapper {
+	return &TarModifierWrapper{
+		DestinationPath: dst,
+		Modifier:        modifier,
+	}
+}
+
+func (m *TarModifierWrapper) GetModifier() archive.TarModifierFunc {
+	return func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+		return m.Modifier(m.DestinationPath, path, header, content)
+	}
+}
+
+func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modifier *TarModifierWrapper) error {
+	var ans error
+
+	if len(protectedFiles) <= 0 {
+		return Untar(src, dst, sameOwner)
+	}
+
+	// POST: we have files to protect. I create a ReplaceFileTarWrapper
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	// Create modifier map
+	mods := make(map[string]archive.TarModifierFunc)
+	for _, file := range protectedFiles {
+		mods[file] = modifier.GetModifier()
+	}
+
+	if sameOwner {
+		// PRE: i have root privileged.
+
+		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
+
+		opts := &archive.TarOptions{
+			// NOTE: NoLchown boolean is used for chmod of the symlink
+			// Probably it's needed set this always to true.
+			NoLchown:        true,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+			ContinueOnError: true,
+		}
+
+		ans = archive.Untar(replacerArchive, dst, opts)
+	} else {
+		ans = unTarIgnoreOwner(dst, in, mods)
+	}
+
+	return ans
+}
+
+func unTarIgnoreOwner(dest string, in io.ReadCloser, mods map[string]archive.TarModifierFunc) error {
+	tr := tar.NewReader(in)
+	for {
+		header, err := tr.Next()
+
+		var data []byte
+		var headerReplaced = false
+
+		switch {
+		case err == io.EOF:
+			goto tarEof
+		case err != nil:
+			return err
+		case header == nil:
+			continue
+		}
+
+		// the target location where the dir/file should be created
+		target := filepath.Join(dest, header.Name)
+		if mods != nil {
+			modifier, ok := mods[header.Name]
+			if ok {
+				header, data, err = modifier(header.Name, header, tr)
+				if err != nil {
+					return err
+				}
+
+				// Override target path
+				target = filepath.Join(dest, header.Name)
+				headerReplaced = true
+			}
+
+		}
+
+		// Check the file type
+		switch header.Typeflag {
+
+		// if its a dir and it doesn't exist create it
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return err
+				}
+			}
+
+			// handle creation of file
+		case tar.TypeReg:
+
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+
+			// copy over contents
+			if headerReplaced {
+				_, err = io.Copy(f, bytes.NewReader(data))
+			} else {
+				_, err = io.Copy(f, tr)
+			}
+			if err != nil {
+				return err
+			}
+
+			// manually close here after each file operation; defering would cause each
+			// file close to wait until all operations have completed.
+			f.Close()
+
+		case tar.TypeSymlink:
+			source := header.Linkname
+			err := os.Symlink(source, target)
+			if err != nil {
+				return err
+			}
+		}
+	}
+tarEof:
+
+	return nil
+}
+
 // Untar just a wrapper around the docker functions
 func Untar(src, dest string, sameOwner bool) error {
 	var ans error
@@ -61,7 +200,7 @@ func Untar(src, dest string, sameOwner bool) error {
 	}
 	defer in.Close()
 
-	if LuetCfg.GetGeneral().SameOwner {
+	if sameOwner {
 		// PRE: i have root privileged.
 
 		opts := &archive.TarOptions{
@@ -69,65 +208,12 @@ func Untar(src, dest string, sameOwner bool) error {
 			// Probably it's needed set this always to true.
 			NoLchown:        true,
 			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+			ContinueOnError: true,
 		}
 
 		ans = archive.Untar(in, dest, opts)
 	} else {
-
-		var fileReader io.ReadCloser = in
-
-		tr := tar.NewReader(fileReader)
-		for {
-			header, err := tr.Next()
-
-			switch {
-			case err == io.EOF:
-				goto tarEof
-			case err != nil:
-				return err
-			case header == nil:
-				continue
-			}
-
-			// the target location where the dir/file should be created
-			target := filepath.Join(dest, header.Name)
-
-			// Check the file type
-			switch header.Typeflag {
-
-			// if its a dir and it doesn't exist create it
-			case tar.TypeDir:
-				if _, err := os.Stat(target); err != nil {
-					if err := os.MkdirAll(target, 0755); err != nil {
-						return err
-					}
-				}
-
-				// handle creation of file
-			case tar.TypeReg:
-				f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
-				if err != nil {
-					return err
-				}
-
-				// copy over contents
-				if _, err := io.Copy(f, tr); err != nil {
-					return err
-				}
-
-				// manually close here after each file operation; defering would cause each
-				// file close to wait until all operations have completed.
-				f.Close()
-
-			case tar.TypeSymlink:
-				source := header.Linkname
-				err := os.Symlink(source, target)
-				if err != nil {
-					return err
-				}
-			}
-		}
-	tarEof:
+		ans = unTarIgnoreOwner(dest, in, nil)
 	}
 
 	return ans

pkg/helpers/archive_test.go

@@ -0,0 +1,134 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/archive"
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// Code from moby/moby pkg/archive/archive_test
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(filepath.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+		if makeLinks {
+			if err := os.Link(filepath.Join(targetPath, fileName), filepath.Join(targetPath, fileName+"-link")); err != nil {
+				return 0, err
+			}
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}
+
+func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	// If the destination path already exists I rename target file name with postfix.
+	var basePath string
+
+	// Read data. TODO: We need change archive callback to permit to return a Reader
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	if header != nil {
+
+		switch header.Typeflag {
+		case tar.TypeReg:
+			basePath = filepath.Base(path)
+		default:
+			// Nothing to do. I return original reader
+			return header, buffer.Bytes(), nil
+		}
+
+		if basePath == "file-0" {
+			name := filepath.Join(filepath.Join(filepath.Dir(path), fmt.Sprintf("._cfg%04d_%s", 1, basePath)))
+			return &tar.Header{
+				Mode:       header.Mode,
+				Typeflag:   header.Typeflag,
+				PAXRecords: header.PAXRecords,
+				Name:       name,
+			}, buffer.Bytes(), nil
+		} else if basePath == "file-1" {
+			return header, []byte("newcontent"), nil
+		}
+
+		// else file not present
+	}
+
+	return header, buffer.Bytes(), nil
+}
+
+var _ = Describe("Helpers Archive", func() {
+	Context("Untar Protect", func() {
+
+		It("Detect existing and not-existing files", func() {
+
+			archiveSourceDir, err := ioutil.TempDir("", "archive-source")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(archiveSourceDir)
+
+			_, err = prepareUntarSourceDirectory(10, archiveSourceDir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			targetDir, err := ioutil.TempDir("", "archive-target")
+			Expect(err).ToNot(HaveOccurred())
+			//	defer os.RemoveAll(targetDir)
+
+			sourceArchive, err := archive.TarWithOptions(archiveSourceDir, &archive.TarOptions{})
+			Expect(err).ToNot(HaveOccurred())
+			defer sourceArchive.Close()
+
+			tarModifier := NewTarModifierWrapper(targetDir, tarModifierWrapperFunc)
+			mods := make(map[string]archive.TarModifierFunc)
+			mods["file-0"] = tarModifier.GetModifier()
+			mods["file-1"] = tarModifier.GetModifier()
+			mods["file-9999"] = tarModifier.GetModifier()
+
+			replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
+			//replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
+			opts := &archive.TarOptions{
+				// NOTE: NoLchown boolean is used for chmod of the symlink
+				// Probably it's needed set this always to true.
+				NoLchown:        true,
+				ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+				ContinueOnError: true,
+			}
+
+			err = archive.Untar(replacerArchive, targetDir, opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
+		})
+	})
+})

pkg/helpers/file.go

@@ -19,10 +19,46 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"time"
 
 	copy "github.com/otiai10/copy"
 )
 
+func ListDir(dir string) ([]string, error) {
+	content := []string{}
+
+	err := filepath.Walk(dir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			content = append(content, path)
+
+			return nil
+		})
+
+	return content, err
+}
+
+// Touch creates an empty file
+func Touch(f string) error {
+	_, err := os.Stat(f)
+	if os.IsNotExist(err) {
+		file, err := os.Create(f)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+	} else {
+		currentTime := time.Now().Local()
+		err = os.Chtimes(f, currentTime, currentTime)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Exists reports whether the named file or directory exists.
 func Exists(name string) bool {
 	if _, err := os.Stat(name); err != nil {
@@ -57,7 +93,7 @@ func ensureDir(fileName string) {
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
 func CopyFile(src, dst string) (err error) {
-	return copy.Copy(src, dst)
+	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }
 
 func IsDirectory(path string) (bool, error) {
@@ -74,5 +110,5 @@ func IsDirectory(path string) (bool, error) {
 func CopyDir(src string, dst string) (err error) {
 	src = filepath.Clean(src)
 	dst = filepath.Clean(dst)
-	return copy.Copy(src, dst)
+	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }

pkg/helpers/helm.go

@@ -0,0 +1,34 @@
+package helpers
+
+import (
+
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+	"github.com/pkg/errors"
+)
+
+// RenderHelm renders the template string with helm
+func RenderHelm(template string, values map[string]interface{}) (string,error) {
+	c := &chart.Chart{
+		Metadata: &chart.Metadata{
+			Name:    "",
+			Version: "",
+		},
+		Templates: []*chart.File{
+			{Name: "templates", Data: []byte(template)},
+		},
+		Values:    map[string]interface{}{"Values":values},
+	}
+
+	v, err := chartutil.CoalesceValues(c, map[string]interface{}{})
+	if err != nil {
+		return "",errors.Wrap(err,"while rendering template")
+	}
+	out, err := engine.Render(c, v)
+	if err != nil {
+		return "",errors.Wrap(err,"while rendering template")
+	}
+
+	return out["templates"],nil
+}

pkg/helpers/helm_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Helpers", func() {
+	Context("RenderHelm", func() {
+		It("Renders templates", func() {
+			out, err := RenderHelm("{{.Values.Test}}",map[string]interface{}{"Test":"foo"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foo"))
+		})
+	})
+})

pkg/helpers/match.go

@@ -0,0 +1,49 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"regexp"
+)
+
+func MapMatchRegex(m *map[string]string, r *regexp.Regexp) bool {
+	ans := false
+
+	if m != nil {
+		for k, v := range *m {
+			if r.MatchString(k + "=" + v) {
+				ans = true
+				break
+			}
+		}
+	}
+
+	return ans
+}
+
+func MapHasKey(m *map[string]string, label string) bool {
+	ans := false
+	if m != nil {
+		for k, _ := range *m {
+			if k == label {
+				ans = true
+				break
+			}
+		}
+	}
+	return ans
+}

pkg/helpers/math.go

@@ -0,0 +1,24 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+func Factorial(n uint64) (result uint64) {
+	if n > 0 {
+		result = n * Factorial(n-1)
+		return result
+	}
+	return 1
+}

pkg/helpers/repository.go

@@ -1,63 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//                  Daniele Rondina <geaaru@sabayonlinux.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package helpers
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	"github.com/mudler/luet/pkg/config"
-)
-
-func GetRepoDatabaseDirPath(name string) string {
-	dbpath := filepath.Join(config.LuetCfg.GetSystem().Rootfs, config.LuetCfg.GetSystem().DatabasePath)
-	dbpath = filepath.Join(dbpath, "repos/"+name)
-	err := os.MkdirAll(dbpath, os.ModePerm)
-	if err != nil {
-		panic(err)
-	}
-	return dbpath
-}
-
-func GetSystemRepoDatabaseDirPath() string {
-	dbpath := filepath.Join(config.LuetCfg.GetSystem().Rootfs,
-		config.LuetCfg.GetSystem().DatabasePath)
-	err := os.MkdirAll(dbpath, os.ModePerm)
-	if err != nil {
-		panic(err)
-	}
-	return dbpath
-}
-
-func GetSystemPkgsCacheDirPath() (ans string) {
-	var cachepath string
-	if config.LuetCfg.GetSystem().PkgsCachePath != "" {
-		cachepath = config.LuetCfg.GetSystem().PkgsCachePath
-	} else {
-		// Create dynamic cache for test suites
-		cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
-	}
-
-	if filepath.IsAbs(cachepath) {
-		ans = cachepath
-	} else {
-		ans = filepath.Join(GetSystemRepoDatabaseDirPath(), cachepath)
-	}
-
-	return
-}

pkg/helpers/sys.go

@@ -0,0 +1,48 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"os"
+	"os/exec"
+	"os/user"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+// This allows a multi-platform switch in the future
+func Exec(cmd string, args []string, env []string) error {
+	path, err := exec.LookPath(cmd)
+	if err != nil {
+		return errors.Wrap(err, "Could not find binary in path: "+cmd)
+	}
+	return syscall.Exec(path, args, env)
+}
+
+func GetHomeDir() (ans string) {
+	// os/user doesn't work in from scratch environments
+	u, err := user.Current()
+	if err == nil {
+		ans = u.HomeDir
+	} else {
+		ans = ""
+	}
+	if os.Getenv("HOME") != "" {
+		ans = os.Getenv("HOME")
+	}
+	return ans
+}

pkg/installer/client/http.go

@@ -17,7 +17,6 @@ package client
 
 import (
 	"fmt"
-	"io/ioutil"
 	"math"
 	"net/url"
 	"os"
@@ -27,6 +26,7 @@ import (
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 
 	"github.com/cavaliercoder/grab"
@@ -70,7 +70,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 	var temp string
 
 	artifactName := path.Base(artifact.GetPath())
-	cacheFile := filepath.Join(helpers.GetSystemPkgsCacheDirPath(), artifactName)
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 	ok := false
 
 	// Check if file is already in cache
@@ -78,7 +78,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 		Info("Use artifact", artifactName, "from cache.")
 	} else {
 
-		temp, err = ioutil.TempDir(os.TempDir(), "tree")
+		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
 		if err != nil {
 			return nil, err
 		}
@@ -138,7 +138,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 
 	ok := false
 
-	temp, err = ioutil.TempDir(os.TempDir(), "tree")
+	temp, err = config.LuetCfg.GetSystem().TempDir("tree")
 	if err != nil {
 		return "", err
 	}
@@ -147,7 +147,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 
 	for _, uri := range c.RepoData.Urls {
 
-		file, err = ioutil.TempFile(os.TempDir(), "HttpClient")
+		file, err = config.LuetCfg.GetSystem().TempFile("HttpClient")
 		if err != nil {
 			continue
 		}

pkg/installer/client/local.go

@@ -16,11 +16,11 @@
 package client
 
 import (
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/mudler/luet/pkg/compiler"
@@ -39,7 +39,7 @@ func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Art
 	var err error
 
 	artifactName := path.Base(artifact.GetPath())
-	cacheFile := filepath.Join(helpers.GetSystemPkgsCacheDirPath(), artifactName)
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 
 	// Check if file is already in cache
 	if helpers.Exists(cacheFile) {
@@ -75,7 +75,7 @@ func (c *LocalClient) DownloadFile(name string) (string, error) {
 	ok := false
 	for _, uri := range c.RepoData.Urls {
 		Info("Downloading file", name, "from", uri)
-		file, err = ioutil.TempFile(os.TempDir(), "localclient")
+		file, err = config.LuetCfg.GetSystem().TempFile("localclient")
 		if err != nil {
 			continue
 		}

pkg/installer/config_protect.go

@@ -0,0 +1,86 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"io/ioutil"
+	"path"
+	"regexp"
+
+	"github.com/ghodss/yaml"
+
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+func LoadConfigProtectConfs(c *LuetConfig) error {
+	var regexConfs = regexp.MustCompile(`.yml$`)
+
+	for _, cdir := range c.ConfigProtectConfDir {
+		Debug("Parsing Config Protect Directory", cdir, "...")
+
+		files, err := ioutil.ReadDir(cdir)
+		if err != nil {
+			Debug("Skip dir", cdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexConfs.MatchString(file.Name()) {
+				Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(cdir, file.Name()))
+			if err != nil {
+				Warning("On read file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := LoadConfigProtectConFile(file.Name(), content)
+			if err != nil {
+				Warning("On parse file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Directories) == 0 {
+				Warning("Invalid config protect file", file.Name())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddConfigProtectConfFile(r)
+		}
+	}
+	return nil
+
+}
+
+func LoadConfigProtectConFile(filename string, data []byte) (*ConfigProtectConfFile, error) {
+	ans := NewConfigProtectConfFile(filename)
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}

pkg/installer/finalizer.go

@@ -0,0 +1,90 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"os/exec"
+
+	"github.com/ghodss/yaml"
+	box "github.com/mudler/luet/pkg/box"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/pkg/errors"
+)
+
+type LuetFinalizer struct {
+	Shell     []string `json:"shell"`
+	Install   []string `json:"install"`
+	Uninstall []string `json:"uninstall"` // TODO: Where to store?
+}
+
+func (f *LuetFinalizer) RunInstall(s *System) error {
+	var cmd string
+	var args []string
+	if len(f.Shell) == 0 {
+		// Default to sh otherwise
+		cmd = "sh"
+		args = []string{"-c"}
+	} else {
+		cmd = f.Shell[0]
+		if len(f.Shell) > 1 {
+			args = f.Shell[1:]
+		}
+	}
+
+	for _, c := range f.Install {
+		toRun := append(args, c)
+		Info("Executing finalizer on ", s.Target, cmd, toRun)
+		if s.Target == "/" {
+			cmd := exec.Command(cmd, toRun...)
+			stdoutStderr, err := cmd.CombinedOutput()
+			if err != nil {
+				return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
+			}
+			Info(string(stdoutStderr))
+		} else {
+			b := box.NewBox(cmd, toRun, []string{}, []string{}, s.Target, false, true, true)
+			err := b.Run()
+			if err != nil {
+				return errors.Wrap(err, "Failed running command ")
+			}
+		}
+	}
+	return nil
+}
+
+// TODO: We don't store uninstall finalizers ?!
+func (f *LuetFinalizer) RunUnInstall() error {
+	for _, c := range f.Uninstall {
+		Debug("finalizer:", "sh", "-c", c)
+		cmd := exec.Command("sh", "-c", c)
+		stdoutStderr, err := cmd.CombinedOutput()
+		if err != nil {
+			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
+		}
+		Info(string(stdoutStderr))
+	}
+	return nil
+}
+
+func NewLuetFinalizerFromYaml(data []byte) (*LuetFinalizer, error) {
+	var p LuetFinalizer
+	err := yaml.Unmarshal(data, &p)
+	if err != nil {
+		return &p, err
+	}
+	return &p, err
+}

pkg/installer/installer.go

@@ -16,15 +16,16 @@
 package installer
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 
-	"github.com/ghodss/yaml"
 	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -34,9 +35,22 @@ import (
 	"github.com/pkg/errors"
 )
 
+type LuetInstallerOptions struct {
+	SolverOptions                                                  config.LuetSolverOptions
+	Concurrency                                                    int
+	NoDeps                                                         bool
+	OnlyDeps                                                       bool
+	Force                                                          bool
+	PreserveSystemEssentialData                                    bool
+	FullUninstall, FullCleanUninstall                              bool
+	CheckConflicts                                                 bool
+	SolverUpgrade, RemoveUnavailableOnUpgrade, UpgradeNewRevisions bool
+}
+
 type LuetInstaller struct {
 	PackageRepositories Repositories
-	Concurrency         int
+
+	Options LuetInstallerOptions
 }
 
 type ArtifactMatch struct {
@@ -45,49 +59,8 @@ type ArtifactMatch struct {
 	Repository Repository
 }
 
-type LuetFinalizer struct {
-	Install   []string `json:"install"`
-	Uninstall []string `json:"uninstall"` // TODO: Where to store?
-}
-
-func (f *LuetFinalizer) RunInstall() error {
-	for _, c := range f.Install {
-		Debug("finalizer:", "sh", "-c", c)
-		cmd := exec.Command("sh", "-c", c)
-		stdoutStderr, err := cmd.CombinedOutput()
-		if err != nil {
-			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
-		}
-		Info(string(stdoutStderr))
-	}
-	return nil
-}
-
-// TODO: We don't store uninstall finalizers ?!
-func (f *LuetFinalizer) RunUnInstall() error {
-	for _, c := range f.Install {
-		Debug("finalizer:", "sh", "-c", c)
-		cmd := exec.Command("sh", "-c", c)
-		stdoutStderr, err := cmd.CombinedOutput()
-		if err != nil {
-			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
-		}
-		Info(string(stdoutStderr))
-	}
-	return nil
-}
-
-func NewLuetFinalizerFromYaml(data []byte) (*LuetFinalizer, error) {
-	var p LuetFinalizer
-	err := yaml.Unmarshal(data, &p)
-	if err != nil {
-		return &p, err
-	}
-	return &p, err
-}
-
-func NewLuetInstaller(concurrency int) Installer {
-	return &LuetInstaller{Concurrency: concurrency}
+func NewLuetInstaller(opts LuetInstallerOptions) Installer {
+	return &LuetInstaller{Options: opts}
 }
 
 func (l *LuetInstaller) Upgrade(s *System) error {
@@ -95,32 +68,87 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 	if err != nil {
 		return err
 	}
+
+	Info(":thinking: Computing upgrade, please hang tight")
+
+	Spinner(32)
+	defer SpinnerStop()
 	// First match packages against repositories by priority
-	//	matches := syncedRepos.PackageMatches(p)
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	// compute a "big" world
-	solv := solver.NewSolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false))
-	uninstall, solution, err := solv.Upgrade()
-	if err != nil {
-		return errors.Wrap(err, "Failed solving solution for upgrade")
-	}
+	solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+	var uninstall pkg.Packages
+	var solution solver.PackagesAssertions
 
-	for _, u := range uninstall {
-		err := l.Uninstall(u, s)
+	if l.Options.SolverUpgrade {
+
+		uninstall, solution, err = solv.UpgradeUniverse(l.Options.RemoveUnavailableOnUpgrade)
 		if err != nil {
-			Warning("Failed uninstall for ", u.GetFingerPrint())
+			return errors.Wrap(err, "Failed solving solution for upgrade")
 		}
+	} else {
+		uninstall, solution, err = solv.Upgrade(!l.Options.FullUninstall, l.Options.NoDeps)
+		if err != nil {
+			return errors.Wrap(err, "Failed solving solution for upgrade")
+		}
+
+	}
+	SpinnerStop()
+
+	if len(uninstall) > 0 {
+		Info("Packages marked for uninstall:")
 	}
 
-	toInstall := []pkg.Package{}
+	for _, p := range uninstall {
+		Info(fmt.Sprintf("- %s", p.HumanReadableString()))
+	}
+
+	if len(solution) > 0 {
+		Info("Packages marked for upgrade:")
+	}
+
+	toInstall := pkg.Packages{}
 	for _, assertion := range solution {
-		if assertion.Value {
+		// Be sure to filter from solutions packages already installed in the system
+		if _, err := s.Database.FindPackage(assertion.Package); err != nil && assertion.Value {
+			Info(fmt.Sprintf("- %s", assertion.Package.HumanReadableString()))
 			toInstall = append(toInstall, assertion.Package)
 		}
 	}
 
-	return l.Install(toInstall, s)
+	if l.Options.UpgradeNewRevisions {
+		Info("Checking packages with new revisions available")
+		for _, p := range s.Database.World() {
+			matches := syncedRepos.PackageMatches(pkg.Packages{p})
+			if len(matches) == 0 {
+				// Package missing. the user should run luet upgrade --universe
+				Info("Installed packages seems to be missing from remote repositories.")
+				Info("It is suggested to run 'luet upgrade --universe'")
+				continue
+			}
+			for _, artefact := range matches[0].Repo.GetIndex() {
+				if artefact.GetCompileSpec().GetPackage() == nil {
+					return errors.New("Package in compilespec empty")
+
+				}
+				if artefact.GetCompileSpec().GetPackage().Matches(p) && artefact.GetCompileSpec().GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
+					toInstall = append(toInstall, matches[0].Package).Unique()
+					uninstall = append(uninstall, p).Unique()
+					Info(
+						fmt.Sprintf("- %s ( %s vs %s ) repo: %s (date: %s)",
+							p.HumanReadableString(),
+							artefact.GetCompileSpec().GetPackage().GetBuildTimestamp(),
+							p.GetBuildTimestamp(),
+							matches[0].Repo.GetName(),
+							matches[0].Repo.GetLastUpdate(),
+						))
+				}
+			}
+		}
+	}
+
+	return l.swap(syncedRepos, uninstall, toInstall, s)
 }
 
 func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
@@ -145,8 +173,156 @@ func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
 	return syncedRepos, nil
 }
 
-func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
-	var p []pkg.Package
+func (l *LuetInstaller) Swap(toRemove pkg.Packages, toInstall pkg.Packages, s *System) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+	return l.swap(syncedRepos, toRemove, toInstall, s)
+}
+
+func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) error {
+	// First match packages against repositories by priority
+	allRepos := pkg.NewInMemoryDatabase(false)
+	syncedRepos.SyncDatabase(allRepos)
+	toInstall = syncedRepos.ResolveSelectors(toInstall)
+
+	if err := l.download(syncedRepos, toInstall); err != nil {
+		return errors.Wrap(err, "Pre-downloading packages")
+	}
+
+	// We don't want any conflict with the installed to raise during the upgrade.
+	// In this way we both force uninstalls and we avoid to check with conflicts
+	// against the current system state which is pending to deletion
+	// E.g. you can't check for conflicts for an upgrade of a new version of A
+	// if the old A results installed in the system. This is due to the fact that
+	// now the solver enforces the constraints and explictly denies two packages
+	// of the same version installed.
+	forced := l.Options.Force
+
+	l.Options.Force = true
+
+	for _, u := range toRemove {
+		Info(":package:", u.HumanReadableString(), "Marked for deletion")
+
+		err := l.Uninstall(u, s)
+		if err != nil && !l.Options.Force {
+			Error("Failed uninstall for ", u.HumanReadableString())
+			return errors.Wrap(err, "uninstalling "+u.HumanReadableString())
+		}
+
+	}
+	l.Options.Force = forced
+
+	return l.install(syncedRepos, toInstall, s)
+}
+
+func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+	return l.install(syncedRepos, cp, s)
+}
+
+func (l *LuetInstaller) download(syncedRepos Repositories, cp pkg.Packages) error {
+	toDownload := map[string]ArtifactMatch{}
+
+	// FIXME: This can be optimized. We don't need to re-match this to the repository
+	// But we could just do it once
+
+	// Gathers things to download
+	for _, currentPack := range cp {
+		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
+		if len(matches) == 0 {
+			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
+		}
+	A:
+		for _, artefact := range matches[0].Repo.GetIndex() {
+			if artefact.GetCompileSpec().GetPackage() == nil {
+				return errors.New("Package in compilespec empty")
+
+			}
+			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+
+				toDownload[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
+
+				break A
+			}
+		}
+	}
+	// Download packages into cache in parallel.
+	all := make(chan ArtifactMatch)
+
+	var wg = new(sync.WaitGroup)
+
+	// Download
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.downloadWorker(i, wg, all)
+	}
+	for _, c := range toDownload {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
+
+	return nil
+}
+
+// Reclaim adds packages to the system database
+// if files from artifacts in the repositories are found
+// in the system target
+func (l *LuetInstaller) Reclaim(s *System) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+
+	var toMerge []ArtifactMatch = []ArtifactMatch{}
+
+	for _, repo := range syncedRepos {
+		for _, artefact := range repo.GetIndex() {
+			Debug("Checking if",
+				artefact.GetCompileSpec().GetPackage().HumanReadableString(),
+				"from", repo.GetName(), "is installed")
+		FILES:
+			for _, f := range artefact.GetFiles() {
+				if helpers.Exists(filepath.Join(s.Target, f)) {
+					p, err := repo.GetTree().GetDatabase().FindPackage(artefact.GetCompileSpec().GetPackage())
+					if err != nil {
+						return err
+					}
+					Info("Found package:", p.HumanReadableString())
+					toMerge = append(toMerge, ArtifactMatch{Artifact: artefact, Package: p})
+					break FILES
+				}
+			}
+		}
+	}
+
+	for _, match := range toMerge {
+		pack := match.Package
+		vers, _ := s.Database.FindPackageVersions(pack)
+
+		if len(vers) >= 1 {
+			Warning("Filtering out package " + pack.HumanReadableString() + ", already reclaimed")
+			continue
+		}
+		_, err := s.Database.CreatePackage(pack)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed creating package")
+		}
+		s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.GetFiles()})
+		Info("Reclaimed package:", pack.HumanReadableString())
+	}
+	Info("Done!")
+
+	return nil
+}
+
+func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *System) error {
+	var p pkg.Packages
 
 	// Check if the package is installed first
 	for _, pi := range cp {
@@ -154,7 +330,7 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 		vers, _ := s.Database.FindPackageVersions(pi)
 
 		if len(vers) >= 1 {
-			Warning("Filtering out package " + pi.GetFingerPrint() + ", it has other versions already installed. Uninstall one of them first ")
+			Warning("Filtering out package " + pi.HumanReadableString() + ", it has other versions already installed. Uninstall one of them first ")
 			continue
 			//return errors.New("Package " + pi.GetFingerPrint() + " has other versions already installed. Uninstall one of them first: " + strings.Join(vers, " "))
 
@@ -168,10 +344,6 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 	}
 	// First get metas from all repos (and decodes trees)
 
-	syncedRepos, err := l.SyncRepositories(true)
-	if err != nil {
-		return err
-	}
 	// First match packages against repositories by priority
 	//	matches := syncedRepos.PackageMatches(p)
 
@@ -179,42 +351,80 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	p = syncedRepos.ResolveSelectors(p)
-	solv := solver.NewSolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false))
-	solution, err := solv.Install(p)
-	if err != nil {
-		return errors.Wrap(err, "Failed solving solution for package")
-	}
-
-	// Gathers things to install
 	toInstall := map[string]ArtifactMatch{}
-	for _, assertion := range solution {
-		if assertion.Value {
-			matches := syncedRepos.PackageMatches([]pkg.Package{assertion.Package})
-			if len(matches) == 0 {
-				return errors.New("Failed matching solutions against repository - where are definitions coming from?!")
-			}
-		A:
-			for _, artefact := range matches[0].Repo.GetIndex() {
-				if artefact.GetCompileSpec().GetPackage() == nil {
-					return errors.New("Package in compilespec empty")
+	var packagesToInstall pkg.Packages
+	var err error
+	var solution solver.PackagesAssertions
 
+	if !l.Options.NoDeps {
+		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		solution, err = solv.Install(p)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed solving solution for package")
+		}
+		// Gathers things to install
+		for _, assertion := range solution {
+			if assertion.Value {
+				packagesToInstall = append(packagesToInstall, assertion.Package)
+			}
+		}
+	} else if !l.Options.OnlyDeps {
+		for _, currentPack := range p {
+			packagesToInstall = append(packagesToInstall, currentPack)
+		}
+	}
+	Info(":deciduous_tree: Finding packages to install")
+	// Gathers things to install
+	for _, currentPack := range packagesToInstall {
+		// Check if package is already installed.
+		if _, err := s.Database.FindPackage(currentPack); err == nil {
+			// skip matching if it is installed already
+			continue
+		}
+		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
+		if len(matches) == 0 {
+			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
+		}
+	A:
+		for _, artefact := range matches[0].Repo.GetIndex() {
+			if artefact.GetCompileSpec().GetPackage() == nil {
+				return errors.New("Package in compilespec empty")
+
+			}
+			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+				currentPack.SetBuildTimestamp(artefact.GetCompileSpec().GetPackage().GetBuildTimestamp())
+				// Filter out already installed
+				if _, err := s.Database.FindPackage(currentPack); err != nil {
+					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
+					Info("\t:package:", currentPack.HumanReadableString(), "from repository", matches[0].Repo.GetName())
 				}
-				if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
-					// Filter out already installed
-					if _, err := s.Database.FindPackage(assertion.Package); err != nil {
-						toInstall[assertion.Package.GetFingerPrint()] = ArtifactMatch{Package: assertion.Package, Artifact: artefact, Repository: matches[0].Repo}
-					}
-					break A
-				}
+				break A
 			}
 		}
 	}
-
 	// Install packages into rootfs in parallel.
 	all := make(chan ArtifactMatch)
 
 	var wg = new(sync.WaitGroup)
-	for i := 0; i < l.Concurrency; i++ {
+
+	// Download first
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.downloadWorker(i, wg, all)
+	}
+
+	for _, c := range toInstall {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
+
+	all = make(chan ArtifactMatch)
+
+	wg = new(sync.WaitGroup)
+
+	// Do the real install
+	for i := 0; i < l.Options.Concurrency; i++ {
 		wg.Add(1)
 		go l.installerWorker(i, wg, all, s)
 	}
@@ -225,82 +435,119 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 	close(all)
 	wg.Wait()
 
+	for _, c := range toInstall {
+		// Annotate to the system that the package was installed
+		_, err := s.Database.CreatePackage(c.Package)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed creating package")
+		}
+	}
 	executedFinalizer := map[string]bool{}
 
-	// TODO: Lower those errors as warning
-	for _, w := range p {
-		// Finalizers needs to run in order and in sequence.
-		ordered := solution.Order(allRepos, w.GetFingerPrint())
-		for _, ass := range ordered {
-			if ass.Value {
-				// Annotate to the system that the package was installed
-				// TODO: Annotate also files that belong to the package, somewhere to uninstall
-				if _, err := s.Database.FindPackage(ass.Package); err == nil {
-					err := s.Database.UpdatePackage(ass.Package)
-					if err != nil {
-						return errors.Wrap(err, "Failed updating package")
-					}
-				} else {
-					_, err := s.Database.CreatePackage(ass.Package)
-					if err != nil {
-						return errors.Wrap(err, "Failed creating package")
-					}
-				}
-				installed, ok := toInstall[ass.Package.GetFingerPrint()]
-				if !ok {
-					return errors.New("Couldn't find ArtifactMatch for " + ass.Package.GetFingerPrint())
-				}
+	if !l.Options.NoDeps {
+		// TODO: Lower those errors as warning
+		for _, w := range p {
+			// Finalizers needs to run in order and in sequence.
+			ordered, err := solution.Order(allRepos, w.GetFingerPrint())
+			if err != nil {
+				return errors.Wrap(err, "While order a solution for "+w.HumanReadableString())
+			}
+		ORDER:
+			for _, ass := range ordered {
+				if ass.Value {
 
-				treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
-				if err != nil {
-					return errors.Wrap(err, "Error getting package "+ass.Package.GetFingerPrint())
-				}
-				if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
-					Info("Executing finalizer for " + ass.Package.GetName())
-					finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
+					installed, ok := toInstall[ass.Package.GetFingerPrint()]
+					if !ok {
+						// It was a dep already installed in the system, so we can skip it safely
+						continue ORDER
+					}
+
+					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
 					if err != nil {
-						return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
+						return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
 					}
-					if _, exists := executedFinalizer[ass.Package.GetFingerPrint()]; !exists {
-						finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
-						if err != nil {
-							return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
+					if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
+						Info("Executing finalizer for " + ass.Package.HumanReadableString())
+						finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
+						if err != nil && !l.Options.Force {
+							return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
 						}
-						err = finalizer.RunInstall()
-						if err != nil {
-							return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
+						if _, exists := executedFinalizer[ass.Package.GetFingerPrint()]; !exists {
+							finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
+							if err != nil && !l.Options.Force {
+								return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
+							}
+							err = finalizer.RunInstall(s)
+							if err != nil && !l.Options.Force {
+								return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
+							}
+							executedFinalizer[ass.Package.GetFingerPrint()] = true
 						}
-						executedFinalizer[ass.Package.GetFingerPrint()] = true
 					}
+
+				}
+			}
+
+		}
+	} else {
+		for _, c := range toInstall {
+			treePackage, err := c.Repository.GetTree().GetDatabase().FindPackage(c.Package)
+			if err != nil {
+				return errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
+			}
+			if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
+				Info("Executing finalizer for " + c.Package.HumanReadableString())
+				finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
+				if err != nil && !l.Options.Force {
+					return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
+				}
+				if _, exists := executedFinalizer[c.Package.GetFingerPrint()]; !exists {
+					finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
+					if err != nil && !l.Options.Force {
+						return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
+					}
+					err = finalizer.RunInstall(s)
+					if err != nil && !l.Options.Force {
+						return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
+					}
+					executedFinalizer[c.Package.GetFingerPrint()] = true
 				}
 			}
 		}
-
 	}
 
 	return nil
 
 }
 
-func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
+func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) {
 
 	artifact, err := a.Repository.Client().DownloadArtifact(a.Artifact)
 	if err != nil {
-		return errors.Wrap(err, "Error on download artifact")
+		return nil, errors.Wrap(err, "Error on download artifact")
 	}
 
 	err = artifact.Verify()
-	if err != nil {
-		return errors.Wrap(err, "Artifact integrity check failure")
+	if err != nil && !l.Options.Force {
+		return nil, errors.Wrap(err, "Artifact integrity check failure")
+	}
+	return artifact, nil
+}
+
+func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
+
+	artifact, err := l.downloadPackage(a)
+	if err != nil && !l.Options.Force {
+		return errors.Wrap(err, "Failed downloading package")
 	}
 
 	files, err := artifact.FileList()
-	if err != nil {
+	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Could not open package archive")
 	}
 
 	err = artifact.Unpack(s.Target, true)
-	if err != nil {
+	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Error met while unpacking rootfs")
 	}
 
@@ -309,17 +556,43 @@ func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
 	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.Package.GetFingerPrint(), Files: files})
 }
 
+func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch) error {
+	defer wg.Done()
+
+	for p := range c {
+		// TODO: Keep trace of what was added from the tar, and save it into system
+		_, err := l.downloadPackage(p)
+		if err != nil && !l.Options.Force {
+			//TODO: Uninstall, rollback.
+			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
+			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
+		}
+		if err == nil {
+			Info(":package: ", p.Package.HumanReadableString(), "downloaded")
+		} else if err != nil && l.Options.Force {
+			Info(":package: ", p.Package.HumanReadableString(), "downloaded with failures (force download)")
+		}
+	}
+
+	return nil
+}
+
 func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System) error {
 	defer wg.Done()
 
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
 		err := l.installPackage(p, s)
-		if err != nil {
+		if err != nil && !l.Options.Force {
 			//TODO: Uninstall, rollback.
 			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
 			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
 		}
+		if err == nil {
+			Info(":package: ", p.Package.HumanReadableString(), "installed")
+		} else if err != nil && l.Options.Force {
+			Info(":package: ", p.Package.HumanReadableString(), "installed with failures (force install)")
+		}
 	}
 
 	return nil
@@ -334,7 +607,15 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 	// Remove from target
 	for _, f := range files {
 		target := filepath.Join(s.Target, f)
-		Info("Removing", target)
+		Debug("Removing", target)
+
+		if l.Options.PreserveSystemEssentialData &&
+			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) ||
+			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath()) {
+			Warning("Preserve ", f, " which is required by luet ( you have to delete it manually if you really need to)")
+			continue
+		}
+
 		err := os.Remove(target)
 		if err != nil {
 			Warning("Failed removing file (not present in the system target ?)", target)
@@ -354,20 +635,62 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 }
 
 func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
-	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) - mark the uninstallation in db
-	// Get installed definition
+	Spinner(32)
+	defer SpinnerStop()
 
-	solv := solver.NewSolver(s.Database, s.Database, pkg.NewInMemoryDatabase(false))
-	solution, err := solv.Uninstall(p)
-	if err != nil {
-		return errors.Wrap(err, "Uninstall failed")
+	Info("Uninstalling :package:", p.HumanReadableString(), "hang tight")
+
+	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) TODO - mark the uninstallation in db
+	// Get installed definition
+	checkConflicts := l.Options.CheckConflicts
+	full := l.Options.FullUninstall
+	if l.Options.Force == true { // IF forced, we want to remove the package and all its requires
+		checkConflicts = false
+		full = false
 	}
-	for _, p := range solution {
-		Info("Uninstalling", p.GetFingerPrint())
-		err := l.uninstall(p, s)
+
+	// Create a temporary DB with the installed packages
+	// so the solver is much faster finding the deptree
+	installedtmp := pkg.NewInMemoryDatabase(false)
+
+	for _, i := range s.Database.World() {
+		_, err := installedtmp.CreatePackage(i)
 		if err != nil {
+			return errors.Wrap(err, "Failed create temporary in-memory db")
+		}
+	}
+
+	if !l.Options.NoDeps {
+		Info("Finding :package:", p.HumanReadableString(), "dependency graph :deciduous_tree:")
+		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		var solution pkg.Packages
+		var err error
+		if l.Options.FullCleanUninstall {
+			solution, err = solv.UninstallUniverse(pkg.Packages{p})
+			if err != nil {
+				return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+			}
+		} else {
+			solution, err = solv.Uninstall(p, checkConflicts, full)
+			if err != nil && !l.Options.Force {
+				return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+			}
+		}
+
+		for _, p := range solution {
+			Info("Uninstalling", p.HumanReadableString())
+			err := l.uninstall(p, s)
+			if err != nil && !l.Options.Force {
+				return errors.Wrap(err, "Uninstall failed")
+			}
+		}
+	} else {
+		Info("Uninstalling", p.HumanReadableString(), "without deps")
+		err := l.uninstall(p, s)
+		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Uninstall failed")
 		}
+		Info(":package:", p.HumanReadableString(), "uninstalled")
 	}
 	return nil
 

pkg/installer/installer_test.go

@@ -24,6 +24,8 @@ import (
 	compiler "github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/helpers"
+	solver "github.com/mudler/luet/pkg/solver"
+
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
@@ -47,9 +49,9 @@ var _ = Describe("Installer", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
@@ -62,9 +64,9 @@ var _ = Describe("Installer", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
+			c.SetConcurrency(2)
 
-			artifact, err := compiler.Compile(false, spec)
+			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -82,16 +84,18 @@ var _ = Describe("Installer", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -99,7 +103,126 @@ var _ = Describe("Installer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(fakeroot) // clean up
 
-			inst := NewLuetInstaller(1)
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+
+			inst.Repositories(Repositories{repo2})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+			systemDB := pkg.NewInMemoryDatabase(false)
+			system := &System{Database: systemDB, Target: fakeroot}
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			files, err := systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(files).To(Equal([]string{"artifact42", "test5", "test6"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.GetPackages())).To(Equal(1))
+			p, err := system.Database.GetPackage(system.Database.GetPackages()[0])
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetName()).To(Equal("b"))
+
+			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			// Nothing should be there anymore (files, packagedb entry)
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+			_, err = systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+		})
+
+	})
+
+	Context("Writes a repository definition without compression", func() {
+		It("Writes a repo and can install packages from it", func() {
+			//repo:=NewLuetSystemRepository()
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
+				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+
+			spec.SetOutputPath(tmpdir)
+			c.SetConcurrency(2)
+
+			artifact, err := c.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			content1, err := helpers.Read(spec.Rel("test5"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := helpers.Read(spec.Rel("test6"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("artifact5\n"))
+			Expect(content2).To(Equal("artifact6\n"))
+
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			treeFile := NewDefaultTreeRepositoryFile()
+			treeFile.SetCompressionType(compiler.None)
+			repo.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
 			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
 name: "test"
 type: "disk"
@@ -160,9 +283,9 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
@@ -175,9 +298,9 @@ urls:
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
+			c.SetConcurrency(2)
 
-			artifact, err := compiler.Compile(false, spec)
+			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -195,16 +318,18 @@ urls:
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -212,7 +337,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(fakeroot) // clean up
 
-			inst := NewLuetInstaller(1)
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
 			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
 name: "test"
 type: "disk"
@@ -262,6 +387,149 @@ urls:
 
 		})
 
+		It("Installs new packages from a syste with others installed", func() {
+			//repo:=NewLuetSystemRepository()
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+
+			spec.SetOutputPath(tmpdir)
+			c.SetConcurrency(2)
+
+			artifact, err := c.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			content1, err := helpers.Read(spec.Rel("test5"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := helpers.Read(spec.Rel("test6"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("artifact5\n"))
+			Expect(content2).To(Equal("artifact6\n"))
+
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+
+			inst.Repositories(Repositories{repo2})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			tmpdir2, err := ioutil.TempDir("", "tree2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe2 := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe2.Load("../../tests/fixtures/alpine")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
+
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			spec.SetOutputPath(tmpdir2)
+			c.SetConcurrency(2)
+
+			artifact, err = c.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+
+			repo, err = GenerateRepository("test", "description", "disk", []string{tmpdir2}, 1, tmpdir2, []string{"../../tests/fixtures/alpine"}, pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Write(tmpdir2, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			fakeroot, err = ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			inst = NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err = NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir2+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			inst.Repositories(Repositories{repo2})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir2))
+			Expect(repo.GetType()).To(Equal("disk"))
+			system.Target = fakeroot
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+		})
+
 	})
 
 	Context("Simple upgrades", func() {
@@ -279,7 +547,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -303,16 +571,18 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade"}, pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -320,7 +590,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(fakeroot) // clean up
 
-			inst := NewLuetInstaller(1)
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
 			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
 name: "test"
 type: "disk"
@@ -376,6 +646,140 @@ urls:
 
 		})
 
+		It("Handles package drops", func() {
+			//repo:=NewLuetSystemRepository()
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			generalRecipeNewRepo := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/upgrade_old_repo")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = generalRecipeNewRepo.Load("../../tests/fixtures/upgrade_new_repo")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+			Expect(len(generalRecipeNewRepo.GetDatabase().GetPackages())).To(Equal(3))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec3, err := c.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec2, err := c2.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			tmpdirnewrepo, err := ioutil.TempDir("", "tree2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdirnewrepo) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdirnewrepo)
+			spec3.SetOutputPath(tmpdir)
+			c.SetConcurrency(2)
+
+			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec3))
+
+			Expect(errs).To(BeEmpty())
+
+			_, errs = c2.CompileParallel(false, compiler.NewLuetCompilationspecs(spec2))
+			Expect(errs).To(BeEmpty())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade_old_repo"}, pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			repoupgrade, err := GenerateRepository("test", "description", "disk", []string{tmpdirnewrepo}, 1, tmpdirnewrepo, []string{"../../tests/fixtures/upgrade_new_repo"}, pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			err = repoupgrade.Write(tmpdirnewrepo, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+
+			repoupgrade2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdirnewrepo+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+
+			inst.Repositories(Repositories{repo2})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.GetPackages())).To(Equal(1))
+			p, err := system.Database.GetPackage(system.Database.GetPackages()[0])
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetName()).To(Equal("b"))
+
+			files, err := systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(files).To(Equal([]string{"artifact42", "test5", "test6"}))
+			Expect(err).ToNot(HaveOccurred())
+			inst.Repositories(Repositories{repoupgrade2})
+
+			err = inst.Upgrade(system)
+			Expect(err).ToNot(HaveOccurred())
+
+			// Nothing should be there anymore (files, packagedb entry)
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+
+			// New version - new files
+			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+
+			// New package should be there
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+		})
+
 	})
 
 	Context("Compressed packages", func() {
@@ -393,7 +797,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -416,18 +820,20 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade"}, pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -435,7 +841,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(fakeroot) // clean up
 
-			inst := NewLuetInstaller(1)
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
 			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
 name: "test"
 type: "disk"
@@ -493,4 +899,267 @@ urls:
 
 	})
 
+	Context("Existing files", func() {
+		It("Reclaims them", func() {
+			//repo:=NewLuetSystemRepository()
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/upgrade")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec3, err := c.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+			spec3.SetOutputPath(tmpdir)
+			c.SetConcurrency(2)
+			c.SetCompressionType(compiler.GZip)
+			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+
+			Expect(errs).To(BeEmpty())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade"}, pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+
+			inst.Repositories(Repositories{repo2})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+
+			Expect(len(system.Database.World())).To(Equal(0))
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeFalse())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeFalse())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "c"))).To(BeFalse())
+
+			Expect(helpers.Touch(filepath.Join(fakeroot, "test5"))).ToNot(HaveOccurred())
+			Expect(helpers.Touch(filepath.Join(fakeroot, "test6"))).ToNot(HaveOccurred())
+			Expect(helpers.Touch(filepath.Join(fakeroot, "c"))).ToNot(HaveOccurred())
+
+			err = inst.Reclaim(system)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.World())).To(Equal(2))
+		})
+
+		It("Upgrades reclaimed packages", func() {
+			//repo:=NewLuetSystemRepository()
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/upgrade_old_repo")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec3, err := c.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			spec.SetOutputPath(tmpdir)
+			spec3.SetOutputPath(tmpdir)
+			c.SetConcurrency(1)
+			c.SetCompressionType(compiler.GZip)
+			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec3))
+
+			Expect(errs).To(BeEmpty())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade_old_repo"}, pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+
+			inst.Repositories(Repositories{repo2})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+
+			Expect(len(system.Database.World())).To(Equal(0))
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeFalse())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeFalse())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "c"))).To(BeFalse())
+
+			Expect(helpers.Touch(filepath.Join(fakeroot, "test5"))).ToNot(HaveOccurred())
+			Expect(helpers.Touch(filepath.Join(fakeroot, "test6"))).ToNot(HaveOccurred())
+			Expect(helpers.Touch(filepath.Join(fakeroot, "c"))).ToNot(HaveOccurred())
+
+			err = inst.Reclaim(system)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.World())).To(Equal(2))
+
+			generalRecipe2 := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe2.Load("../../tests/fixtures/upgrade_new_repo")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(3))
+
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+			tmpdir2, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir2) // clean up
+			spec.SetOutputPath(tmpdir2)
+
+			_, errs = c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec))
+
+			Expect(errs).To(BeEmpty())
+
+			repo, err = GenerateRepository("test", "description", "disk", []string{tmpdir2}, 1, tmpdir2, []string{"../../tests/fixtures/upgrade_new_repo"}, pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			err = repo.Write(tmpdir2, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			inst = NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err = NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir2+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+
+			inst.Repositories(Repositories{repo2})
+			err = inst.Upgrade(system)
+			Expect(err).ToNot(HaveOccurred())
+
+			// Nothing should be there anymore (files, packagedb entry)
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+
+			// New version - new files
+			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+
+			// New package should be there
+			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+		})
+	})
+
 })

pkg/installer/interface.go

@@ -23,11 +23,14 @@ import (
 )
 
 type Installer interface {
-	Install([]pkg.Package, *System) error
+	Install(pkg.Packages, *System) error
 	Uninstall(pkg.Package, *System) error
 	Upgrade(s *System) error
+	Reclaim(s *System) error
+
 	Repositories([]Repository)
 	SyncRepositories(bool) (Repositories, error)
+	Swap(pkg.Packages, pkg.Packages, *System) error
 }
 
 type Client interface {
@@ -45,12 +48,15 @@ type Repository interface {
 	AddUrl(string)
 	GetPriority() int
 	GetIndex() compiler.ArtifactIndex
+	SetIndex(i compiler.ArtifactIndex)
 	GetTree() tree.Builder
 	SetTree(tree.Builder)
 	Write(path string, resetRevision bool) error
 	Sync(bool) (Repository, error)
 	GetTreePath() string
 	SetTreePath(string)
+	GetMetaPath() string
+	SetMetaPath(string)
 	GetType() string
 	SetType(string)
 	SetAuthentication(map[string]string)
@@ -61,8 +67,9 @@ type Repository interface {
 	SetLastUpdate(string)
 	Client() Client
 
-	GetTreeChecksums() compiler.Checksums
-	GetTreeCompressionType() compiler.CompressionImplementation
-	SetTreeCompressionType(c compiler.CompressionImplementation)
-	SetTreeChecksums(c compiler.Checksums)
+	SetPriority(int)
+	GetRepositoryFile(string) (LuetRepositoryFile, error)
+	SetRepositoryFile(string, LuetRepositoryFile)
+	SetName(p string)
+	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
 }

pkg/installer/repository.go

@@ -21,7 +21,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -36,47 +35,159 @@ import (
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/ghodss/yaml"
-	. "github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 )
 
 const (
+	REPOSITORY_METAFILE = "repository.meta.yaml"
 	REPOSITORY_SPECFILE = "repository.yaml"
 	TREE_TARBALL        = "tree.tar"
+
+	REPOFILE_TREE_KEY = "tree"
+	REPOFILE_META_KEY = "meta"
 )
 
+type LuetRepositoryFile struct {
+	FileName        string                             `json:"filename"`
+	CompressionType compiler.CompressionImplementation `json:"compressiontype,omitempty"`
+	Checksums       compiler.Checksums                 `json:"checksums,omitempty"`
+}
+
 type LuetSystemRepository struct {
 	*config.LuetRepository
 
-	Index               compiler.ArtifactIndex             `json:"index"`
-	Tree                tree.Builder                       `json:"-"`
-	TreePath            string                             `json:"treepath"`
-	TreeCompressionType compiler.CompressionImplementation `json:"treecompressiontype"`
-	TreeChecksums       compiler.Checksums                 `json:"treechecksums"`
+	Index           compiler.ArtifactIndex        `json:"index"`
+	Tree            tree.Builder                  `json:"-"`
+	RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"`
 }
 
 type LuetSystemRepositorySerialized struct {
-	Name                string                             `json:"name"`
-	Description         string                             `json:"description,omitempty"`
-	Urls                []string                           `json:"urls"`
-	Priority            int                                `json:"priority"`
-	Index               []*compiler.PackageArtifact        `json:"index"`
-	Type                string                             `json:"type"`
-	Revision            int                                `json:"revision,omitempty"`
-	LastUpdate          string                             `json:"last_update,omitempty"`
-	TreePath            string                             `json:"treepath"`
-	TreeCompressionType compiler.CompressionImplementation `json:"treecompressiontype"`
-	TreeChecksums       compiler.Checksums                 `json:"treechecksums"`
+	Name            string                        `json:"name"`
+	Description     string                        `json:"description,omitempty"`
+	Urls            []string                      `json:"urls"`
+	Priority        int                           `json:"priority"`
+	Type            string                        `json:"type"`
+	Revision        int                           `json:"revision,omitempty"`
+	LastUpdate      string                        `json:"last_update,omitempty"`
+	TreePath        string                        `json:"treepath"`
+	MetaPath        string                        `json:"metapath"`
+	RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"`
 }
 
-func GenerateRepository(name, descr, t string, urls []string, priority int, src, treeDir string, db pkg.PackageDatabase) (Repository, error) {
+type LuetSystemRepositoryMetadata struct {
+	Index []*compiler.PackageArtifact `json:"index,omitempty"`
+}
 
-	art, err := buildPackageIndex(src)
+type LuetSearchModeType string
+
+const (
+	SLabel      LuetSearchModeType = "label"
+	SRegexPkg   LuetSearchModeType = "regexPkg"
+	SRegexLabel LuetSearchModeType = "regexLabel"
+)
+
+type LuetSearchOpts struct {
+	Pattern string
+	Mode    LuetSearchModeType
+}
+
+func NewLuetSystemRepositoryMetadata(file string, removeFile bool) (*LuetSystemRepositoryMetadata, error) {
+	ans := &LuetSystemRepositoryMetadata{}
+	err := ans.ReadFile(file, removeFile)
 	if err != nil {
 		return nil, err
 	}
+	return ans, nil
+}
+
+func (m *LuetSystemRepositoryMetadata) WriteFile(path string) error {
+	data, err := yaml.Marshal(m)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(path, data, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *LuetSystemRepositoryMetadata) ReadFile(file string, removeFile bool) error {
+	if file == "" {
+		return errors.New("Invalid path for repository metadata")
+	}
+
+	dat, err := ioutil.ReadFile(file)
+	if err != nil {
+		return err
+	}
+	if removeFile {
+		defer os.Remove(file)
+	}
+
+	err = yaml.Unmarshal(dat, m)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *LuetSystemRepositoryMetadata) ToArtifactIndex() (ans compiler.ArtifactIndex) {
+	for _, a := range m.Index {
+		ans = append(ans, a)
+	}
+	return
+}
+
+func NewDefaultTreeRepositoryFile() LuetRepositoryFile {
+	return LuetRepositoryFile{
+		FileName:        TREE_TARBALL,
+		CompressionType: compiler.GZip,
+	}
+}
+
+func NewDefaultMetaRepositoryFile() LuetRepositoryFile {
+	return LuetRepositoryFile{
+		FileName:        REPOSITORY_METAFILE + ".tar",
+		CompressionType: compiler.None,
+	}
+}
+
+func (f *LuetRepositoryFile) SetFileName(n string) {
+	f.FileName = n
+}
+
+func (f *LuetRepositoryFile) GetFileName() string {
+	return f.FileName
+}
+func (f *LuetRepositoryFile) SetCompressionType(c compiler.CompressionImplementation) {
+	f.CompressionType = c
+}
+func (f *LuetRepositoryFile) GetCompressionType() compiler.CompressionImplementation {
+	return f.CompressionType
+}
+func (f *LuetRepositoryFile) SetChecksums(c compiler.Checksums) {
+	f.Checksums = c
+}
+func (f *LuetRepositoryFile) GetChecksums() compiler.Checksums {
+	return f.Checksums
+}
+
+func GenerateRepository(name, descr, t string, urls []string, priority int, src string, treesDir []string, db pkg.PackageDatabase) (Repository, error) {
+
 	tr := tree.NewInstallerRecipe(db)
-	err = tr.Load(treeDir)
+
+	for _, treeDir := range treesDir {
+		err := tr.Load(treeDir)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	art, err := buildPackageIndex(src, tr.GetDatabase())
 	if err != nil {
 		return nil, err
 	}
@@ -88,15 +199,17 @@ func GenerateRepository(name, descr, t string, urls []string, priority int, src,
 
 func NewSystemRepository(repo config.LuetRepository) Repository {
 	return &LuetSystemRepository{
-		LuetRepository: &repo,
+		LuetRepository:  &repo,
+		RepositoryFiles: map[string]LuetRepositoryFile{},
 	}
 }
 
 func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder) Repository {
 	return &LuetSystemRepository{
-		LuetRepository: repo,
-		Index:          art,
-		Tree:           builder,
+		LuetRepository:  repo,
+		Index:           art,
+		Tree:            builder,
+		RepositoryFiles: map[string]LuetRepositoryFile{},
 	}
 }
 
@@ -106,6 +219,7 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 	if err != nil {
 		return nil, err
 	}
+
 	r := &LuetSystemRepository{
 		LuetRepository: config.NewLuetRepository(
 			p.Name,
@@ -116,9 +230,7 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 			true,
 			false,
 		),
-		TreeCompressionType: p.TreeCompressionType,
-		TreeChecksums:       p.TreeChecksums,
-		TreePath:            p.TreePath,
+		RepositoryFiles: p.RepositoryFiles,
 	}
 	if p.Revision > 0 {
 		r.Revision = p.Revision
@@ -126,17 +238,12 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 	if p.LastUpdate != "" {
 		r.LastUpdate = p.LastUpdate
 	}
-	i := compiler.ArtifactIndex{}
-	for _, ii := range p.Index {
-		i = append(i, ii)
-	}
-	r.Index = i
 	r.Tree = tree.NewInstallerRecipe(db)
 
 	return r, err
 }
 
-func buildPackageIndex(path string) ([]compiler.Artifact, error) {
+func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) {
 
 	var art []compiler.Artifact
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
@@ -154,6 +261,15 @@ func buildPackageIndex(path string) ([]compiler.Artifact, error) {
 		if err != nil {
 			return errors.Wrap(err, "Error reading yaml "+currentpath)
 		}
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
+			Info(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				artifact.GetCompileSpec().GetPackage().HumanReadableString()))
+			return nil
+		}
+
 		art = append(art, artifact)
 
 		return nil
@@ -167,6 +283,10 @@ func buildPackageIndex(path string) ([]compiler.Artifact, error) {
 	return art, nil
 }
 
+func (r *LuetSystemRepository) SetPriority(n int) {
+	r.LuetRepository.Priority = n
+}
+
 func (r *LuetSystemRepository) GetName() string {
 	return r.LuetRepository.Name
 }
@@ -178,28 +298,17 @@ func (r *LuetSystemRepository) GetAuthentication() map[string]string {
 	return r.LuetRepository.Authentication
 }
 
-func (r *LuetSystemRepository) GetTreeCompressionType() compiler.CompressionImplementation {
-	return r.TreeCompressionType
-}
-
-func (r *LuetSystemRepository) GetTreeChecksums() compiler.Checksums {
-	return r.TreeChecksums
-}
-
-func (r *LuetSystemRepository) SetTreeCompressionType(c compiler.CompressionImplementation) {
-	r.TreeCompressionType = c
-}
-
-func (r *LuetSystemRepository) SetTreeChecksums(c compiler.Checksums) {
-	r.TreeChecksums = c
-}
-
 func (r *LuetSystemRepository) GetType() string {
 	return r.LuetRepository.Type
 }
 func (r *LuetSystemRepository) SetType(p string) {
 	r.LuetRepository.Type = p
 }
+
+func (r *LuetSystemRepository) SetName(p string) {
+	r.LuetRepository.Name = p
+}
+
 func (r *LuetSystemRepository) AddUrl(p string) {
 	r.LuetRepository.Urls = append(r.LuetRepository.Urls, p)
 }
@@ -218,12 +327,21 @@ func (r *LuetSystemRepository) GetTreePath() string {
 func (r *LuetSystemRepository) SetTreePath(p string) {
 	r.TreePath = p
 }
+func (r *LuetSystemRepository) GetMetaPath() string {
+	return r.MetaPath
+}
+func (r *LuetSystemRepository) SetMetaPath(p string) {
+	r.MetaPath = p
+}
 func (r *LuetSystemRepository) SetTree(b tree.Builder) {
 	r.Tree = b
 }
 func (r *LuetSystemRepository) GetIndex() compiler.ArtifactIndex {
 	return r.Index
 }
+func (r *LuetSystemRepository) SetIndex(i compiler.ArtifactIndex) {
+	r.Index = i
+}
 func (r *LuetSystemRepository) GetTree() tree.Builder {
 	return r.Tree
 }
@@ -239,10 +357,19 @@ func (r *LuetSystemRepository) SetLastUpdate(u string) {
 func (r *LuetSystemRepository) IncrementRevision() {
 	r.LuetRepository.Revision++
 }
-
 func (r *LuetSystemRepository) SetAuthentication(auth map[string]string) {
 	r.LuetRepository.Authentication = auth
 }
+func (r *LuetSystemRepository) GetRepositoryFile(name string) (LuetRepositoryFile, error) {
+	ans, ok := r.RepositoryFiles[name]
+	if ok {
+		return ans, nil
+	}
+	return ans, errors.New("Repository file " + name + " not found!")
+}
+func (r *LuetSystemRepository) SetRepositoryFile(name string, f LuetRepositoryFile) {
+	r.RepositoryFiles[name] = f
+}
 
 func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repository, error) {
 	dat, err := ioutil.ReadFile(file)
@@ -259,6 +386,16 @@ func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repos
 		return nil, errors.Wrap(err, "Error reading repository from file "+file)
 	}
 
+	// Check if mandatory key are present
+	_, err = repo.GetRepositoryFile(REPOFILE_TREE_KEY)
+	if err != nil {
+		return nil, errors.New("Invalid repository without the " + REPOFILE_TREE_KEY + " key file.")
+	}
+	_, err = repo.GetRepositoryFile(REPOFILE_META_KEY)
+	if err != nil {
+		return nil, errors.New("Invalid repository without the " + REPOFILE_META_KEY + " key file.")
+	}
+
 	return repo, err
 }
 
@@ -267,7 +404,6 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 	if err != nil {
 		return err
 	}
-	r.Index = r.Index.CleanPath()
 	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
 
 	repospec := filepath.Join(dst, REPOSITORY_SPECFILE)
@@ -290,7 +426,8 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 		r.Name, r.Revision, r.LastUpdate,
 	))
 
-	archive, err := ioutil.TempDir(os.TempDir(), "archive")
+	// Create tree and repository file
+	archive, err := config.LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for archive")
 	}
@@ -299,26 +436,68 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 	if err != nil {
 		return errors.Wrap(err, "Error met while saving the tree")
 	}
-	tpath := r.GetTreePath()
-	if tpath == "" {
-		tpath = TREE_TARBALL
+
+	treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY)
+	if err != nil {
+		treeFile = NewDefaultTreeRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
 	}
 
-	a := compiler.NewPackageArtifact(filepath.Join(dst, tpath))
-	a.SetCompressionType(r.TreeCompressionType)
+	a := compiler.NewPackageArtifact(filepath.Join(dst, treeFile.GetFileName()))
+	a.SetCompressionType(treeFile.GetCompressionType())
 	err = a.Compress(archive, 1)
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating package archive")
 	}
 
-	r.TreePath = path.Base(a.GetPath())
+	// Update the tree name with the name created by compression selected.
+	treeFile.SetFileName(path.Base(a.GetPath()))
 	err = a.Hash()
 	if err != nil {
 		return errors.Wrap(err, "Failed generating checksums for tree")
 	}
-	r.TreeChecksums = a.GetChecksums()
+	treeFile.SetChecksums(a.GetChecksums())
+	r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
 
-	data, err := yaml.Marshal(r)
+	// Create Metadata struct and serialized repository
+	meta, serialized := r.Serialize()
+
+	// Create metadata file and repository file
+	metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	defer os.RemoveAll(metaTmpDir) // clean up
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+
+	metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY)
+	if err != nil {
+		metaFile = NewDefaultMetaRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	}
+
+	repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE)
+	// Create repository.meta.yaml file
+	err = meta.WriteFile(repoMetaSpec)
+	if err != nil {
+		return err
+	}
+
+	a = compiler.NewPackageArtifact(filepath.Join(dst, metaFile.GetFileName()))
+	a.SetCompressionType(metaFile.GetCompressionType())
+	err = a.Compress(metaTmpDir, 1)
+	if err != nil {
+		return errors.Wrap(err, "Error met while archiving repository metadata")
+	}
+
+	metaFile.SetFileName(path.Base(a.GetPath()))
+	r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for metadata")
+	}
+	metaFile.SetChecksums(a.GetChecksums())
+
+	data, err := yaml.Marshal(serialized)
 	if err != nil {
 		return err
 	}
@@ -346,7 +525,8 @@ func (r *LuetSystemRepository) Client() Client {
 }
 func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	var repoUpdated bool = false
-	var treefs string
+	var treefs, metafs string
+	aurora := GetAurora()
 
 	Debug("Sync of the repository", r.Name, "in progress...")
 	c := r.Client()
@@ -360,7 +540,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		return nil, errors.Wrap(err, "While downloading "+REPOSITORY_SPECFILE)
 	}
 
-	repobasedir := helpers.GetRepoDatabaseDirPath(r.GetName())
+	repobasedir := config.LuetCfg.GetSystem().GetRepoDatabaseDirPath(r.GetName())
 	repo, err := r.ReadSpecFile(file, false)
 	if err != nil {
 		return nil, err
@@ -384,37 +564,71 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		} else {
 			treefs = r.GetTreePath()
 		}
+		if r.GetMetaPath() == "" {
+			metafs = filepath.Join(repobasedir, "metafs")
+		} else {
+			metafs = r.GetMetaPath()
+		}
 
 	} else {
-		treefs, err = ioutil.TempDir(os.TempDir(), "treefs")
+		treefs, err = config.LuetCfg.GetSystem().TempDir("treefs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 		}
+		// Note: If we always remove them, later on, no other structure can access
+		// to the tree for e.g. to retrieve finalizers
+
+		metafs, err = config.LuetCfg.GetSystem().TempDir("metafs")
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met whilte creating tempdir for metafs")
+		}
+		//defer os.RemoveAll(metafs)
+
 	}
 
+	// POST: treeFile and metaFile are present. I check this inside
+	// ReadSpecFile and NewLuetSystemRepositoryFromYaml
+	treeFile, _ := repo.GetRepositoryFile(REPOFILE_TREE_KEY)
+	metaFile, _ := repo.GetRepositoryFile(REPOFILE_META_KEY)
+
 	if !repoUpdated {
-		tpath := repo.GetTreePath()
-		if tpath == "" {
-			tpath = TREE_TARBALL
-		}
-		a := compiler.NewPackageArtifact(tpath)
 
-		artifact, err := c.DownloadArtifact(a)
+		// Get Tree
+		a := compiler.NewPackageArtifact(treeFile.GetFileName())
+		artifactTree, err := c.DownloadArtifact(a)
 		if err != nil {
-			return nil, errors.Wrap(err, "While downloading "+tpath)
+			return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName())
 		}
-		defer os.Remove(artifact.GetPath())
+		defer os.Remove(artifactTree.GetPath())
 
-		artifact.SetChecksums(repo.GetTreeChecksums())
-		artifact.SetCompressionType(repo.GetTreeCompressionType())
+		artifactTree.SetChecksums(treeFile.GetChecksums())
+		artifactTree.SetCompressionType(treeFile.GetCompressionType())
 
-		err = artifact.Verify()
+		err = artifactTree.Verify()
 		if err != nil {
 			return nil, errors.Wrap(err, "Tree integrity check failure")
 		}
 
 		Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
 
+		// Get Repository Metadata
+		a = compiler.NewPackageArtifact(metaFile.GetFileName())
+		artifactMeta, err := c.DownloadArtifact(a)
+		if err != nil {
+			return nil, errors.Wrap(err, "While downloading "+metaFile.GetFileName())
+		}
+		defer os.Remove(artifactMeta.GetPath())
+
+		artifactMeta.SetChecksums(metaFile.GetChecksums())
+		artifactMeta.SetCompressionType(metaFile.GetCompressionType())
+
+		err = artifactMeta.Verify()
+		if err != nil {
+			return nil, errors.Wrap(err, "Metadata integrity check failure")
+		}
+
+		Debug("Metadata tarball for the repository " + r.GetName() + " downloaded correctly.")
+
 		if r.Cached {
 			// Copy updated repository.yaml file to repo dir now that the tree is synced.
 			err = helpers.CopyFile(file, filepath.Join(repobasedir, REPOSITORY_SPECFILE))
@@ -423,26 +637,45 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 			}
 			// Remove previous tree
 			os.RemoveAll(treefs)
+			// Remove previous meta dir
+			os.RemoveAll(metafs)
 		}
 		Debug("Decompress tree of the repository " + r.Name + "...")
 
-		err = artifact.Unpack(treefs, true)
+		err = artifactTree.Unpack(treefs, true)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while unpacking tree")
 		}
 
+		// FIXME: It seems that tar with only one file doesn't create destination
+		//       directory. I create directory directly for now.
+		os.MkdirAll(metafs, os.ModePerm)
+		err = artifactMeta.Unpack(metafs, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while unpacking metadata")
+		}
+
 		tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64)
 
 		InfoC(
-			Bold(Red(":house: Repository "+r.GetName()+" revision: ")).String() +
-				Bold(Green(repo.GetRevision())).String() + " - " +
-				Bold(Green(time.Unix(tsec, 0).String())).String(),
+			aurora.Bold(
+				aurora.Red(":house: Repository "+repo.GetName()+" revision: ")).String() +
+				aurora.Bold(aurora.Green(repo.GetRevision())).String() + " - " +
+				aurora.Bold(aurora.Green(time.Unix(tsec, 0).String())).String(),
 		)
 
 	} else {
-		Info("Repository", r.GetName(), "is already up to date.")
+		Info("Repository", repo.GetName(), "is already up to date.")
 	}
 
+	meta, err := NewLuetSystemRepositoryMetadata(
+		filepath.Join(metafs, REPOSITORY_METAFILE), false,
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "While processing "+REPOSITORY_METAFILE)
+	}
+	repo.SetIndex(meta.ToArtifactIndex())
+
 	reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
 	err = reciper.Load(treefs)
 	if err != nil {
@@ -451,21 +684,61 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 
 	repo.SetTree(reciper)
 	repo.SetTreePath(treefs)
+
+	// Copy the local available data to the one which was synced
+	// e.g. locally we can override the type (disk), or priority
+	// while remotely it could be advertized differently
 	repo.SetUrls(r.GetUrls())
 	repo.SetAuthentication(r.GetAuthentication())
-
+	repo.SetType(r.GetType())
+	repo.SetPriority(r.GetPriority())
+	repo.SetName(r.GetName())
+	InfoC(
+		aurora.Bold(
+			aurora.Yellow(":information_source: Repository "+repo.GetName()+" priority: ")).String() +
+			aurora.Bold(aurora.Green(repo.GetPriority())).String() + " - type " +
+			aurora.Bold(aurora.Green(repo.GetType())).String(),
+	)
 	return repo, nil
 }
 
+func (r *LuetSystemRepository) Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized) {
+
+	serialized := LuetSystemRepositorySerialized{
+		Name:            r.Name,
+		Description:     r.Description,
+		Urls:            r.Urls,
+		Priority:        r.Priority,
+		Type:            r.Type,
+		Revision:        r.Revision,
+		LastUpdate:      r.LastUpdate,
+		RepositoryFiles: r.RepositoryFiles,
+	}
+
+	// Check if is needed set the index or simply use
+	// value returned by CleanPath
+	r.Index = r.Index.CleanPath()
+
+	meta := &LuetSystemRepositoryMetadata{
+		Index: []*compiler.PackageArtifact{},
+	}
+	for _, a := range r.Index {
+		art := a.(*compiler.PackageArtifact)
+		meta.Index = append(meta.Index, art)
+	}
+
+	return meta, serialized
+}
+
 func (r Repositories) Len() int      { return len(r) }
 func (r Repositories) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
 func (r Repositories) Less(i, j int) bool {
 	return r[i].GetPriority() < r[j].GetPriority()
 }
 
-func (r Repositories) World() []pkg.Package {
+func (r Repositories) World() pkg.Packages {
 	cache := map[string]pkg.Package{}
-	world := []pkg.Package{}
+	world := pkg.Packages{}
 
 	// Get Uniques. Walk in reverse so the definitions of most prio-repo overwrites lower ones
 	// In this way, when we will walk again later the deps sorting them by most higher prio we have better chance of success.
@@ -503,7 +776,7 @@ type PackageMatch struct {
 	Package pkg.Package
 }
 
-func (re Repositories) PackageMatches(p []pkg.Package) []PackageMatch {
+func (re Repositories) PackageMatches(p pkg.Packages) []PackageMatch {
 	// TODO: Better heuristic. here we pick the first repo that contains the atom, sorted by priority but
 	// we should do a permutations and get the best match, and in case there are more solutions the user should be able to pick
 	sort.Sort(re)
@@ -524,20 +797,27 @@ PACKAGE:
 
 }
 
-func (re Repositories) ResolveSelectors(p []pkg.Package) []pkg.Package {
+func (re Repositories) ResolveSelectors(p pkg.Packages) pkg.Packages {
 	// If a selector is given, get the best from each repo
 	sort.Sort(re) // respect prio
-	var matches []pkg.Package
+	var matches pkg.Packages
 PACKAGE:
 	for _, pack := range p {
+	REPOSITORY:
 		for _, r := range re {
 			if pack.IsSelector() {
 				c, err := r.GetTree().GetDatabase().FindPackageCandidate(pack)
+				// If FindPackageCandidate returns the same package, it means it couldn't find one.
+				// Skip this repository and keep looking.
+				if c.String() == pack.String() {
+					continue REPOSITORY
+				}
 				if err == nil {
 					matches = append(matches, c)
 					continue PACKAGE
 				}
 			} else {
+				// If it's not a selector, just append it
 				matches = append(matches, pack)
 			}
 		}
@@ -547,14 +827,25 @@ PACKAGE:
 
 }
 
-func (re Repositories) Search(s string) []PackageMatch {
+func (re Repositories) SearchPackages(p string, o LuetSearchOpts) []PackageMatch {
 	sort.Sort(re)
-	var term = regexp.MustCompile(s)
 	var matches []PackageMatch
+	var err error
 
 	for _, r := range re {
-		for _, pack := range r.GetTree().GetDatabase().World() {
-			if term.MatchString(pack.GetName()) {
+		var repoMatches pkg.Packages
+
+		switch o.Mode {
+		case SRegexPkg:
+			repoMatches, err = r.GetTree().GetDatabase().FindPackageMatch(p)
+		case SLabel:
+			repoMatches, err = r.GetTree().GetDatabase().FindPackageLabel(p)
+		case SRegexLabel:
+			repoMatches, err = r.GetTree().GetDatabase().FindPackageLabelMatch(p)
+		}
+
+		if err == nil && len(repoMatches) > 0 {
+			for _, pack := range repoMatches {
 				matches = append(matches, PackageMatch{Package: pack, Repo: r})
 			}
 		}
@@ -562,3 +853,15 @@ func (re Repositories) Search(s string) []PackageMatch {
 
 	return matches
 }
+
+func (re Repositories) SearchLabelMatch(s string) []PackageMatch {
+	return re.SearchPackages(s, LuetSearchOpts{Pattern: s, Mode: SRegexLabel})
+}
+
+func (re Repositories) SearchLabel(s string) []PackageMatch {
+	return re.SearchPackages(s, LuetSearchOpts{Pattern: s, Mode: SLabel})
+}
+
+func (re Repositories) Search(s string) []PackageMatch {
+	return re.SearchPackages(s, LuetSearchOpts{Pattern: s, Mode: SRegexPkg})
+}

pkg/installer/repository_test.go

@@ -18,6 +18,7 @@ package installer_test
 import (
 
 	//	. "github.com/mudler/luet/pkg/installer"
+
 	"io/ioutil"
 	"os"
 
@@ -27,6 +28,7 @@ import (
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -34,7 +36,7 @@ import (
 
 var _ = Describe("Repository", func() {
 	Context("Generation", func() {
-		It("Generate repository metadat", func() {
+		It("Generate repository metadata", func() {
 
 			tmpdir, err := ioutil.TempDir("", "tree")
 			Expect(err).ToNot(HaveOccurred())
@@ -47,7 +49,7 @@ var _ = Describe("Repository", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -82,16 +84,117 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+		})
+
+		It("Generate repository metadata of files ONLY referenced in a tree", func() {
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			generalRecipe2 := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe2.Load("../../tests/fixtures/finalizers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+			Expect(spec2.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			compiler2.SetConcurrency(1)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			artifact2, err := compiler2.Compile(false, spec2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact2.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact2.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			content1, err := helpers.Read(spec.Rel("test5"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := helpers.Read(spec.Rel("test6"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("artifact5\n"))
+			Expect(content2).To(Equal("artifact6\n"))
+
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+
+			// We check now that the artifact not referenced in the tree
+			// (spec2) is not indexed in the repository
+			repository, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			repos, err := repository.Sync(true)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = repos.GetTree().GetDatabase().FindPackage(spec.GetPackage())
+			Expect(err).ToNot(HaveOccurred())
+			_, err = repos.GetTree().GetDatabase().FindPackage(spec2.GetPackage())
+			Expect(err).To(HaveOccurred()) // should throw error
 		})
 	})
 	Context("Matching packages", func() {

pkg/installer/system.go

@@ -9,6 +9,6 @@ type System struct {
 	Target   string
 }
 
-func (s *System) World() ([]pkg.Package, error) {
+func (s *System) World() (pkg.Packages, error) {
 	return s.Database.World(), nil
 }

pkg/logger/logger.go

@@ -3,6 +3,7 @@ package logger
 import (
 	"fmt"
 	"os"
+	"regexp"
 
 	. "github.com/mudler/luet/pkg/config"
 
@@ -15,6 +16,7 @@ import (
 
 var s *spinner.Spinner = nil
 var z *zap.Logger = nil
+var aurora Aurora = nil
 
 func NewSpinner() {
 	if s == nil {
@@ -24,6 +26,16 @@ func NewSpinner() {
 	}
 }
 
+func InitAurora() {
+	if aurora == nil {
+		aurora = NewAurora(LuetCfg.GetLogging().Color)
+	}
+}
+
+func GetAurora() Aurora {
+	return aurora
+}
+
 func ZapLogger() error {
 	var err error
 	if z == nil {
@@ -53,33 +65,52 @@ func ZapLogger() error {
 }
 
 func Spinner(i int) {
-
+	var confLevel int
+	if LuetCfg.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = level2Number(LuetCfg.GetLogging().Level)
+	}
+	if 2 > confLevel {
+		return
+	}
 	if i > 43 {
 		i = 43
 	}
 
-	if !LuetCfg.GetGeneral().Debug && !s.Active() {
+	if s != nil && !s.Active() {
 		//	s.UpdateCharSet(spinner.CharSets[i])
 		s.Start() // Start the spinner
 	}
 }
 
 func SpinnerText(suffix, prefix string) {
-	s.Lock()
-	defer s.Unlock()
-	if LuetCfg.GetGeneral().Debug {
-		fmt.Println(fmt.Sprintf("%s %s",
-			Bold(Cyan(prefix)).String(),
-			Bold(Magenta(suffix)).BgBlack().String(),
-		))
-	} else {
-		s.Suffix = Bold(Magenta(suffix)).BgBlack().String()
-		s.Prefix = Bold(Cyan(prefix)).String()
+	if s != nil {
+		s.Lock()
+		defer s.Unlock()
+		if LuetCfg.GetGeneral().Debug {
+			fmt.Println(fmt.Sprintf("%s %s",
+				Bold(Cyan(prefix)).String(),
+				Bold(Magenta(suffix)).BgBlack().String(),
+			))
+		} else {
+			s.Suffix = Bold(Magenta(suffix)).BgBlack().String()
+			s.Prefix = Bold(Cyan(prefix)).String()
+		}
 	}
 }
 
 func SpinnerStop() {
-	if !LuetCfg.GetGeneral().Debug {
+	var confLevel int
+	if LuetCfg.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = level2Number(LuetCfg.GetLogging().Level)
+	}
+	if 2 > confLevel {
+		return
+	}
+	if s != nil {
 		s.Stop()
 	}
 }
@@ -143,7 +174,7 @@ func msg(level string, withoutColor bool, msg ...interface{}) {
 
 	var levelMsg string
 
-	if withoutColor {
+	if withoutColor || !LuetCfg.GetLogging().Color {
 		levelMsg = message
 	} else {
 		switch level {
@@ -158,7 +189,12 @@ func msg(level string, withoutColor bool, msg ...interface{}) {
 		}
 	}
 
-	levelMsg = emoji.Sprint(levelMsg)
+	if LuetCfg.GetLogging().EnableEmoji {
+		levelMsg = emoji.Sprint(levelMsg)
+	} else {
+		re := regexp.MustCompile(`[:][\w]+[:]`)
+		levelMsg = re.ReplaceAllString(levelMsg, "")
+	}
 
 	if z != nil {
 		log2File(level, message)

pkg/package/annotations.go

@@ -0,0 +1,23 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+type AnnotationKey string
+
+const (
+	ConfigProtectAnnnotation AnnotationKey = "config_protect"
+)

pkg/package/database.go

@@ -33,7 +33,7 @@ type PackageSet interface {
 	GetPackage(ID string) (Package, error)
 	Clean() error
 	FindPackage(Package) (Package, error)
-	FindPackages(p Package) ([]Package, error)
+	FindPackages(p Package) (Packages, error)
 	UpdatePackage(p Package) error
 	GetAllPackages(packages chan Package) error
 	RemovePackage(Package) error
@@ -41,10 +41,13 @@ type PackageSet interface {
 	GetPackageFiles(Package) ([]string, error)
 	SetPackageFiles(*PackageFile) error
 	RemovePackageFiles(Package) error
-	FindPackageVersions(p Package) ([]Package, error)
-	World() []Package
+	FindPackageVersions(p Package) (Packages, error)
+	World() Packages
 
 	FindPackageCandidate(p Package) (Package, error)
+	FindPackageLabel(labelKey string) (Packages, error)
+	FindPackageLabelMatch(pattern string) (Packages, error)
+	FindPackageMatch(pattern string) (Packages, error)
 }
 
 type PackageFile struct {

pkg/package/database_bench_test.go

@@ -0,0 +1,116 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	"io/ioutil"
+	"os"
+	"strconv"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Database  benchmark", func() {
+
+	Context("BoltDB", func() {
+
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		var db PackageSet
+
+		BeforeEach(func() {
+
+			tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			db = NewBoltDatabase(tmpfile.Name())
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+
+		Measure("it should be fast in computing world from a 50000 dataset", func(b Benchmarker) {
+			for i := 0; i < 50000; i++ {
+				a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			runtime := b.Time("runtime", func() {
+				packs := db.World()
+				Expect(len(packs)).To(Equal(50000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 30), "World() shouldn't take too long.")
+
+		}, 1)
+
+		Measure("it should be fast in computing world from a 100000 dataset", func(b Benchmarker) {
+			for i := 0; i < 100000; i++ {
+				a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			runtime := b.Time("runtime", func() {
+				packs := db.World()
+				Expect(len(packs)).To(Equal(100000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 30), "World() shouldn't take too long.")
+
+		}, 1)
+	})
+
+	Context("InMemory", func() {
+
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		var db PackageSet
+
+		BeforeEach(func() {
+
+			tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			db = NewInMemoryDatabase(false)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+
+		Measure("it should be fast in computing world from a 100000 dataset", func(b Benchmarker) {
+
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 100000; i++ {
+					a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					_, err := db.CreatePackage(a)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				packs := db.World()
+				Expect(len(packs)).To(Equal(100000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 10), "World() shouldn't take too long.")
+
+		}, 2)
+	})
+})

pkg/package/database_boltdb.go

@@ -17,7 +17,9 @@ package pkg
 
 import (
 	"encoding/base64"
+	"fmt"
 	"os"
+	"regexp"
 	"strconv"
 	"sync"
 	"time"
@@ -160,26 +162,17 @@ func (db *BoltDatabase) GetAllPackages(packages chan Package) error {
 		return err
 	}
 	defer bolt.Close()
-	// Fetching records one by one (useful when the bucket contains a lot of records)
-	//query := bolt.Select()
-
-	var packs []Package
+	var packs []DefaultPackage
 	err = bolt.All(&packs)
 	if err != nil {
 		return err
 	}
 
 	for _, r := range packs {
-		packages <- r
+		packages <- &r
 	}
 
 	return nil
-
-	// return query.Each(new(DefaultPackage), func(record interface{}) error {
-	// 	u := record.(*DefaultPackage)
-	// 	packages <- u
-	// 	return err
-	// })
 }
 
 // Encode encodes the package to string.
@@ -227,19 +220,19 @@ func (db *BoltDatabase) getProvide(p Package) (Package, error) {
 		db.Unlock()
 
 		if !ok {
-			return nil, errors.New("No versions found for package")
+			return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 		}
 
 		for ve, _ := range versions {
 
-			match, err := p.VersionMatchSelector(ve)
+			match, err := p.VersionMatchSelector(ve, nil)
 			if err != nil {
 				return nil, errors.Wrap(err, "Error on match version")
 			}
 			if match {
 				pa, ok := db.ProvidesDatabase[p.GetPackageName()][ve]
 				if !ok {
-					return nil, errors.New("No versions found for package")
+					return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 				}
 				return pa, nil //pick the first (we shouldn't have providers that are conflicting)
 				// TODO: A find dbcall here would recurse, but would give chance to have providers of providers
@@ -308,22 +301,29 @@ func (db *BoltDatabase) RemovePackage(p Package) error {
 	var found DefaultPackage
 	err = bolt.Select(q.Eq("Name", p.GetName()), q.Eq("Category", p.GetCategory()), q.Eq("Version", p.GetVersion())).Limit(1).Delete(&found)
 	if err != nil {
-		return errors.Wrap(err, "No package found to delete")
+		return errors.New(fmt.Sprintf("Package not found: %s", p.HumanReadableString()))
 	}
 	return nil
 }
 
-func (db *BoltDatabase) World() []Package {
+func (db *BoltDatabase) World() Packages {
+	var packs []DefaultPackage
 
-	var all []Package
-	// FIXME: This should all be locked in the db - for now forbid the solver to be run in threads.
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err == nil {
-			all = append(all, pack)
-		}
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return Packages([]Package{})
 	}
-	return all
+	defer bolt.Close()
+	err = bolt.All(&packs)
+	if err != nil {
+		return Packages([]Package{})
+	}
+	models := make([]Package, len(packs))
+	for i, _ := range packs {
+		models[i] = &packs[i]
+	}
+
+	return Packages(models)
 }
 
 func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
@@ -337,7 +337,7 @@ func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
 		if err != nil || len(packages) == 0 {
 			required = p
 		} else {
-			required = Best(packages)
+			required = packages.Best(nil)
 
 		}
 		return required, nil
@@ -350,7 +350,7 @@ func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
 
 // FindPackages return the list of the packages beloging to cat/name  (any versions in requested range)
 // FIXME: Optimize, see inmemorydb
-func (db *BoltDatabase) FindPackages(p Package) ([]Package, error) {
+func (db *BoltDatabase) FindPackages(p Package) (Packages, error) {
 	// Provides: Treat as the replaced package here
 	if provided, err := db.getProvide(p); err == nil {
 		p = provided
@@ -361,7 +361,7 @@ func (db *BoltDatabase) FindPackages(p Package) ([]Package, error) {
 			continue
 		}
 
-		match, err := p.SelectorMatchVersion(w.GetVersion())
+		match, err := p.SelectorMatchVersion(w.GetVersion(), nil)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error on match selector")
 		}
@@ -369,11 +369,11 @@ func (db *BoltDatabase) FindPackages(p Package) ([]Package, error) {
 			versionsInWorld = append(versionsInWorld, w)
 		}
 	}
-	return versionsInWorld, nil
+	return Packages(versionsInWorld), nil
 }
 
 // FindPackageVersions return the list of the packages beloging to cat/name
-func (db *BoltDatabase) FindPackageVersions(p Package) ([]Package, error) {
+func (db *BoltDatabase) FindPackageVersions(p Package) (Packages, error) {
 	var versionsInWorld []Package
 	for _, w := range db.World() {
 		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
@@ -382,5 +382,50 @@ func (db *BoltDatabase) FindPackageVersions(p Package) ([]Package, error) {
 
 		versionsInWorld = append(versionsInWorld, w)
 	}
-	return versionsInWorld, nil
+	return Packages(versionsInWorld), nil
+}
+
+func (db *BoltDatabase) FindPackageLabel(labelKey string) (Packages, error) {
+	var ans []Package
+
+	for _, pack := range db.World() {
+		if pack.HasLabel(labelKey) {
+			ans = append(ans, pack)
+		}
+	}
+	return Packages(ans), nil
+}
+
+func (db *BoltDatabase) FindPackageLabelMatch(pattern string) (Packages, error) {
+	var ans []Package
+
+	re := regexp.MustCompile(pattern)
+	if re == nil {
+		return nil, errors.New("Invalid regex " + pattern + "!")
+	}
+
+	for _, pack := range db.World() {
+		if pack.MatchLabel(re) {
+			ans = append(ans, pack)
+		}
+	}
+
+	return Packages(ans), nil
+}
+
+func (db *BoltDatabase) FindPackageMatch(pattern string) (Packages, error) {
+	var ans []Package
+
+	re := regexp.MustCompile(pattern)
+	if re == nil {
+		return nil, errors.New("Invalid regex " + pattern + "!")
+	}
+
+	for _, pack := range db.World() {
+		if re.MatchString(pack.HumanReadableString()) {
+			ans = append(ans, pack)
+		}
+	}
+
+	return Packages(ans), nil
 }

pkg/package/database_boltdb_test.go

@@ -0,0 +1,152 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or ItNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	"io/ioutil"
+	"os"
+	"regexp"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("BoltDB Database", func() {
+
+	tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+	defer os.Remove(tmpfile.Name()) // clean up
+	var db PackageDatabase
+
+	BeforeEach(func() {
+		tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		db = NewBoltDatabase(tmpfile.Name())
+	})
+	Context("Simple package", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		It("Find packages", func() {
+			ID, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.GetPackage(ID)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack).To(Equal(a))
+			ids := db.GetPackages()
+
+			Expect(ids).To(Equal([]string{"1"}))
+
+			pack, err = db.FindPackage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Expands correctly", func() {
+
+			a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+			re := regexp.MustCompile("project[0-9][=].*")
+			for _, p := range []Package{a1, a11, a01} {
+				_, err := db.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			lst, err := a.Expand(db)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(lst).To(ContainElement(a11))
+			Expect(lst).To(ContainElement(a1))
+			Expect(lst).ToNot(ContainElement(a01))
+			Expect(len(lst)).To(Equal(2))
+			p := lst.Best(nil)
+			Expect(p).To(Equal(a11))
+			// Test annotation with null map
+			Expect(a.MatchAnnotation(re)).To(Equal(false))
+		})
+
+		It("Find best package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+
+		})
+
+		It("Find specific package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Provides replaces definitions", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+			a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+			Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+
+			s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackage(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+		})
+
+	})
+
+})

pkg/package/database_mem.go

@@ -18,6 +18,8 @@ package pkg
 import (
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
+	"regexp"
 	"sync"
 
 	"github.com/pkg/errors"
@@ -58,7 +60,7 @@ func (db *InMemoryDatabase) Get(s string) (string, error) {
 	defer db.Unlock()
 	pa, ok := db.Database[s]
 	if !ok {
-		return "", errors.New("No key found with that id")
+		return "", errors.New(fmt.Sprintf("No key found for: %s", s))
 	}
 	return pa, nil
 }
@@ -177,7 +179,7 @@ func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
 
 		for ve, _ := range versions {
 
-			match, err := p.VersionMatchSelector(ve)
+			match, err := p.VersionMatchSelector(ve, nil)
 			if err != nil {
 				return nil, errors.Wrap(err, "Error on match version")
 			}
@@ -222,7 +224,7 @@ func (db *InMemoryDatabase) FindPackage(p Package) (Package, error) {
 }
 
 // FindPackages return the list of the packages beloging to cat/name
-func (db *InMemoryDatabase) FindPackageVersions(p Package) ([]Package, error) {
+func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) {
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
 		return nil, errors.New("No versions found for package")
@@ -235,11 +237,11 @@ func (db *InMemoryDatabase) FindPackageVersions(p Package) ([]Package, error) {
 		}
 		versionsInWorld = append(versionsInWorld, w)
 	}
-	return versionsInWorld, nil
+	return Packages(versionsInWorld), nil
 }
 
 // FindPackages return the list of the packages beloging to cat/name (any versions in requested range)
-func (db *InMemoryDatabase) FindPackages(p Package) ([]Package, error) {
+func (db *InMemoryDatabase) FindPackages(p Package) (Packages, error) {
 
 	// Provides: Treat as the replaced package here
 	if provided, err := db.getProvide(p); err == nil {
@@ -247,11 +249,11 @@ func (db *InMemoryDatabase) FindPackages(p Package) ([]Package, error) {
 	}
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
-		return nil, errors.New("No versions found for package")
+		return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 	}
 	var versionsInWorld []Package
 	for ve, _ := range versions {
-		match, err := p.SelectorMatchVersion(ve)
+		match, err := p.SelectorMatchVersion(ve, nil)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error on match selector")
 		}
@@ -264,7 +266,7 @@ func (db *InMemoryDatabase) FindPackages(p Package) ([]Package, error) {
 			versionsInWorld = append(versionsInWorld, w)
 		}
 	}
-	return versionsInWorld, nil
+	return Packages(versionsInWorld), nil
 }
 
 func (db *InMemoryDatabase) UpdatePackage(p Package) error {
@@ -276,7 +278,7 @@ func (db *InMemoryDatabase) UpdatePackage(p Package) error {
 
 	return db.Set(p.GetFingerPrint(), enc)
 
-	return errors.New("Package not found")
+	return errors.New(fmt.Sprintf("Package not found: %s", p.HumanReadableString()))
 }
 
 func (db *InMemoryDatabase) GetPackages() []string {
@@ -301,7 +303,7 @@ func (db *InMemoryDatabase) GetPackageFiles(p Package) ([]string, error) {
 
 	pa, ok := db.FileDatabase[p.GetFingerPrint()]
 	if !ok {
-		return pa, errors.New("No key found with that id")
+		return pa, errors.New(fmt.Sprintf("No key found for: %s", p.HumanReadableString()))
 	}
 
 	return pa, nil
@@ -326,7 +328,7 @@ func (db *InMemoryDatabase) RemovePackage(p Package) error {
 	delete(db.Database, p.GetFingerPrint())
 	return nil
 }
-func (db *InMemoryDatabase) World() []Package {
+func (db *InMemoryDatabase) World() Packages {
 	var all []Package
 	// FIXME: This should all be locked in the db - for now forbid the solver to be run in threads.
 	for _, k := range db.GetPackages() {
@@ -335,7 +337,7 @@ func (db *InMemoryDatabase) World() []Package {
 			all = append(all, pack)
 		}
 	}
-	return all
+	return Packages(all)
 }
 
 func (db *InMemoryDatabase) FindPackageCandidate(p Package) (Package, error) {
@@ -348,7 +350,7 @@ func (db *InMemoryDatabase) FindPackageCandidate(p Package) (Package, error) {
 		if err != nil || len(packages) == 0 {
 			required = p
 		} else {
-			required = Best(packages)
+			required = packages.Best(nil)
 
 		}
 		return required, nil
@@ -358,3 +360,62 @@ func (db *InMemoryDatabase) FindPackageCandidate(p Package) (Package, error) {
 	return required, err
 
 }
+
+func (db *InMemoryDatabase) FindPackageLabel(labelKey string) (Packages, error) {
+	var ans []Package
+
+	for _, k := range db.GetPackages() {
+		pack, err := db.GetPackage(k)
+		if err != nil {
+			return ans, err
+		}
+		if pack.HasLabel(labelKey) {
+			ans = append(ans, pack)
+		}
+	}
+
+	return Packages(ans), nil
+}
+
+func (db *InMemoryDatabase) FindPackageLabelMatch(pattern string) (Packages, error) {
+	var ans []Package
+
+	re := regexp.MustCompile(pattern)
+	if re == nil {
+		return nil, errors.New("Invalid regex " + pattern + "!")
+	}
+
+	for _, k := range db.GetPackages() {
+		pack, err := db.GetPackage(k)
+		if err != nil {
+			return ans, err
+		}
+		if pack.MatchLabel(re) {
+			ans = append(ans, pack)
+		}
+	}
+
+	return Packages(ans), nil
+}
+
+func (db *InMemoryDatabase) FindPackageMatch(pattern string) (Packages, error) {
+	var ans []Package
+
+	re := regexp.MustCompile(pattern)
+	if re == nil {
+		return nil, errors.New("Invalid regex " + pattern + "!")
+	}
+
+	for _, k := range db.GetPackages() {
+		pack, err := db.GetPackage(k)
+		if err != nil {
+			return ans, err
+		}
+
+		if re.MatchString(pack.HumanReadableString()) {
+			ans = append(ans, pack)
+		}
+	}
+
+	return Packages(ans), nil
+}

pkg/package/package.go

@@ -17,19 +17,23 @@ package pkg
 
 import (
 	"bytes"
+	"crypto/md5"
 	"encoding/json"
 	"fmt"
+	"io"
 	"path/filepath"
-	"sort"
+	"regexp"
+	"strconv"
 	"strings"
 
-	//	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/pkg/helpers"
+	version "github.com/mudler/luet/pkg/versioner"
 
+	gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/crillab/gophersat/bf"
-	version "github.com/hashicorp/go-version"
-	"github.com/jinzhu/copier"
-
 	"github.com/ghodss/yaml"
+	"github.com/jinzhu/copier"
+	"github.com/pkg/errors"
 )
 
 // Package is a package interface (TBD)
@@ -38,29 +42,32 @@ type Package interface {
 	Encode(PackageDatabase) (string, error)
 
 	BuildFormula(PackageDatabase, PackageDatabase) ([]bf.Formula, error)
-	IsFlagged(bool) Package
-	Flagged() bool
+
 	GetFingerPrint() string
 	GetPackageName() string
 	Requires([]*DefaultPackage) Package
 	Conflicts([]*DefaultPackage) Package
-	Revdeps(PackageDatabase) []Package
+	Revdeps(PackageDatabase) Packages
+	ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages
+	LabelDeps(PackageDatabase, string) Packages
 
 	GetProvides() []*DefaultPackage
 	SetProvides([]*DefaultPackage) Package
 
 	GetRequires() []*DefaultPackage
 	GetConflicts() []*DefaultPackage
-	Expand(PackageDatabase) ([]Package, error)
+	Expand(PackageDatabase) (Packages, error)
 	SetCategory(string)
 
 	GetName() string
+	SetName(string)
 	GetCategory() string
 
 	GetVersion() string
+	SetVersion(string)
 	RequiresContains(PackageDatabase, Package) (bool, error)
 	Matches(m Package) bool
-	Bigger(m Package) bool
+	BumpBuildVersion() error
 
 	AddUse(use string)
 	RemoveUse(use string)
@@ -82,19 +89,41 @@ type Package interface {
 	SetLicense(string)
 	GetLicense() string
 
+	AddLabel(string, string)
+	GetLabels() map[string]string
+	HasLabel(string) bool
+	MatchLabel(*regexp.Regexp) bool
+
+	AddAnnotation(string, string)
+	GetAnnotations() map[string]string
+	HasAnnotation(string) bool
+	MatchAnnotation(*regexp.Regexp) bool
+
+	IsHidden() bool
 	IsSelector() bool
-	VersionMatchSelector(string) (bool, error)
-	SelectorMatchVersion(string) (bool, error)
+	VersionMatchSelector(string, version.Versioner) (bool, error)
+	SelectorMatchVersion(string, version.Versioner) (bool, error)
+
+	String() string
+	HumanReadableString() string
+	HashFingerprint(string) string
+
+	SetBuildTimestamp(s string)
+	GetBuildTimestamp() string
+
+	Clone() Package
 }
 
 type Tree interface {
 	GetPackageSet() PackageDatabase
 	Prelude() string // A tree might have a prelude to be able to consume a tree
 	SetPackageSet(s PackageDatabase)
-	World() ([]Package, error)
+	World() (Packages, error)
 	FindPackage(Package) (Package, error)
 }
 
+type Packages []Package
+
 // >> Unmarshallers
 // DefaultPackageFromYaml decodes a package from yaml bytes
 func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) {
@@ -128,25 +157,29 @@ func (t *DefaultPackage) JSON() ([]byte, error) {
 
 // DefaultPackage represent a standard package definition
 type DefaultPackage struct {
-	ID               int      `json:"-" storm:"id,increment"` // primary key with auto increment
-	Name             string   `json:"name"`                   // Affects YAML field names too.
-	Version          string   `json:"version"`                // Affects YAML field names too.
-	Category         string   `json:"category"`               // Affects YAML field names too.
-	UseFlags         []string `json:"use_flags"`              // Affects YAML field names too.
-	State            State
-	PackageRequires  []*DefaultPackage `json:"requires"`  // Affects YAML field names too.
-	PackageConflicts []*DefaultPackage `json:"conflicts"` // Affects YAML field names too.
-	IsSet            bool              `json:"set"`       // Affects YAML field names too.
-	Provides         []*DefaultPackage `json:"provides"`  // Affects YAML field names too.
+	ID               int               `storm:"id,increment" json:"id"` // primary key with auto increment
+	Name             string            `json:"name"`                    // Affects YAML field names too.
+	Version          string            `json:"version"`                 // Affects YAML field names too.
+	Category         string            `json:"category"`                // Affects YAML field names too.
+	UseFlags         []string          `json:"use_flags,omitempty"`     // Affects YAML field names too.
+	State            State             `json:"state,omitempty"`
+	PackageRequires  []*DefaultPackage `json:"requires"`           // Affects YAML field names too.
+	PackageConflicts []*DefaultPackage `json:"conflicts"`          // Affects YAML field names too.
+	Provides         []*DefaultPackage `json:"provides,omitempty"` // Affects YAML field names too.
+	Hidden           bool              `json:"hidden,omitempty"`   // Affects YAML field names too.
 
-	// TODO: Annotations?
+	// Annotations are used for core features/options
+	Annotations map[string]string `json:"annotations,omitempty"` // Affects YAML field names too
 
 	// Path is set only internally when tree is loaded from disk
 	Path string `json:"path,omitempty"`
 
-	Description string   `json:"description"`
-	Uri         []string `json:"uri"`
-	License     string   `json:"license"`
+	Description    string   `json:"description,omitempty"`
+	Uri            []string `json:"uri,omitempty"`
+	License        string   `json:"license,omitempty"`
+	BuildTimestamp string   `json:"buildtimestamp,omitempty"`
+
+	Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
 }
 
 // State represent the package state
@@ -154,13 +187,19 @@ type State string
 
 // NewPackage returns a new package
 func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*DefaultPackage) *DefaultPackage {
-	return &DefaultPackage{Name: name, Version: version, PackageRequires: requires, PackageConflicts: conflicts}
+	return &DefaultPackage{
+		Name:             name,
+		Version:          version,
+		PackageRequires:  requires,
+		PackageConflicts: conflicts,
+		Labels:           nil,
+	}
 }
 
 func (p *DefaultPackage) String() string {
 	b, err := p.JSON()
 	if err != nil {
-		return fmt.Sprintf("{ id: \"%d\", name: \"%s\" }", p.ID, p.Name)
+		return fmt.Sprintf("{ id: \"%d\", name: \"%s\", version: \"%s\", category: \"%s\"  }", p.ID, p.Name, p.Version, p.Category)
 	}
 	return fmt.Sprintf("%s", string(b))
 }
@@ -171,10 +210,40 @@ func (p *DefaultPackage) GetFingerPrint() string {
 	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version)
 }
 
+func (p *DefaultPackage) HashFingerprint(salt string) string {
+	h := md5.New()
+	io.WriteString(h, fmt.Sprintf("%s-%s", p.GetFingerPrint(), salt))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func (p *DefaultPackage) HumanReadableString() string {
+	return fmt.Sprintf("%s/%s-%s", p.Category, p.Name, p.Version)
+}
+
+func FromString(s string) Package {
+	var unescaped DefaultPackage
+
+	err := json.Unmarshal([]byte(s), &unescaped)
+	if err != nil {
+		return &unescaped
+	}
+	return &unescaped
+}
+
 func (p *DefaultPackage) GetPackageName() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.Category)
 }
 
+// GetBuildTimestamp returns the package build timestamp
+func (p *DefaultPackage) GetBuildTimestamp() string {
+	return p.BuildTimestamp
+}
+
+// SetBuildTimestamp sets the package Build timestamp
+func (p *DefaultPackage) SetBuildTimestamp(s string) {
+	p.BuildTimestamp = s
+}
+
 // GetPath returns the path where the definition file was found
 func (p *DefaultPackage) GetPath() string {
 	return p.Path
@@ -192,6 +261,26 @@ func (p *DefaultPackage) IsSelector() bool {
 	return strings.ContainsAny(p.GetVersion(), "<>=")
 }
 
+func (p *DefaultPackage) IsHidden() bool {
+	return p.Hidden
+}
+
+func (p *DefaultPackage) HasLabel(label string) bool {
+	return helpers.MapHasKey(&p.Labels, label)
+}
+
+func (p *DefaultPackage) MatchLabel(r *regexp.Regexp) bool {
+	return helpers.MapMatchRegex(&p.Labels, r)
+}
+
+func (p *DefaultPackage) HasAnnotation(label string) bool {
+	return helpers.MapHasKey(&p.Annotations, label)
+}
+
+func (p *DefaultPackage) MatchAnnotation(r *regexp.Regexp) bool {
+	return helpers.MapMatchRegex(&p.Annotations, r)
+}
+
 // AddUse adds a use to a package
 func (p *DefaultPackage) AddUse(use string) {
 	for _, v := range p.UseFlags {
@@ -222,7 +311,6 @@ func (p *DefaultPackage) Encode(db PackageDatabase) (string, error) {
 func (p *DefaultPackage) Yaml() ([]byte, error) {
 	j, err := p.JSON()
 	if err != nil {
-
 		return []byte{}, err
 	}
 	y, err := yaml.JSONToYAML(j)
@@ -233,15 +321,6 @@ func (p *DefaultPackage) Yaml() ([]byte, error) {
 	return y, nil
 }
 
-func (p *DefaultPackage) IsFlagged(b bool) Package {
-	p.IsSet = b
-	return p
-}
-
-func (p *DefaultPackage) Flagged() bool {
-	return p.IsSet
-}
-
 func (p *DefaultPackage) GetName() string {
 	return p.Name
 }
@@ -249,6 +328,9 @@ func (p *DefaultPackage) GetName() string {
 func (p *DefaultPackage) GetVersion() string {
 	return p.Version
 }
+func (p *DefaultPackage) SetVersion(v string) {
+	p.Version = v
+}
 func (p *DefaultPackage) GetDescription() string {
 	return p.Description
 }
@@ -273,9 +355,32 @@ func (p *DefaultPackage) GetCategory() string {
 func (p *DefaultPackage) SetCategory(s string) {
 	p.Category = s
 }
+
+func (p *DefaultPackage) SetName(s string) {
+	p.Name = s
+}
+
 func (p *DefaultPackage) GetUses() []string {
 	return p.UseFlags
 }
+func (p *DefaultPackage) AddLabel(k, v string) {
+	if p.Labels == nil {
+		p.Labels = make(map[string]string, 0)
+	}
+	p.Labels[k] = v
+}
+func (p *DefaultPackage) AddAnnotation(k, v string) {
+	if p.Annotations == nil {
+		p.Annotations = make(map[string]string, 0)
+	}
+	p.Annotations[k] = v
+}
+func (p *DefaultPackage) GetLabels() map[string]string {
+	return p.Labels
+}
+func (p *DefaultPackage) GetAnnotations() map[string]string {
+	return p.Annotations
+}
 func (p *DefaultPackage) GetProvides() []*DefaultPackage {
 	return p.Provides
 }
@@ -308,23 +413,16 @@ func (p *DefaultPackage) Matches(m Package) bool {
 	}
 	return false
 }
-func (p *DefaultPackage) Bigger(m Package) bool {
-	low := Lower([]Package{p, m})
-	if low.Matches(m) {
-		return true
-	}
-	return false
-}
 
-func (p *DefaultPackage) Expand(definitiondb PackageDatabase) ([]Package, error) {
-	var versionsInWorld []Package
+func (p *DefaultPackage) Expand(definitiondb PackageDatabase) (Packages, error) {
+	var versionsInWorld Packages
 
 	all, err := definitiondb.FindPackages(p)
 	if err != nil {
 		return nil, err
 	}
 	for _, w := range all {
-		match, err := p.SelectorMatchVersion(w.GetVersion())
+		match, err := p.SelectorMatchVersion(w.GetVersion(), nil)
 		if err != nil {
 			return nil, err
 		}
@@ -336,8 +434,8 @@ func (p *DefaultPackage) Expand(definitiondb PackageDatabase) ([]Package, error)
 	return versionsInWorld, nil
 }
 
-func (p *DefaultPackage) Revdeps(definitiondb PackageDatabase) []Package {
-	var versionsInWorld []Package
+func (p *DefaultPackage) Revdeps(definitiondb PackageDatabase) Packages {
+	var versionsInWorld Packages
 	for _, w := range definitiondb.World() {
 		if w.Matches(p) {
 			continue
@@ -353,11 +451,74 @@ func (p *DefaultPackage) Revdeps(definitiondb PackageDatabase) []Package {
 	return versionsInWorld
 }
 
+// ExpandedRevdeps returns the package reverse dependencies,
+// matching also selectors in versions (>, <, >=, <=)
+func (p *DefaultPackage) ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages {
+	var versionsInWorld Packages
+	if _, ok := visited[p.HumanReadableString()]; ok {
+		return versionsInWorld
+	}
+	visited[p.HumanReadableString()] = true
+
+	for _, w := range definitiondb.World() {
+		if w.Matches(p) {
+			continue
+		}
+		match := false
+
+		for _, re := range w.GetRequires() {
+			if re.Matches(p) {
+				match = true
+			}
+			if !match {
+
+				packages, _ := re.Expand(definitiondb)
+				for _, pa := range packages {
+					if pa.Matches(p) {
+						match = true
+					}
+				}
+			}
+
+			//	if ok, _ := w.RequiresContains(definitiondb, p); ok {
+
+		}
+
+		if match {
+			versionsInWorld = append(versionsInWorld, w)
+
+			versionsInWorld = append(versionsInWorld, w.ExpandedRevdeps(definitiondb, visited).Unique()...)
+
+		}
+		//	}
+
+	}
+	//visited[p.HumanReadableString()] = true
+	return versionsInWorld.Unique()
+}
+
+func (p *DefaultPackage) LabelDeps(definitiondb PackageDatabase, labelKey string) Packages {
+	var pkgsWithLabelInWorld Packages
+	// TODO: check if integrate some index to improve
+	// research instead of iterate all list.
+	for _, w := range definitiondb.World() {
+		if w.HasLabel(labelKey) {
+			pkgsWithLabelInWorld = append(pkgsWithLabelInWorld, w)
+		}
+	}
+
+	return pkgsWithLabelInWorld
+}
+
 func DecodePackage(ID string, db PackageDatabase) (Package, error) {
 	return db.GetPackage(ID)
 }
 
-func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Package) (bool, error) {
+func (pack *DefaultPackage) scanRequires(definitiondb PackageDatabase, s Package, visited map[string]interface{}) (bool, error) {
+	if _, ok := visited[pack.HumanReadableString()]; ok {
+		return false, nil
+	}
+	visited[pack.HumanReadableString()] = true
 	p, err := definitiondb.FindPackage(pack)
 	if err != nil {
 		p = pack //relax things
@@ -375,14 +536,27 @@ func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Pac
 				return true, nil
 			}
 		}
-		if contains, err := re.RequiresContains(definitiondb, s); err == nil && contains {
+		if contains, err := re.scanRequires(definitiondb, s, visited); err == nil && contains {
 			return true, nil
 		}
 	}
 
 	return false, nil
 }
-func Lower(set []Package) Package {
+
+// RequiresContains recursively scans into the database packages dependencies to find a match with the given package
+// It is used by the solver during uninstall.
+func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Package) (bool, error) {
+	return pack.scanRequires(definitiondb, s, make(map[string]interface{}))
+}
+
+// Best returns the best version of the package (the most bigger) from a list
+// Accepts a versioner interface to change the ordering policy. If null is supplied
+// It defaults to version.WrappedVersioner which supports both semver and debian versioning
+func (set Packages) Best(v version.Versioner) Package {
+	if v == nil {
+		v = &version.WrappedVersioner{}
+	}
 	var versionsMap map[string]Package = make(map[string]Package)
 	if len(set) == 0 {
 		panic("Best needs a list with elements")
@@ -393,43 +567,28 @@ func Lower(set []Package) Package {
 		versionsRaw = append(versionsRaw, p.GetVersion())
 		versionsMap[p.GetVersion()] = p
 	}
+	sorted := v.Sort(versionsRaw)
 
-	versions := make([]*version.Version, len(versionsRaw))
-	for i, raw := range versionsRaw {
-		v, _ := version.NewVersion(raw)
-		versions[i] = v
-	}
-
-	// After this, the versions are properly sorted
-	sort.Sort(version.Collection(versions))
-
-	return versionsMap[versions[0].Original()]
+	return versionsMap[sorted[len(sorted)-1]]
 }
-func Best(set []Package) Package {
-	var versionsMap map[string]Package = make(map[string]Package)
-	if len(set) == 0 {
-		panic("Best needs a list with elements")
-	}
 
-	versionsRaw := []string{}
+func (set Packages) Unique() Packages {
+	var result Packages
+	uniq := make(map[string]Package)
 	for _, p := range set {
-		versionsRaw = append(versionsRaw, p.GetVersion())
-		versionsMap[p.GetVersion()] = p
+		uniq[p.GetFingerPrint()] = p
 	}
-
-	versions := make([]*version.Version, len(versionsRaw))
-	for i, raw := range versionsRaw {
-		v, _ := version.NewVersion(raw)
-		versions[i] = v
+	for _, p := range uniq {
+		result = append(result, p)
 	}
-
-	// After this, the versions are properly sorted
-	sort.Sort(version.Collection(versions))
-
-	return versionsMap[versions[len(versions)-1].Original()]
+	return result
 }
 
-func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db PackageDatabase) ([]bf.Formula, error) {
+func (pack *DefaultPackage) buildFormula(definitiondb PackageDatabase, db PackageDatabase, visited map[string]interface{}) ([]bf.Formula, error) {
+	if _, ok := visited[pack.HumanReadableString()]; ok {
+		return nil, nil
+	}
+	visited[pack.HumanReadableString()] = true
 	p, err := definitiondb.FindPackage(pack)
 	if err != nil {
 		p = pack // Relax failures and trust the def
@@ -442,6 +601,22 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 	A := bf.Var(encodedA)
 
 	var formulas []bf.Formula
+
+	// Do conflict with other packages versions (if A is selected, then conflict with other versions of A)
+	packages, _ := definitiondb.FindPackageVersions(p)
+	if len(packages) > 0 {
+		for _, cp := range packages {
+			encodedB, err := cp.Encode(db)
+			if err != nil {
+				return nil, err
+			}
+			B := bf.Var(encodedB)
+			if !p.Matches(cp) {
+				formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Not(A), bf.Not(B))))
+			}
+		}
+	}
+
 	for _, requiredDef := range p.GetRequires() {
 		required, err := definitiondb.FindPackage(requiredDef)
 		if err != nil || requiredDef.IsSelector() {
@@ -453,62 +628,60 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 			if err != nil || len(packages) == 0 {
 				required = requiredDef
 			} else {
-				if len(packages) == 1 {
-					required = packages[0]
-				} else {
-					var ALO, priorityConstraints, priorityALO []bf.Formula
 
-					// Try to prio best match
-					// Force the solver to consider first our candidate (if does exists).
-					// Then builds ALO and AMO for the requires.
-					c, candidateErr := definitiondb.FindPackageCandidate(requiredDef)
-					var C bf.Formula
-					if candidateErr == nil {
-						// We have a desired candidate, try to look a solution with that included first
-						for _, o := range packages {
-							encodedB, err := o.Encode(db)
-							if err != nil {
-								return nil, err
-							}
-							B := bf.Var(encodedB)
-							if !o.Matches(c) {
-								priorityConstraints = append(priorityConstraints, bf.Not(B))
-								priorityALO = append(priorityALO, B)
-							}
-						}
-						encodedC, err := c.Encode(db)
-						if err != nil {
-							return nil, err
-						}
-						C = bf.Var(encodedC)
-						// Or the Candidate is true, or all the others might be not true
-						// This forces the CDCL sat implementation to look first at a solution with C=true
-						formulas = append(formulas, bf.Or(bf.Or(C, bf.Or(priorityConstraints...)), bf.Or(bf.Not(C), bf.Or(priorityALO...))))
-					}
+				var ALO, priorityConstraints, priorityALO []bf.Formula
 
-					// AMO - At most one
+				// Try to prio best match
+				// Force the solver to consider first our candidate (if does exists).
+				// Then builds ALO and AMO for the requires.
+				c, candidateErr := definitiondb.FindPackageCandidate(requiredDef)
+				var C bf.Formula
+				if candidateErr == nil {
+					// We have a desired candidate, try to look a solution with that included first
 					for _, o := range packages {
 						encodedB, err := o.Encode(db)
 						if err != nil {
 							return nil, err
 						}
 						B := bf.Var(encodedB)
-						ALO = append(ALO, B)
-						for _, i := range packages {
-							encodedI, err := i.Encode(db)
-							if err != nil {
-								return nil, err
-							}
-							I := bf.Var(encodedI)
-							if !o.Matches(i) {
-								formulas = append(formulas, bf.Or(bf.Not(I), bf.Not(B)))
-							}
+						if !o.Matches(c) {
+							priorityConstraints = append(priorityConstraints, bf.Not(B))
+							priorityALO = append(priorityALO, B)
 						}
 					}
-					formulas = append(formulas, bf.Or(ALO...)) // ALO - At least one
-					continue
+					encodedC, err := c.Encode(db)
+					if err != nil {
+						return nil, err
+					}
+					C = bf.Var(encodedC)
+					// Or the Candidate is true, or all the others might be not true
+					// This forces the CDCL sat implementation to look first at a solution with C=true
+					formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.And(C, bf.Or(priorityConstraints...)), bf.And(bf.Not(C), bf.Or(priorityALO...)))))
 				}
+
+				// AMO - At most one
+				for _, o := range packages {
+					encodedB, err := o.Encode(db)
+					if err != nil {
+						return nil, err
+					}
+					B := bf.Var(encodedB)
+					ALO = append(ALO, B)
+					for _, i := range packages {
+						encodedI, err := i.Encode(db)
+						if err != nil {
+							return nil, err
+						}
+						I := bf.Var(encodedI)
+						if !o.Matches(i) {
+							formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Not(I), bf.Not(B))))
+						}
+					}
+				}
+				formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(ALO...))) // ALO - At least one
+				continue
 			}
+
 		}
 
 		encodedB, err := required.Encode(db)
@@ -517,8 +690,8 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 		}
 		B := bf.Var(encodedB)
 		formulas = append(formulas, bf.Or(bf.Not(A), B))
-
-		f, err := required.BuildFormula(definitiondb, db)
+		r := required.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
+		f, err := r.buildFormula(definitiondb, db, visited)
 		if err != nil {
 			return nil, err
 		}
@@ -547,8 +720,8 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 						B := bf.Var(encodedB)
 						formulas = append(formulas, bf.Or(bf.Not(A),
 							bf.Not(B)))
-
-						f, err := p.BuildFormula(definitiondb, db)
+						r := p.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
+						f, err := r.buildFormula(definitiondb, db, visited)
 						if err != nil {
 							return nil, err
 						}
@@ -567,23 +740,28 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 		formulas = append(formulas, bf.Or(bf.Not(A),
 			bf.Not(B)))
 
-		f, err := required.BuildFormula(definitiondb, db)
+		r := required.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
+		f, err := r.buildFormula(definitiondb, db, visited)
 		if err != nil {
 			return nil, err
 		}
 		formulas = append(formulas, f...)
 
 	}
+
 	return formulas, nil
 }
 
+func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db PackageDatabase) ([]bf.Formula, error) {
+	return pack.buildFormula(definitiondb, db, make(map[string]interface{}))
+}
+
 func (p *DefaultPackage) Explain() {
 
 	fmt.Println("====================")
 	fmt.Println("Name: ", p.GetName())
 	fmt.Println("Category: ", p.GetCategory())
 	fmt.Println("Version: ", p.GetVersion())
-	fmt.Println("Installed: ", p.IsSet)
 
 	for _, req := range p.GetRequires() {
 		fmt.Println("\t-> ", req)
@@ -596,3 +774,108 @@ func (p *DefaultPackage) Explain() {
 	fmt.Println("====================")
 
 }
+
+func (p *DefaultPackage) BumpBuildVersion() error {
+	cat := p.Category
+	if cat == "" {
+		// Use fake category for parse package
+		cat = "app"
+	}
+	gp, err := gentoo.ParsePackageStr(
+		fmt.Sprintf("%s/%s-%s", cat,
+			p.Name, p.GetVersion()))
+	if err != nil {
+		return errors.Wrap(err, "Error on parser version")
+	}
+
+	buildPrefix := ""
+	buildId := 0
+
+	if gp.VersionBuild != "" {
+		// Check if version build is a number
+		buildId, err = strconv.Atoi(gp.VersionBuild)
+		if err == nil {
+			goto end
+		}
+		// POST: is not only a number
+
+		// TODO: check if there is a better way to handle all use cases.
+
+		r1 := regexp.MustCompile(`^r[0-9]*$`)
+		if r1 == nil {
+			return errors.New("Error on create regex for -r[0-9]")
+		}
+		if r1.MatchString(gp.VersionBuild) {
+			buildId, err = strconv.Atoi(strings.ReplaceAll(gp.VersionBuild, "r", ""))
+			if err == nil {
+				buildPrefix = "r"
+				goto end
+			}
+		}
+
+		p1 := regexp.MustCompile(`^p[0-9]*$`)
+		if p1 == nil {
+			return errors.New("Error on create regex for -p[0-9]")
+		}
+		if p1.MatchString(gp.VersionBuild) {
+			buildId, err = strconv.Atoi(strings.ReplaceAll(gp.VersionBuild, "p", ""))
+			if err == nil {
+				buildPrefix = "p"
+				goto end
+			}
+		}
+
+		rc1 := regexp.MustCompile(`^rc[0-9]*$`)
+		if rc1 == nil {
+			return errors.New("Error on create regex for -rc[0-9]")
+		}
+		if rc1.MatchString(gp.VersionBuild) {
+			buildId, err = strconv.Atoi(strings.ReplaceAll(gp.VersionBuild, "rc", ""))
+			if err == nil {
+				buildPrefix = "rc"
+				goto end
+			}
+		}
+
+		// Check if version build contains a dot
+		dotIdx := strings.LastIndex(gp.VersionBuild, ".")
+		if dotIdx > 0 {
+			buildPrefix = gp.VersionBuild[0 : dotIdx+1]
+			bVersion := gp.VersionBuild[dotIdx+1:]
+			buildId, err = strconv.Atoi(bVersion)
+			if err == nil {
+				goto end
+			}
+		}
+
+		buildPrefix = gp.VersionBuild + "."
+		buildId = 0
+	}
+
+end:
+
+	buildId++
+	p.Version = fmt.Sprintf("%s%s+%s%d",
+		gp.Version, gp.VersionSuffix, buildPrefix, buildId)
+
+	return nil
+}
+
+func (p *DefaultPackage) SelectorMatchVersion(ver string, v version.Versioner) (bool, error) {
+	if !p.IsSelector() {
+		return false, errors.New("Package is not a selector")
+	}
+	if v == nil {
+		v = &version.WrappedVersioner{}
+	}
+
+	return v.ValidateSelector(ver, p.GetVersion()), nil
+}
+
+func (p *DefaultPackage) VersionMatchSelector(selector string, v version.Versioner) (bool, error) {
+	if v == nil {
+		v = &version.WrappedVersioner{}
+	}
+
+	return v.ValidateSelector(p.GetVersion(), selector), nil
+}

pkg/package/package_test.go

@@ -16,6 +16,8 @@
 package pkg_test
 
 import (
+	"regexp"
+
 	. "github.com/mudler/luet/pkg/package"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -23,11 +25,28 @@ import (
 
 var _ = Describe("Package", func() {
 
+	Context("Encoding/Decoding", func() {
+		a := &DefaultPackage{Name: "test", Version: "1", Category: "t"}
+		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		It("Encodes and decodes correctly", func() {
+			Expect(a.String()).ToNot(Equal(""))
+			p := FromString(a.String())
+			Expect(p).To(Equal(a))
+		})
+
+		It("Generates packages fingerprint's hashes", func() {
+			Expect(a.HashFingerprint("")).ToNot(Equal(a1.HashFingerprint("")))
+			Expect(a.HashFingerprint("")).To(Equal("76972ef6991ec6102f33b401105c1351"))
+		})
+	})
+
 	Context("Simple package", func() {
 		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
 		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+		re := regexp.MustCompile("project[0-9][=].*")
 		It("Expands correctly", func() {
 			definitions := NewInMemoryDatabase(false)
 			for _, p := range []Package{a1, a11, a01} {
@@ -40,8 +59,66 @@ var _ = Describe("Package", func() {
 			Expect(lst).To(ContainElement(a1))
 			Expect(lst).ToNot(ContainElement(a01))
 			Expect(len(lst)).To(Equal(2))
-			p := Best(lst)
+			p := lst.Best(nil)
 			Expect(p).To(Equal(a11))
+			// Test annotation with null map
+			Expect(a.MatchAnnotation(re)).To(Equal(false))
+		})
+	})
+
+	Context("Find label on packages", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.AddLabel("project1", "test1")
+		a.AddLabel("label2", "value1")
+		b := NewPackage("B", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b.AddLabel("project2", "test2")
+		b.AddLabel("label2", "value1")
+		It("Expands correctly", func() {
+			var err error
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b} {
+				_, err = definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			re := regexp.MustCompile("project[0-9][=].*")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(re).ToNot(BeNil())
+			Expect(a.HasLabel("label2")).To(Equal(true))
+			Expect(a.HasLabel("label3")).To(Equal(false))
+			Expect(a.HasLabel("project1")).To(Equal(true))
+			Expect(b.HasLabel("project2")).To(Equal(true))
+			Expect(b.HasLabel("label2")).To(Equal(true))
+			Expect(b.MatchLabel(re)).To(Equal(true))
+			Expect(a.MatchLabel(re)).To(Equal(true))
+
+		})
+	})
+
+	Context("Find annotations on packages", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.AddAnnotation("project1", "test1")
+		a.AddAnnotation("label2", "value1")
+		b := NewPackage("B", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b.AddAnnotation("project2", "test2")
+		b.AddAnnotation("label2", "value1")
+		It("Expands correctly", func() {
+			var err error
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b} {
+				_, err = definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			re := regexp.MustCompile("project[0-9][=].*")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(re).ToNot(BeNil())
+			Expect(a.HasAnnotation("label2")).To(Equal(true))
+			Expect(a.HasAnnotation("label3")).To(Equal(false))
+			Expect(a.HasAnnotation("project1")).To(Equal(true))
+			Expect(b.HasAnnotation("project2")).To(Equal(true))
+			Expect(b.HasAnnotation("label2")).To(Equal(true))
+			Expect(b.MatchAnnotation(re)).To(Equal(true))
+			Expect(a.MatchAnnotation(re)).To(Equal(true))
+
 		})
 	})
 
@@ -113,6 +190,93 @@ var _ = Describe("Package", func() {
 		})
 	})
 
+	Context("revdeps", func() {
+		a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b := NewPackage("B", "1.0", []*DefaultPackage{&DefaultPackage{Name: "A", Version: ">=1.0"}}, []*DefaultPackage{})
+		c := NewPackage("C", "1.1", []*DefaultPackage{&DefaultPackage{Name: "B", Version: ">=1.0"}}, []*DefaultPackage{})
+		d := NewPackage("D", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
+		e := NewPackage("E", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
+
+		It("doesn't resolve selectors", func() {
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b, c, d, e} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			lst := a.Revdeps(definitions)
+			Expect(len(lst)).To(Equal(0))
+		})
+	})
+	Context("Expandedrevdeps", func() {
+		a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b := NewPackage("B", "1.0", []*DefaultPackage{&DefaultPackage{Name: "A", Version: ">=1.0"}}, []*DefaultPackage{})
+		c := NewPackage("C", "1.1", []*DefaultPackage{&DefaultPackage{Name: "B", Version: ">=1.0"}}, []*DefaultPackage{})
+		d := NewPackage("D", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
+		e := NewPackage("E", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
+
+		It("Computes correctly", func() {
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b, c, d, e} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			visited := make(map[string]interface{})
+			lst := a.ExpandedRevdeps(definitions, visited)
+			Expect(lst).To(ContainElement(c))
+			Expect(lst).To(ContainElement(d))
+			Expect(lst).To(ContainElement(e))
+			Expect(len(lst)).To(Equal(4))
+		})
+	})
+
+	Context("Expandedrevdeps", func() {
+		a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b := NewPackage("B", "1.0", []*DefaultPackage{&DefaultPackage{Name: "A", Version: ">=1.0"}}, []*DefaultPackage{})
+		c := NewPackage("C", "1.1", []*DefaultPackage{&DefaultPackage{Name: "B", Version: ">=1.0"}}, []*DefaultPackage{})
+		d := NewPackage("D", "0.1", []*DefaultPackage{&DefaultPackage{Name: "C", Version: ">=1.0"}}, []*DefaultPackage{})
+		e := NewPackage("E", "0.1", []*DefaultPackage{&DefaultPackage{Name: "C", Version: ">=1.0"}}, []*DefaultPackage{})
+
+		It("Computes correctly", func() {
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b, c, d, e} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			visited := make(map[string]interface{})
+
+			lst := a.ExpandedRevdeps(definitions, visited)
+			Expect(lst).To(ContainElement(b))
+			Expect(lst).To(ContainElement(c))
+			Expect(lst).To(ContainElement(d))
+			Expect(lst).To(ContainElement(e))
+			Expect(len(lst)).To(Equal(4))
+		})
+	})
+
+	Context("Expandedrevdeps", func() {
+		a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b := NewPackage("B", "1.0", []*DefaultPackage{&DefaultPackage{Name: "A", Version: ">=1.0"}}, []*DefaultPackage{})
+		c := NewPackage("C", "1.1", []*DefaultPackage{&DefaultPackage{Name: "B", Version: ">=1.0"}, &DefaultPackage{Name: "A", Version: ">=0"}}, []*DefaultPackage{})
+		d := NewPackage("D", "0.1", []*DefaultPackage{&DefaultPackage{Name: "C", Version: ">=1.0"}}, []*DefaultPackage{})
+		e := NewPackage("E", "0.1", []*DefaultPackage{&DefaultPackage{Name: "C", Version: ">=1.0"}}, []*DefaultPackage{})
+
+		It("Computes correctly", func() {
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b, c, d, e} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			visited := make(map[string]interface{})
+
+			lst := a.ExpandedRevdeps(definitions, visited)
+			Expect(lst).To(ContainElement(b))
+			Expect(lst).To(ContainElement(c))
+			Expect(lst).To(ContainElement(d))
+			Expect(lst).To(ContainElement(e))
+			Expect(len(lst)).To(Equal(4))
+		})
+	})
+
 	Context("RequiresContains", func() {
 		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a1 := NewPackage("A", "1.0", []*DefaultPackage{a}, []*DefaultPackage{})
@@ -150,7 +314,6 @@ var _ = Describe("Package", func() {
 
 			Expect(p.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(p.GetName()).To(Equal(a.GetName()))
-			Expect(p.Flagged()).To(Equal(a.Flagged()))
 			Expect(p.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(p.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(p.GetRequires())).To(Equal(len(a.GetRequires())))
@@ -195,7 +358,7 @@ var _ = Describe("Package", func() {
 
 			f, err := a1.BuildFormula(definitions, db)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(len(f)).To(Equal(2))
+			Expect(len(f)).To(Equal(8))
 			//	Expect(f[0].String()).To(Equal("or(not(c31f5842), a4910f77)"))
 			//	Expect(f[1].String()).To(Equal("or(not(c31f5842), not(a97670be))"))
 		})
@@ -210,7 +373,6 @@ var _ = Describe("Package", func() {
 			a2 := a.Clone()
 			Expect(a2.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(a2.GetName()).To(Equal(a.GetName()))
-			Expect(a2.Flagged()).To(Equal(a.Flagged()))
 			Expect(a2.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(a2.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(a2.GetRequires())).To(Equal(len(a.GetRequires())))
@@ -236,4 +398,69 @@ var _ = Describe("Package", func() {
 			Expect(len(a1.GetUses())).To(Equal(0))
 		})
 	})
+
+	Context("Check Bump build Version", func() {
+		It("Bump without build version", func() {
+			a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			err := a1.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(a1.GetVersion()).To(Equal("1.0+1"))
+		})
+		It("Bump 2", func() {
+			p := NewPackage("A", "1.0+1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+2"))
+		})
+
+		It("Bump 3", func() {
+			p := NewPackage("A", "1.0+100", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+101"))
+		})
+
+		It("Bump 4", func() {
+			p := NewPackage("A", "1.0+r1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+r2"))
+		})
+
+		It("Bump 5", func() {
+			p := NewPackage("A", "1.0+p1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+p2"))
+		})
+
+		It("Bump 6", func() {
+			p := NewPackage("A", "1.0+pre20200315", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+pre20200315.1"))
+		})
+
+		It("Bump 7", func() {
+			p := NewPackage("A", "1.0+pre20200315.1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+pre20200315.2"))
+		})
+
+		It("Bump 8", func() {
+			p := NewPackage("A", "1.0+d-r1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+d-r1.1"))
+		})
+
+		It("Bump 9", func() {
+			p := NewPackage("A", "1.0+p20200315.1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+p20200315.2"))
+		})
+	})
+
 })

pkg/repository/loader.go

@@ -28,14 +28,14 @@ import (
 )
 
 func LoadRepositories(c *LuetConfig) error {
-	var regexRepo = regexp.MustCompile(`.yml$`)
+	var regexRepo = regexp.MustCompile(`.yml$|.yaml$`)
 
 	for _, rdir := range c.RepositoriesConfDir {
 		Debug("Parsing Repository Directory", rdir, "...")
 
 		files, err := ioutil.ReadDir(rdir)
 		if err != nil {
-			Warning("Skip dir", rdir, ":", err.Error())
+			Debug("Skip dir", rdir, ":", err.Error())
 			continue
 		}
 

pkg/repository/repository_test.go

@@ -35,12 +35,22 @@ var _ = Describe("Repository", func() {
 
 		It("Chec Load Repository 1", func() {
 			Expect(err).Should(BeNil())
-			Expect(len(cfg.SystemRepositories)).Should(Equal(1))
+			Expect(len(cfg.SystemRepositories)).Should(Equal(2))
 			Expect(cfg.SystemRepositories[0].Name).Should(Equal("test1"))
 			Expect(cfg.SystemRepositories[0].Priority).Should(Equal(999))
 			Expect(cfg.SystemRepositories[0].Type).Should(Equal("disk"))
 			Expect(len(cfg.SystemRepositories[0].Urls)).Should(Equal(1))
 			Expect(cfg.SystemRepositories[0].Urls[0]).Should(Equal("tests/repos/test1"))
 		})
+
+		It("Chec Load Repository 2", func() {
+			Expect(err).Should(BeNil())
+			Expect(len(cfg.SystemRepositories)).Should(Equal(2))
+			Expect(cfg.SystemRepositories[1].Name).Should(Equal("test2"))
+			Expect(cfg.SystemRepositories[1].Priority).Should(Equal(1000))
+			Expect(cfg.SystemRepositories[1].Type).Should(Equal("disk"))
+			Expect(len(cfg.SystemRepositories[1].Urls)).Should(Equal(1))
+			Expect(cfg.SystemRepositories[1].Urls[0]).Should(Equal("tests/repos/test2"))
+		})
 	})
 })

pkg/solver/benchmark_test.go

@@ -0,0 +1,142 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver_test
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"os"
+	"strconv"
+
+	. "github.com/mudler/luet/pkg/solver"
+)
+
+var _ = Describe("Solver Benchmarks", func() {
+	db := pkg.NewInMemoryDatabase(false)
+	dbInstalled := pkg.NewInMemoryDatabase(false)
+	dbDefinitions := pkg.NewInMemoryDatabase(false)
+	var s PackageSolver
+
+	Context("Complex data sets", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+		Measure("it should be fast in resolution from a 50000 dataset", func(b Benchmarker) {
+
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 50000; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					_, err := dbInstalled.CreatePackage(C)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for i := 0; i < 1; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+					solution, err := s.Install([]pkg.Package{A})
+					Expect(err).ToNot(HaveOccurred())
+
+					Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: H, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: G, Value: true}))
+				}
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 120), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+	Context("Complex data sets - Parallel", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: ParallelSimple, Concurrency: 10}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+		Measure("it should be fast in resolution from a 50000 dataset", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 50000; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					_, err := dbInstalled.CreatePackage(C)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				for i := 0; i < 1; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+					solution, err := s.Install([]pkg.Package{A})
+					Expect(err).ToNot(HaveOccurred())
+
+					Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: H, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: G, Value: true}))
+
+					//	Expect(len(solution)).To(Equal(6))
+				}
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+})

pkg/solver/decoder.go

@@ -23,6 +23,7 @@ import (
 
 	pkg "github.com/mudler/luet/pkg/package"
 	toposort "github.com/philopon/go-toposort"
+	"github.com/pkg/errors"
 	"github.com/stevenle/topsort"
 )
 
@@ -145,7 +146,7 @@ func (assertions PackagesAssertions) Search(f string) *PackageAssert {
 	return nil
 }
 
-func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fingerprint string) PackagesAssertions {
+func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fingerprint string) (PackagesAssertions, error) {
 
 	orderedAssertions := PackagesAssertions{}
 	unorderedAssertions := PackagesAssertions{}
@@ -169,10 +170,10 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 
 	sort.Sort(unorderedAssertions)
 	// Build a topological graph
-	//graph := toposort.NewGraph(len(unorderedAssertions))
-	//	graph.AddNodes(fingerprints...)
 	for _, a := range unorderedAssertions {
 		currentPkg := a.Package
+		added := map[string]interface{}{}
+	REQUIRES:
 		for _, requiredDef := range currentPkg.GetRequires() {
 			if def, err := definitiondb.FindPackage(requiredDef); err == nil { // Provides: Get a chance of being override here
 				requiredDef = def.(*pkg.DefaultPackage)
@@ -184,26 +185,29 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 			if req != nil {
 				requiredDef = req.Package
 			}
-
+			if _, ok := added[requiredDef.GetFingerPrint()]; ok {
+				continue REQUIRES
+			}
 			// Expand also here, as we need to order them (or instead the solver should give back the dep correctly?)
 			graph.AddEdge(currentPkg.GetFingerPrint(), requiredDef.GetFingerPrint())
+			added[requiredDef.GetFingerPrint()] = nil
 		}
 	}
 	result, err := graph.TopSort(fingerprint)
 	if err != nil {
-		panic(err)
+		return nil, errors.Wrap(err, "fail on sorting "+fingerprint)
 	}
 	for _, res := range result {
 		a, ok := tmpMap[res]
 		if !ok {
-			panic("fail looking for " + res)
+			return nil, errors.New("fail looking for " + res)
 			//	continue
 		}
 		orderedAssertions = append(orderedAssertions, a)
 		//	orderedAssertions = append(PackagesAssertions{a}, orderedAssertions...) // push upfront
 	}
 	//helpers.ReverseAny(orderedAssertions)
-	return orderedAssertions
+	return orderedAssertions, nil
 }
 
 func (assertions PackagesAssertions) Explain() string {
@@ -247,6 +251,34 @@ func (a PackagesAssertions) Less(i, j int) bool {
 
 }
 
+func (a PackagesAssertions) TrueLen() int {
+	count := 0
+	for _, ass := range a {
+		if ass.Value {
+			count++
+		}
+	}
+
+	return count
+}
+
+// HashFrom computes the assertion hash From a given package. It drops it from the assertions
+// and checks it's not the only one. if it's unique it marks it specially - so the hash
+// which is generated is unique for the selected package
+func (assertions PackagesAssertions) HashFrom(p pkg.Package) string {
+
+	var assertionhash string
+
+	// When we don't have any solution to hash for, we need to generate an UUID by ourselves
+	latestsolution := assertions.Drop(p)
+	if latestsolution.TrueLen() == 0 {
+		assertionhash = assertions.Mark(p).AssertionHash()
+	} else {
+		assertionhash = latestsolution.AssertionHash()
+	}
+	return assertionhash
+}
+
 func (assertions PackagesAssertions) AssertionHash() string {
 	var fingerprint string
 	for _, assertion := range assertions { // Note: Always order them first!
@@ -283,3 +315,18 @@ func (assertions PackagesAssertions) Cut(p pkg.Package) PackagesAssertions {
 	}
 	return ass
 }
+
+// Mark returns a new assertion with the package marked
+func (assertions PackagesAssertions) Mark(p pkg.Package) PackagesAssertions {
+	ass := PackagesAssertions{}
+
+	for _, a := range assertions {
+		if a.Package.Matches(p) {
+			marked := a.Package.Clone()
+			marked.SetName("@@" + marked.GetName())
+			a = PackageAssert{Package: marked.(*pkg.DefaultPackage), Value: a.Value, Hash: a.Hash}
+		}
+		ass = append(ass, a)
+	}
+	return ass
+}

pkg/solver/decoder_test.go

@@ -19,6 +19,7 @@ import (
 	"strconv"
 
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -29,13 +30,13 @@ var _ = Describe("Decoder", func() {
 	db := pkg.NewInMemoryDatabase(false)
 	dbInstalled := pkg.NewInMemoryDatabase(false)
 	dbDefinitions := pkg.NewInMemoryDatabase(false)
-	s := NewSolver(dbInstalled, dbDefinitions, db)
+	s := NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 	BeforeEach(func() {
 		db = pkg.NewInMemoryDatabase(false)
 		dbInstalled = pkg.NewInMemoryDatabase(false)
 		dbDefinitions = pkg.NewInMemoryDatabase(false)
-		s = NewSolver(dbInstalled, dbDefinitions, db)
+		s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 	})
 
 	Context("Assertion ordering", func() {
@@ -72,7 +73,8 @@ var _ = Describe("Decoder", func() {
 
 				Expect(len(solution)).To(Equal(6))
 				Expect(err).ToNot(HaveOccurred())
-				solution = solution.Order(dbDefinitions, A.GetFingerPrint())
+				solution, err = solution.Order(dbDefinitions, A.GetFingerPrint())
+				Expect(err).ToNot(HaveOccurred())
 				//	Expect(len(solution)).To(Equal(6))
 				Expect(solution[0].Package.GetName()).To(Equal("G"))
 				Expect(solution[1].Package.GetName()).To(Equal("H"))
@@ -188,7 +190,8 @@ var _ = Describe("Decoder", func() {
 
 			Expect(len(solution)).To(Equal(6))
 			Expect(err).ToNot(HaveOccurred())
-			solution = solution.Order(dbDefinitions, A.GetFingerPrint())
+			solution, err = solution.Order(dbDefinitions, A.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
 			//	Expect(len(solution)).To(Equal(6))
 			Expect(solution[0].Package.GetName()).To(Equal("G"))
 			Expect(solution[1].Package.GetName()).To(Equal("H"))
@@ -206,7 +209,8 @@ var _ = Describe("Decoder", func() {
 
 			Expect(len(solution)).To(Equal(6))
 			Expect(err).ToNot(HaveOccurred())
-			solution = solution.Order(dbDefinitions, B.GetFingerPrint())
+			solution, err = solution.Order(dbDefinitions, B.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
 			hash2 := solution.AssertionHash()
 
 			//	Expect(len(solution)).To(Equal(6))
@@ -243,7 +247,11 @@ var _ = Describe("Decoder", func() {
 
 			solution2, err := s.Install([]pkg.Package{Z})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(solution.Order(dbDefinitions, Y.GetFingerPrint()).Drop(Y).AssertionHash() == solution2.Order(dbDefinitions, Z.GetFingerPrint()).Drop(Z).AssertionHash()).To(BeTrue())
+			orderY, err := solution.Order(dbDefinitions, Y.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
+			orderZ, err := solution2.Order(dbDefinitions, Z.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(orderY.Drop(Y).AssertionHash() == orderZ.Drop(Z).AssertionHash()).To(BeTrue())
 		})
 
 		It("Hashes them, Cuts them and could be used for comparison", func() {
@@ -267,10 +275,120 @@ var _ = Describe("Decoder", func() {
 
 			solution2, err := s.Install([]pkg.Package{Z})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(solution.Order(dbDefinitions, Y.GetFingerPrint()).Cut(Y).Drop(Y)).To(Equal(solution2.Order(dbDefinitions, Z.GetFingerPrint()).Cut(Z).Drop(Z)))
+			orderY, err := solution.Order(dbDefinitions, Y.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
+			orderZ, err := solution2.Order(dbDefinitions, Z.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(orderY.Cut(Y).Drop(Y)).To(Equal(orderZ.Cut(Z).Drop(Z)))
 
-			Expect(solution.Order(dbDefinitions, Y.GetFingerPrint()).Cut(Y).Drop(Y).AssertionHash()).To(Equal(solution2.Order(dbDefinitions, Z.GetFingerPrint()).Cut(Z).Drop(Z).AssertionHash()))
+			Expect(orderY.Cut(Y).Drop(Y).AssertionHash()).To(Equal(orderZ.Cut(Z).Drop(Z).AssertionHash()))
 		})
 
+		It("HashFrom can be used equally", func() {
+
+			X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+			Z := pkg.NewPackage("Z", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{X, Y, Z} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			solution, err := s.Install([]pkg.Package{Y})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution2, err := s.Install([]pkg.Package{Z})
+			Expect(err).ToNot(HaveOccurred())
+			orderY, err := solution.Order(dbDefinitions, Y.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
+			orderZ, err := solution2.Order(dbDefinitions, Z.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(orderY.Cut(Y).Drop(Y)).To(Equal(orderZ.Cut(Z).Drop(Z)))
+
+			Expect(orderY.Cut(Y).HashFrom(Y)).To(Equal(orderZ.Cut(Z).HashFrom(Z)))
+		})
+
+		It("Unique hashes for single packages", func() {
+
+			X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{X, F, D} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			solution, err := s.Install([]pkg.Package{X})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution2, err := s.Install([]pkg.Package{F})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution3, err := s.Install([]pkg.Package{D})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(solution.AssertionHash()).ToNot(Equal(solution2.AssertionHash()))
+			Expect(solution3.AssertionHash()).To(Equal(solution.AssertionHash()))
+
+		})
+
+		It("Unique hashes for empty assertions", func() {
+			empty := solver.PackagesAssertions{}
+			empty2 := solver.PackagesAssertions{}
+
+			Expect(empty.AssertionHash()).To(Equal(empty2.AssertionHash()))
+		})
+
+		It("Unique hashes for single packages with HashFrom", func() {
+
+			X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+
+			empty := solver.PackagesAssertions{}
+
+			for _, p := range []pkg.Package{X, F, D, Y} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			solution, err := s.Install([]pkg.Package{X})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution2, err := s.Install([]pkg.Package{F})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution3, err := s.Install([]pkg.Package{D})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution4, err := s.Install([]pkg.Package{Y})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(solution.HashFrom(X)).ToNot(Equal(solution2.HashFrom(F)))
+			Expect(solution3.HashFrom(D)).To(Equal(solution.HashFrom(X)))
+
+			Expect(empty.AssertionHash()).ToNot(Equal(solution3.HashFrom(D)))
+			Expect(empty.AssertionHash()).ToNot(Equal(solution2.HashFrom(F)))
+
+			Expect(solution4.Drop(Y).AssertionHash()).To(Equal(solution4.HashFrom(Y)))
+		})
 	})
 })

pkg/solver/parallel.go

@@ -0,0 +1,792 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver
+
+import (
+
+	//. "github.com/mudler/luet/pkg/logger"
+	"fmt"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"github.com/crillab/gophersat/bf"
+	pkg "github.com/mudler/luet/pkg/package"
+)
+
+// Parallel is the default Parallel for luet
+type Parallel struct {
+	Concurrency        int
+	DefinitionDatabase pkg.PackageDatabase
+	ParallelDatabase   pkg.PackageDatabase
+	Wanted             pkg.Packages
+	InstalledDatabase  pkg.PackageDatabase
+
+	Resolver PackageResolver
+}
+
+func (s *Parallel) SetDefinitionDatabase(db pkg.PackageDatabase) {
+	s.DefinitionDatabase = db
+}
+
+// SetReSolver is a setter for the unsat ReSolver backend
+func (s *Parallel) SetResolver(r PackageResolver) {
+	s.Resolver = r
+}
+
+func (s *Parallel) World() pkg.Packages {
+	return s.DefinitionDatabase.World()
+}
+
+func (s *Parallel) Installed() pkg.Packages {
+
+	return s.InstalledDatabase.World()
+}
+
+func (s *Parallel) noRulesWorld() bool {
+	for _, p := range s.World() {
+		if len(p.GetConflicts()) != 0 || len(p.GetRequires()) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (s *Parallel) noRulesInstalled() bool {
+	for _, p := range s.Installed() {
+		if len(p.GetConflicts()) != 0 || len(p.GetRequires()) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (s *Parallel) buildParallelFormula(formulas []bf.Formula, packages pkg.Packages) (bf.Formula, error) {
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				solvable, err := p.BuildFormula(s.DefinitionDatabase, s.ParallelDatabase)
+				if err != nil {
+					panic(err)
+				}
+				for _, s := range solvable {
+					results <- s
+				}
+			}
+		}(wg, all)
+	}
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	for _, p := range packages {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+	return bf.And(formulas...), nil
+}
+
+func (s *Parallel) BuildInstalled() (bf.Formula, error) {
+	var formulas []bf.Formula
+	return s.buildParallelFormula(formulas, s.Installed())
+}
+
+// BuildWorld builds the formula which olds the requirements from the package definitions
+// which are available (global state)
+func (s *Parallel) BuildWorld(includeInstalled bool) (bf.Formula, error) {
+	var formulas []bf.Formula
+	// NOTE: This block should be enabled in case of very old systems with outdated world sets
+	if includeInstalled {
+		solvable, err := s.BuildInstalled()
+		if err != nil {
+			return nil, err
+		}
+		//f = bf.And(f, solvable)
+		formulas = append(formulas, solvable)
+	}
+
+	return s.buildParallelFormula(formulas, s.World())
+}
+
+func (s *Parallel) getList(db pkg.PackageDatabase, lsp pkg.Packages) (pkg.Packages, error) {
+	var ls pkg.Packages
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan pkg.Package, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				cp, err := db.FindPackage(p)
+				if err != nil {
+					packages, err := p.Expand(db)
+					// Expand, and relax search - if not found pick the same one
+					if err != nil || len(packages) == 0 {
+						cp = p
+					} else {
+						cp = packages.Best(nil)
+					}
+				}
+				results <- cp
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func(wg *sync.WaitGroup) {
+		defer wg2.Done()
+		for t := range results {
+			ls = append(ls, t)
+		}
+	}(wg)
+
+	for _, pp := range lsp {
+		all <- pp
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	return ls, nil
+}
+
+// Conflicts acts like ConflictsWith, but uses package's reverse dependencies to
+// determine if it conflicts with the given set
+func (s *Parallel) Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error) {
+	p, err := s.DefinitionDatabase.FindPackage(pack)
+	if err != nil {
+		p = pack
+	}
+
+	ls, err := s.getList(s.DefinitionDatabase, lsp)
+	if err != nil {
+		return false, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	if s.noRulesWorld() {
+		return false, nil
+	}
+
+	temporarySet := pkg.NewInMemoryDatabase(false)
+	for _, p := range ls {
+		temporarySet.CreatePackage(p)
+	}
+	visited := make(map[string]interface{})
+	revdeps := p.ExpandedRevdeps(temporarySet, visited)
+
+	var revdepsErr error
+	for _, r := range revdeps {
+		if revdepsErr == nil {
+			revdepsErr = errors.New("")
+		}
+		revdepsErr = errors.New(fmt.Sprintf("%s\n%s", revdepsErr.Error(), r.HumanReadableString()))
+	}
+
+	return len(revdeps) != 0, revdepsErr
+}
+
+// ConflictsWith return true if a package is part of the requirement set of a list of package
+// return false otherwise (and thus it is NOT relevant to the given list)
+func (s *Parallel) ConflictsWith(pack pkg.Package, lsp pkg.Packages) (bool, error) {
+	p, err := s.DefinitionDatabase.FindPackage(pack)
+	if err != nil {
+		p = pack //Relax search, otherwise we cannot compute solutions for packages not in definitions
+	}
+
+	ls, err := s.getList(s.DefinitionDatabase, lsp)
+	if err != nil {
+		return false, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	var formulas []bf.Formula
+
+	if s.noRulesWorld() {
+		return false, nil
+	}
+
+	encodedP, err := p.Encode(s.ParallelDatabase)
+	if err != nil {
+		return false, err
+	}
+	P := bf.Var(encodedP)
+
+	r, err := s.BuildWorld(false)
+	if err != nil {
+		return false, err
+	}
+	formulas = append(formulas, bf.And(bf.Not(P), r))
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for i := range c {
+				if i.Matches(p) {
+					continue
+				}
+
+				// XXX: Skip check on any of its requires ?  ( Drop to avoid removing system packages when selecting an uninstall)
+				// if i.RequiresContains(p) {
+				// 	fmt.Println("Requires found")
+				// 	continue
+				// }
+
+				encodedI, err := i.Encode(s.ParallelDatabase)
+				if err != nil {
+					panic(err)
+				}
+				I := bf.Var(encodedI)
+
+				results <- bf.And(I, r)
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	for _, p := range ls {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return true, nil
+	}
+
+	return false, nil
+
+}
+
+func (s *Parallel) ConflictsWithInstalled(p pkg.Package) (bool, error) {
+	return s.ConflictsWith(p, s.Installed())
+}
+
+// UninstallUniverse takes a list of candidate package and return a list of packages that would be removed
+// in order to purge the candidate. Uses the Parallel to check constraints and nothing else
+//
+// It can be compared to the counterpart Uninstall as this method acts like a uninstall --full
+// it removes all the packages and its deps. taking also in consideration other packages that might have
+// revdeps
+func (s *Parallel) UninstallUniverse(toremove pkg.Packages) (pkg.Packages, error) {
+
+	if s.noRulesInstalled() {
+		return s.getList(s.InstalledDatabase, toremove)
+	}
+
+	// resolve to packages from the db
+	toRemove, err := s.getList(s.InstalledDatabase, toremove)
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	var formulas []bf.Formula
+	r, err := s.BuildInstalled()
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	// SAT encode the clauses against the world
+	for _, p := range toRemove.Unique() {
+		encodedP, err := p.Encode(s.InstalledDatabase)
+		if err != nil {
+			return nil, errors.Wrap(err, "Package not found in definition db")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(bf.Not(P), r))
+	}
+
+	markedForRemoval := pkg.Packages{}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, errors.New("Failed finding a solution")
+	}
+	assertion, err := DecodeModel(model, s.InstalledDatabase)
+	if err != nil {
+		return nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+	}
+	return markedForRemoval, nil
+}
+
+// UpgradeUniverse mark packages for removal and returns a solution. It considers
+// the Universe db as authoritative
+// See also on the subject: https://arxiv.org/pdf/1007.1021.pdf
+func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssertions, error) {
+	// we first figure out which aren't up-to-date
+	// which has to be removed
+	// and which needs to be upgraded
+	notUptodate := pkg.Packages{}
+	removed := pkg.Packages{}
+	toUpgrade := pkg.Packages{}
+
+	// TODO: this is memory expensive, we need to optimize this
+	universe := pkg.NewInMemoryDatabase(false)
+
+	for _, p := range s.DefinitionDatabase.World() {
+		universe.CreatePackage(p)
+	}
+	for _, p := range s.Installed() {
+		universe.CreatePackage(p)
+	}
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan []pkg.Package, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				available, err := universe.FindPackageVersions(p)
+				if err != nil {
+					removed = append(removed, p) /// FIXME: Racy
+				}
+				if len(available) == 0 {
+					continue
+				}
+
+				bestmatch := available.Best(nil)
+				// Found a better version available
+				if !bestmatch.Matches(p) {
+					results <- []pkg.Package{p, bestmatch}
+				}
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			notUptodate = append(notUptodate, t[0])
+			toUpgrade = append(toUpgrade, t[1])
+		}
+	}()
+
+	// Grab all the installed ones, see if they are eligible for update
+	for _, p := range s.Installed() {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	// resolve to packages from the db to be able to encode correctly
+	oldPackages, err := s.getList(universe, notUptodate)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't get package marked for removal from universe")
+	}
+
+	updates, err := s.getList(universe, toUpgrade)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't get package marked for update from universe")
+	}
+
+	var formulas []bf.Formula
+
+	// Build constraints for the whole defdb
+	r, err := s.BuildWorld(true)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't build world constraints")
+	}
+
+	// Treat removed packages from universe as marked for deletion
+	if dropremoved {
+		oldPackages = append(oldPackages, removed...)
+	}
+
+	// SAT encode the clauses against the world
+	for _, p := range oldPackages.Unique() {
+		encodedP, err := p.Encode(universe)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "couldn't encode package")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(bf.Not(P), r))
+	}
+
+	for _, p := range updates {
+		encodedP, err := p.Encode(universe)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "couldn't encode package")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(P, r))
+	}
+
+	markedForRemoval := pkg.Packages{}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, nil, errors.New("Failed finding a solution")
+	}
+
+	assertion, err := DecodeModel(model, universe)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+
+	}
+	return markedForRemoval, assertion, nil
+}
+
+// Upgrade compute upgrades of the package against the world definition.
+// It accepts two boolean indicating if it has to check for conflicts or try to attempt a full upgrade
+func (s *Parallel) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAssertions, error) {
+
+	// First get candidates that needs to be upgraded..
+
+	toUninstall := pkg.Packages{}
+	toInstall := pkg.Packages{}
+
+	availableCache := map[string]pkg.Packages{}
+	for _, p := range s.DefinitionDatabase.World() {
+		// Each one, should be expanded
+		availableCache[p.GetName()+p.GetCategory()] = append(availableCache[p.GetName()+p.GetCategory()], p)
+	}
+
+	installedcopy := pkg.NewInMemoryDatabase(false)
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan []pkg.Package, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				installedcopy.CreatePackage(p)
+				packages, ok := availableCache[p.GetName()+p.GetCategory()]
+				if ok && len(packages) != 0 {
+					best := packages.Best(nil)
+					if best.GetVersion() != p.GetVersion() {
+						results <- []pkg.Package{p, best}
+					}
+				}
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			toUninstall = append(toUninstall, t[0])
+			toInstall = append(toInstall, t[1])
+		}
+	}()
+
+	for _, p := range s.InstalledDatabase.World() {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	s2 := &Parallel{Concurrency: s.Concurrency, InstalledDatabase: installedcopy, DefinitionDatabase: s.DefinitionDatabase, ParallelDatabase: pkg.NewInMemoryDatabase(false)}
+	s2.SetResolver(s.Resolver)
+	if !full {
+		ass := PackagesAssertions{}
+		for _, i := range toInstall {
+			ass = append(ass, PackageAssert{Package: i.(*pkg.DefaultPackage), Value: true})
+		}
+	}
+
+	// Then try to uninstall the versions in the system, and store that tree
+	for _, p := range toUninstall {
+		r, err := s.Uninstall(p, checkconflicts, false)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't uninstall selected candidate "+p.GetFingerPrint())
+		}
+		for _, z := range r {
+			err = installedcopy.RemovePackage(z)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't remove copy of package targetted for removal")
+			}
+		}
+	}
+
+	r, e := s2.Install(toInstall)
+	return toUninstall, r, e
+	// To that tree, ask to install the versions that should be upgraded, and try to solve
+	// Return the solution
+
+}
+
+// Uninstall takes a candidate package and return a list of packages that would be removed
+// in order to purge the candidate. Returns error if unsat.
+func (s *Parallel) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packages, error) {
+	var res pkg.Packages
+	candidate, err := s.InstalledDatabase.FindPackage(c)
+	if err != nil {
+
+		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
+		packages, err := c.Expand(s.InstalledDatabase)
+		//	Info("Expanded", packages, err)
+		if err != nil || len(packages) == 0 {
+			candidate = c
+		} else {
+			candidate = packages.Best(nil)
+		}
+		//Relax search, otherwise we cannot compute solutions for packages not in definitions
+		//	return nil, errors.Wrap(err, "Package not found between installed")
+	}
+	// Build a fake "Installed" - Candidate and its requires tree
+	var InstalledMinusCandidate pkg.Packages
+
+	// We are asked to not perform a full uninstall (checking all the possible requires that could
+	// be removed). Let's only check if we can remove the selected package
+	if !full && checkconflicts {
+		if conflicts, err := s.Conflicts(candidate, s.Installed()); conflicts {
+			return nil, err
+		} else {
+			return pkg.Packages{candidate}, nil
+		}
+	}
+
+	// TODO: Can be optimized
+	for _, i := range s.Installed() {
+		if !i.Matches(candidate) {
+			contains, err := candidate.RequiresContains(s.ParallelDatabase, i)
+			if err != nil {
+				return nil, errors.Wrap(err, "Failed getting installed list")
+			}
+			if !contains {
+				InstalledMinusCandidate = append(InstalledMinusCandidate, i)
+			}
+		}
+	}
+
+	s2 := &Parallel{Concurrency: s.Concurrency, InstalledDatabase: pkg.NewInMemoryDatabase(false), DefinitionDatabase: s.DefinitionDatabase, ParallelDatabase: pkg.NewInMemoryDatabase(false)}
+	s2.SetResolver(s.Resolver)
+	// Get the requirements to install the candidate
+	asserts, err := s2.Install(pkg.Packages{candidate})
+	if err != nil {
+		return nil, err
+	}
+	for _, a := range asserts {
+		if a.Value {
+			if !checkconflicts {
+				res = append(res, a.Package)
+				continue
+			}
+
+			c, err := s.ConflictsWithInstalled(a.Package)
+			if err != nil {
+				return nil, err
+			}
+
+			// If doesn't conflict with installed we just consider it for removal and look for the next one
+			if !c {
+				res = append(res, a.Package)
+				continue
+			}
+
+			// If does conflicts, give it another chance by checking conflicts if in case we didn't installed our candidate and all the required packages in the system
+			c, err = s.ConflictsWith(a.Package, InstalledMinusCandidate)
+			if err != nil {
+				return nil, err
+			}
+			if !c {
+				res = append(res, a.Package)
+			}
+
+		}
+
+	}
+
+	return res, nil
+}
+
+// BuildFormula builds the main solving formula that is evaluated by the sat Parallel.
+func (s *Parallel) BuildFormula() (bf.Formula, error) {
+	var formulas []bf.Formula
+	r, err := s.BuildWorld(false)
+	if err != nil {
+		return nil, err
+	}
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for wanted := range c {
+				encodedW, err := wanted.Encode(s.ParallelDatabase)
+				if err != nil {
+					panic(err)
+				}
+				W := bf.Var(encodedW)
+				installedWorld := s.Installed()
+				//TODO:Optimize
+				if len(installedWorld) == 0 {
+					results <- W
+					continue
+				}
+
+				for _, installed := range installedWorld {
+					encodedI, err := installed.Encode(s.ParallelDatabase)
+					if err != nil {
+						panic(err)
+					}
+					I := bf.Var(encodedI)
+					results <- bf.And(W, I)
+				}
+			}
+		}(wg, all)
+	}
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	for _, wanted := range s.Wanted {
+		all <- wanted
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	formulas = append(formulas, r)
+
+	return bf.And(formulas...), nil
+}
+
+func (s *Parallel) solve(f bf.Formula) (map[string]bool, bf.Formula, error) {
+	model := bf.Solve(f)
+	if model == nil {
+		return model, f, errors.New("Unsolvable")
+	}
+
+	return model, f, nil
+}
+
+// Solve builds the formula given the current state and returns package assertions
+func (s *Parallel) Solve() (PackagesAssertions, error) {
+	var model map[string]bool
+	var err error
+
+	f, err := s.BuildFormula()
+
+	if err != nil {
+		return nil, err
+	}
+
+	model, _, err = s.solve(f)
+	if err != nil && s.Resolver != nil {
+		return s.Resolver.Solve(f, s)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return DecodeModel(model, s.ParallelDatabase)
+}
+
+// Install given a list of packages, returns package assertions to indicate the packages that must be installed in the system in order
+// to statisfy all the constraints
+func (s *Parallel) Install(c pkg.Packages) (PackagesAssertions, error) {
+
+	coll, err := s.getList(s.DefinitionDatabase, c)
+	if err != nil {
+		return nil, errors.Wrap(err, "Packages not found in definition db")
+	}
+
+	s.Wanted = coll
+
+	if s.noRulesWorld() {
+		var ass PackagesAssertions
+		for _, p := range s.Installed() {
+			ass = append(ass, PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
+
+		}
+		for _, p := range s.Wanted {
+			ass = append(ass, PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
+		}
+		return ass, nil
+	}
+
+	return s.Solve()
+}

pkg/solver/resolver.go

@@ -0,0 +1,336 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/crillab/gophersat/bf"
+	"github.com/mudler/luet/pkg/helpers"
+	"gopkg.in/yaml.v2"
+
+	"github.com/ecooper/qlearning"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/pkg/errors"
+)
+
+type ActionType int
+
+const (
+	NoAction      = 0
+	Solved        = iota
+	NoSolution    = iota
+	Going         = iota
+	ActionRemoved = iota
+	ActionAdded   = iota
+
+	DoNoop = false
+
+	ActionDomains = 3 // Bump it if you increase the number of actions
+
+	DefaultMaxAttempts     = 9000
+	DefaultLearningRate    = 0.7
+	DefaultDiscount        = 1.0
+	DefaultInitialObserved = 999999
+
+	QLearningResolverType = "qlearning"
+)
+
+//. "github.com/mudler/luet/pkg/logger"
+
+// PackageResolver assists PackageSolver on unsat cases
+type PackageResolver interface {
+	Solve(bf.Formula, PackageSolver) (PackagesAssertions, error)
+}
+
+type DummyPackageResolver struct {
+}
+
+func (*DummyPackageResolver) Solve(bf.Formula, PackageSolver) (PackagesAssertions, error) {
+	return nil, errors.New("Could not satisfy the constraints. Try again by removing deps ")
+}
+
+type QLearningResolver struct {
+	Attempts int
+
+	ToAttempt int
+
+	attempts int
+
+	Attempted map[string]bool
+
+	Solver  PackageSolver
+	Formula bf.Formula
+
+	Targets pkg.Packages
+	Current pkg.Packages
+
+	observedDelta       int
+	observedDeltaChoice pkg.Packages
+
+	Agent *qlearning.SimpleAgent
+}
+
+func SimpleQLearningSolver() PackageResolver {
+	return NewQLearningResolver(DefaultLearningRate, DefaultDiscount, DefaultMaxAttempts, DefaultInitialObserved)
+}
+
+// Defaults LearningRate 0.7, Discount 1.0
+func NewQLearningResolver(LearningRate, Discount float32, MaxAttempts, initialObservedDelta int) PackageResolver {
+	return &QLearningResolver{
+		Agent:         qlearning.NewSimpleAgent(LearningRate, Discount),
+		observedDelta: initialObservedDelta,
+		Attempts:      MaxAttempts,
+	}
+}
+
+func (resolver *QLearningResolver) Solve(f bf.Formula, s PackageSolver) (PackagesAssertions, error) {
+	//	Info("Using QLearning solver to resolve conflicts. Please be patient.")
+	resolver.Solver = s
+
+	s.SetResolver(&DummyPackageResolver{}) // Set dummy. Otherwise the attempts will run again a QLearning instance.
+	defer s.SetResolver(resolver)          // Set back ourselves as resolver
+
+	resolver.Formula = f
+
+	// Our agent by default has a learning rate of 0.7 and discount of 1.0.
+	if resolver.Agent == nil {
+		resolver.Agent = qlearning.NewSimpleAgent(DefaultLearningRate, DefaultDiscount) // FIXME: Remove hardcoded values
+	}
+
+	// 3 are the action domains, counting noop regardless if enabled or not
+	// get the permutations to attempt
+	resolver.ToAttempt = int(helpers.Factorial(uint64(len(resolver.Solver.(*Solver).Wanted)-1) * ActionDomains)) // TODO: type assertions must go away
+	resolver.Targets = resolver.Solver.(*Solver).Wanted
+
+	resolver.attempts = resolver.Attempts
+
+	resolver.Attempted = make(map[string]bool, len(resolver.Targets))
+
+	for resolver.IsComplete() == Going {
+		// Pick the next move, which is going to be a letter choice.
+		action := qlearning.Next(resolver.Agent, resolver)
+
+		// Whatever that choice is, let's update our model for its
+		// impact. If the package chosen makes the formula sat,
+		// then this action will be positive. Otherwise, it will be
+		// negative.
+		resolver.Agent.Learn(action, resolver)
+
+		// Reward doesn't change state so we can check what the
+		// reward would be for this action, and report how the
+		// env changed.
+		//	score := resolver.Reward(action)
+		//	if score > 0.0 {
+		//	resolver.Log("%s was correct", action.Action.String())
+		//	} else {
+		//	resolver.Log("%s was incorrect", action.Action.String())
+		//	}
+	}
+
+	// If we get good result, take it
+	// Take the result also if we did  reached overall maximum attempts
+	if resolver.IsComplete() == Solved || resolver.IsComplete() == NoSolution {
+
+		if len(resolver.observedDeltaChoice) != 0 {
+			// Take the minimum delta observed choice result, and consume it (Try sets the wanted list)
+			resolver.Solver.(*Solver).Wanted = resolver.observedDeltaChoice
+		}
+
+		return resolver.Solver.Solve()
+	} else {
+		return nil, errors.New("QLearning resolver failed ")
+	}
+
+}
+
+// Returns the current state.
+func (resolver *QLearningResolver) IsComplete() int {
+	if resolver.attempts < 1 {
+		return NoSolution
+	}
+
+	if resolver.ToAttempt > 0 {
+		return Going
+	}
+
+	return Solved
+}
+
+func (resolver *QLearningResolver) Try(c Choice) error {
+	pack := c.Package
+	packtoAdd := pkg.FromString(pack)
+	resolver.Attempted[pack+strconv.Itoa(int(c.Action))] = true // increase the count
+	s, _ := resolver.Solver.(*Solver)
+	var filtered pkg.Packages
+
+	switch c.Action {
+	case ActionAdded:
+		found := false
+		for _, p := range s.Wanted {
+			if p.String() == pack {
+				found = true
+				break
+			}
+		}
+		if !found {
+			resolver.Solver.(*Solver).Wanted = append(resolver.Solver.(*Solver).Wanted, packtoAdd)
+		}
+
+	case ActionRemoved:
+		for _, p := range s.Wanted {
+			if p.String() != pack {
+				filtered = append(filtered, p)
+			}
+		}
+
+		resolver.Solver.(*Solver).Wanted = filtered
+	}
+
+	_, err := resolver.Solver.Solve()
+
+	return err
+}
+
+// Choose applies a pack attempt, returning
+// true if the formula returns sat.
+//
+// Choose updates the resolver's state.
+func (resolver *QLearningResolver) Choose(c Choice) bool {
+	//pack := pkg.FromString(c.Package)
+
+	err := resolver.Try(c)
+
+	if err == nil {
+		resolver.ToAttempt--
+		resolver.attempts-- // Decrease attempts - it's a barrier. We could also do not decrease it here, allowing more attempts to be made
+	} else {
+		resolver.attempts--
+		return false
+	}
+
+	return true
+}
+
+// Reward returns a score for a given qlearning.StateAction. Reward is a
+// member of the qlearning.Rewarder interface. If the choice will make sat the formula, a positive score is returned.
+// Otherwise, a static -1000 is returned.
+func (resolver *QLearningResolver) Reward(action *qlearning.StateAction) float32 {
+	choice := action.Action.(*Choice)
+
+	//_, err := resolver.Solver.Solve()
+	err := resolver.Try(*choice)
+
+	toBeInstalled := len(resolver.Solver.(*Solver).Wanted)
+	originalTarget := len(resolver.Targets)
+	noaction := choice.Action == NoAction
+	delta := originalTarget - toBeInstalled
+
+	if err == nil {
+		// if toBeInstalled == originalTarget { // Base case: all the targets matches (it shouldn't happen, but lets put a higher)
+		// 	Debug("Target match, maximum score")
+		// 	return 24.0 / float32(len(resolver.Attempted))
+
+		// }
+		if DoNoop {
+			if noaction && toBeInstalled == 0 { // We decided to stay in the current state, and no targets have been chosen
+				return -100
+			}
+		}
+
+		if delta <= resolver.observedDelta { // Try to maximise observedDelta
+			resolver.observedDelta = delta
+			resolver.observedDeltaChoice = resolver.Solver.(*Solver).Wanted // we store it as this is our return value at the end
+			return 24.0 / float32(len(resolver.Attempted))
+		} else if toBeInstalled > 0 { // If we installed something, at least give a good score
+			return 24.0 / float32(len(resolver.Attempted))
+		}
+
+	}
+
+	return -1000
+}
+
+// Next creates a new slice of qlearning.Action instances. A possible
+// action is created for each package that could be removed from the formula's target
+func (resolver *QLearningResolver) Next() []qlearning.Action {
+	actions := make([]qlearning.Action, 0, (len(resolver.Targets)-1)*3)
+
+TARGETS:
+	for _, pack := range resolver.Targets {
+		for _, current := range resolver.Solver.(*Solver).Wanted {
+			if current.String() == pack.String() {
+				actions = append(actions, &Choice{Package: pack.String(), Action: ActionRemoved})
+				continue TARGETS
+			}
+
+		}
+		actions = append(actions, &Choice{Package: pack.String(), Action: ActionAdded})
+	}
+
+	if DoNoop {
+		actions = append(actions, &Choice{Package: "", Action: NoAction}) // NOOP
+	}
+
+	return actions
+}
+
+// Log is a wrapper of fmt.Printf. If Game.debug is true, Log will print
+// to stdout.
+func (resolver *QLearningResolver) Log(msg string, args ...interface{}) {
+	logMsg := fmt.Sprintf("(%d moves, %d remaining attempts) %s\n", len(resolver.Attempted), resolver.attempts, msg)
+	fmt.Println(fmt.Sprintf(logMsg, args...))
+}
+
+// String returns a consistent hash for the current env state to be
+// used in a qlearning.Agent.
+func (resolver *QLearningResolver) String() string {
+	return fmt.Sprintf("%v", resolver.Solver.(*Solver).Wanted)
+}
+
+// Choice implements qlearning.Action for a package choice for removal from wanted targets
+type Choice struct {
+	Package string     `json:"pack"`
+	Action  ActionType `json:"action"`
+}
+
+func ChoiceFromString(s string) (*Choice, error) {
+	var p *Choice
+	err := yaml.Unmarshal([]byte(s), &p)
+	if err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+// String returns the character for the current action.
+func (choice *Choice) String() string {
+	data, err := json.Marshal(choice)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+// Apply updates the state of the solver for the package choice.
+func (choice *Choice) Apply(state qlearning.State) qlearning.State {
+	resolver := state.(*QLearningResolver)
+	resolver.Choose(*choice)
+
+	return resolver
+}

pkg/solver/resolver_test.go

@@ -0,0 +1,182 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver_test
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "github.com/mudler/luet/pkg/solver"
+)
+
+var _ = Describe("Resolver", func() {
+
+	db := pkg.NewInMemoryDatabase(false)
+	dbInstalled := pkg.NewInMemoryDatabase(false)
+	dbDefinitions := pkg.NewInMemoryDatabase(false)
+	s := NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+
+	BeforeEach(func() {
+		db = pkg.NewInMemoryDatabase(false)
+		dbInstalled = pkg.NewInMemoryDatabase(false)
+		dbDefinitions = pkg.NewInMemoryDatabase(false)
+		s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+	})
+
+	Context("Conflict set", func() {
+		Context("DummyPackageResolver", func() {
+			It("is unsolvable - as we something we ask to install conflict with system stuff", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A})
+				Expect(len(solution)).To(Equal(0))
+				Expect(err).To(HaveOccurred())
+			})
+			It("succeeds to install D and F if explictly requested", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D, E, F} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{D, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(len(solution)).To(Equal(6))
+
+				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
+
+			})
+
+		})
+		Context("QLearningResolver", func() {
+			It("will find out that we can install D by ignoring A", func() {
+				s.SetResolver(SimpleQLearningSolver())
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A, D})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+
+				Expect(len(solution)).To(Equal(4))
+			})
+
+			It("will find out that we can install D and F by ignoring E and A", func() {
+				s.SetResolver(SimpleQLearningSolver())
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D, E, F} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A, D, E, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) // Was already installed
+				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
+				Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
+				Expect(len(solution)).To(Equal(6))
+
+			})
+		})
+
+		Context("DummyPackageResolver", func() {
+			It("cannot find a solution", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{C} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{A, D})
+				Expect(err).To(HaveOccurred())
+
+				Expect(len(solution)).To(Equal(0))
+			})
+
+		})
+	})
+
+})

pkg/solver/solver.go

@@ -18,48 +18,93 @@ package solver
 import (
 
 	//. "github.com/mudler/luet/pkg/logger"
+	"fmt"
+
 	"github.com/pkg/errors"
 
 	"github.com/crillab/gophersat/bf"
 	pkg "github.com/mudler/luet/pkg/package"
 )
 
+type SolverType int
+
+const (
+	SingleCoreSimple = 0
+	ParallelSimple   = iota
+)
+
 // PackageSolver is an interface to a generic package solving algorithm
 type PackageSolver interface {
 	SetDefinitionDatabase(pkg.PackageDatabase)
-	Install(p []pkg.Package) (PackagesAssertions, error)
-	Uninstall(candidate pkg.Package) ([]pkg.Package, error)
+	Install(p pkg.Packages) (PackagesAssertions, error)
+	Uninstall(candidate pkg.Package, checkconflicts, full bool) (pkg.Packages, error)
 	ConflictsWithInstalled(p pkg.Package) (bool, error)
-	ConflictsWith(p pkg.Package, ls []pkg.Package) (bool, error)
-	World() []pkg.Package
-	Upgrade() ([]pkg.Package, PackagesAssertions, error)
+	ConflictsWith(p pkg.Package, ls pkg.Packages) (bool, error)
+	Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error)
+
+	World() pkg.Packages
+	Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAssertions, error)
+
+	UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssertions, error)
+	UninstallUniverse(toremove pkg.Packages) (pkg.Packages, error)
+
+	SetResolver(PackageResolver)
+
+	Solve() (PackagesAssertions, error)
 }
 
 // Solver is the default solver for luet
 type Solver struct {
 	DefinitionDatabase pkg.PackageDatabase
 	SolverDatabase     pkg.PackageDatabase
-	Wanted             []pkg.Package
+	Wanted             pkg.Packages
 	InstalledDatabase  pkg.PackageDatabase
+
+	Resolver PackageResolver
+}
+
+type Options struct {
+	Type        SolverType
+	Concurrency int
 }
 
 // NewSolver accepts as argument two lists of packages, the first is the initial set,
 // the second represent all the known packages.
-func NewSolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase) PackageSolver {
-	return &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb}
+func NewSolver(t Options, installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase) PackageSolver {
+	return NewResolver(t, installed, definitiondb, solverdb, &DummyPackageResolver{})
 }
 
-// SetWorld is a setter for the list of all known packages to the solver
+// NewResolver accepts as argument two lists of packages, the first is the initial set,
+// the second represent all the known packages.
+// Using constructors as in the future we foresee warmups for hot-restore solver cache
+func NewResolver(t Options, installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase, re PackageResolver) PackageSolver {
+	var s PackageSolver
+	switch t.Type {
+	case SingleCoreSimple:
+		s = &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb, Resolver: re}
+	case ParallelSimple:
+		s = &Parallel{InstalledDatabase: installed, DefinitionDatabase: definitiondb, ParallelDatabase: solverdb, Resolver: re, Concurrency: t.Concurrency}
+	}
+
+	return s
+}
+
+// SetDefinitionDatabase is a setter for the definition Database
 
 func (s *Solver) SetDefinitionDatabase(db pkg.PackageDatabase) {
 	s.DefinitionDatabase = db
 }
 
-func (s *Solver) World() []pkg.Package {
+// SetResolver is a setter for the unsat resolver backend
+func (s *Solver) SetResolver(r PackageResolver) {
+	s.Resolver = r
+}
+
+func (s *Solver) World() pkg.Packages {
 	return s.DefinitionDatabase.World()
 }
 
-func (s *Solver) Installed() []pkg.Package {
+func (s *Solver) Installed() pkg.Packages {
 
 	return s.InstalledDatabase.World()
 }
@@ -74,6 +119,16 @@ func (s *Solver) noRulesWorld() bool {
 	return true
 }
 
+func (s *Solver) noRulesInstalled() bool {
+	for _, p := range s.Installed() {
+		if len(p.GetConflicts()) != 0 || len(p.GetRequires()) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
 func (s *Solver) BuildInstalled() (bf.Formula, error) {
 	var formulas []bf.Formula
 	for _, p := range s.Installed() {
@@ -113,8 +168,8 @@ func (s *Solver) BuildWorld(includeInstalled bool) (bf.Formula, error) {
 	return bf.And(formulas...), nil
 }
 
-func (s *Solver) getList(db pkg.PackageDatabase, lsp []pkg.Package) ([]pkg.Package, error) {
-	var ls []pkg.Package
+func (s *Solver) getList(db pkg.PackageDatabase, lsp pkg.Packages) (pkg.Packages, error) {
+	var ls pkg.Packages
 
 	for _, pp := range lsp {
 		cp, err := db.FindPackage(pp)
@@ -124,7 +179,7 @@ func (s *Solver) getList(db pkg.PackageDatabase, lsp []pkg.Package) ([]pkg.Packa
 			if err != nil || len(packages) == 0 {
 				cp = pp
 			} else {
-				cp = pkg.Best(packages)
+				cp = packages.Best(nil)
 			}
 		}
 		ls = append(ls, cp)
@@ -132,7 +187,44 @@ func (s *Solver) getList(db pkg.PackageDatabase, lsp []pkg.Package) ([]pkg.Packa
 	return ls, nil
 }
 
-func (s *Solver) ConflictsWith(pack pkg.Package, lsp []pkg.Package) (bool, error) {
+// Conflicts acts like ConflictsWith, but uses package's reverse dependencies to
+// determine if it conflicts with the given set
+func (s *Solver) Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error) {
+	p, err := s.DefinitionDatabase.FindPackage(pack)
+	if err != nil {
+		p = pack
+	}
+
+	ls, err := s.getList(s.DefinitionDatabase, lsp)
+	if err != nil {
+		return false, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	if s.noRulesWorld() {
+		return false, nil
+	}
+
+	temporarySet := pkg.NewInMemoryDatabase(false)
+	for _, p := range ls {
+		temporarySet.CreatePackage(p)
+	}
+	visited := make(map[string]interface{})
+	revdeps := p.ExpandedRevdeps(temporarySet, visited)
+
+	var revdepsErr error
+	for _, r := range revdeps {
+		if revdepsErr == nil {
+			revdepsErr = errors.New("")
+		}
+		revdepsErr = errors.New(fmt.Sprintf("%s\n%s", revdepsErr.Error(), r.HumanReadableString()))
+	}
+
+	return len(revdeps) != 0, revdepsErr
+}
+
+// ConflictsWith return true if a package is part of the requirement set of a list of package
+// return false otherwise (and thus it is NOT relevant to the given list)
+func (s *Solver) ConflictsWith(pack pkg.Package, lsp pkg.Packages) (bool, error) {
 	p, err := s.DefinitionDatabase.FindPackage(pack)
 	if err != nil {
 		p = pack //Relax search, otherwise we cannot compute solutions for packages not in definitions
@@ -193,14 +285,171 @@ func (s *Solver) ConflictsWithInstalled(p pkg.Package) (bool, error) {
 	return s.ConflictsWith(p, s.Installed())
 }
 
-func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
+// UninstallUniverse takes a list of candidate package and return a list of packages that would be removed
+// in order to purge the candidate. Uses the solver to check constraints and nothing else
+//
+// It can be compared to the counterpart Uninstall as this method acts like a uninstall --full
+// it removes all the packages and its deps. taking also in consideration other packages that might have
+// revdeps
+func (s *Solver) UninstallUniverse(toremove pkg.Packages) (pkg.Packages, error) {
+
+	if s.noRulesInstalled() {
+		return s.getList(s.InstalledDatabase, toremove)
+	}
+
+	// resolve to packages from the db
+	toRemove, err := s.getList(s.InstalledDatabase, toremove)
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	var formulas []bf.Formula
+	r, err := s.BuildInstalled()
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	// SAT encode the clauses against the world
+	for _, p := range toRemove.Unique() {
+		encodedP, err := p.Encode(s.InstalledDatabase)
+		if err != nil {
+			return nil, errors.Wrap(err, "Package not found in definition db")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(bf.Not(P), r))
+	}
+
+	markedForRemoval := pkg.Packages{}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, errors.New("Failed finding a solution")
+	}
+	assertion, err := DecodeModel(model, s.InstalledDatabase)
+	if err != nil {
+		return nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+	}
+	return markedForRemoval, nil
+}
+
+// UpgradeUniverse mark packages for removal and returns a solution. It considers
+// the Universe db as authoritative
+// See also on the subject: https://arxiv.org/pdf/1007.1021.pdf
+func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssertions, error) {
+	// we first figure out which aren't up-to-date
+	// which has to be removed
+	// and which needs to be upgraded
+	notUptodate := pkg.Packages{}
+	removed := pkg.Packages{}
+	toUpgrade := pkg.Packages{}
+
+	// TODO: this is memory expensive, we need to optimize this
+	universe := pkg.NewInMemoryDatabase(false)
+	for _, p := range s.DefinitionDatabase.World() {
+		universe.CreatePackage(p)
+	}
+	for _, p := range s.Installed() {
+		universe.CreatePackage(p)
+	}
+
+	// Grab all the installed ones, see if they are eligible for update
+	for _, p := range s.Installed() {
+		available, err := universe.FindPackageVersions(p)
+		if err != nil {
+			removed = append(removed, p)
+		}
+		if len(available) == 0 {
+			continue
+		}
+
+		bestmatch := available.Best(nil)
+		// Found a better version available
+		if !bestmatch.Matches(p) {
+			notUptodate = append(notUptodate, p)
+			toUpgrade = append(toUpgrade, bestmatch)
+		}
+	}
+
+	// resolve to packages from the db to be able to encode correctly
+	oldPackages, err := s.getList(universe, notUptodate)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't get package marked for removal from universe")
+	}
+
+	updates, err := s.getList(universe, toUpgrade)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't get package marked for update from universe")
+	}
+
+	var formulas []bf.Formula
+
+	// Build constraints for the whole defdb
+	r, err := s.BuildWorld(true)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't build world constraints")
+	}
+
+	// Treat removed packages from universe as marked for deletion
+	if dropremoved {
+		oldPackages = append(oldPackages, removed...)
+	}
+
+	// SAT encode the clauses against the world
+	for _, p := range oldPackages.Unique() {
+		encodedP, err := p.Encode(universe)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "couldn't encode package")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(bf.Not(P), r))
+	}
+
+	for _, p := range updates {
+		encodedP, err := p.Encode(universe)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "couldn't encode package")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(P, r))
+	}
+
+	markedForRemoval := pkg.Packages{}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, nil, errors.New("Failed finding a solution")
+	}
+
+	assertion, err := DecodeModel(model, universe)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+
+	}
+	return markedForRemoval, assertion, nil
+}
+
+func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAssertions, error) {
 
 	// First get candidates that needs to be upgraded..
 
-	toUninstall := []pkg.Package{}
-	toInstall := []pkg.Package{}
+	toUninstall := pkg.Packages{}
+	toInstall := pkg.Packages{}
 
-	availableCache := map[string][]pkg.Package{}
+	availableCache := map[string]pkg.Packages{}
 	for _, p := range s.DefinitionDatabase.World() {
 		// Each one, should be expanded
 		availableCache[p.GetName()+p.GetCategory()] = append(availableCache[p.GetName()+p.GetCategory()], p)
@@ -212,7 +461,7 @@ func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
 		installedcopy.CreatePackage(p)
 		packages, ok := availableCache[p.GetName()+p.GetCategory()]
 		if ok && len(packages) != 0 {
-			best := pkg.Best(packages)
+			best := packages.Best(nil)
 			if best.GetVersion() != p.GetVersion() {
 				toUninstall = append(toUninstall, p)
 				toInstall = append(toInstall, best)
@@ -220,10 +469,18 @@ func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
 		}
 	}
 
-	s2 := NewSolver(installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+	s2 := NewSolver(Options{Type: SingleCoreSimple}, installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+	s2.SetResolver(s.Resolver)
+	if !full {
+		ass := PackagesAssertions{}
+		for _, i := range toInstall {
+			ass = append(ass, PackageAssert{Package: i.(*pkg.DefaultPackage), Value: true})
+		}
+	}
+
 	// Then try to uninstall the versions in the system, and store that tree
 	for _, p := range toUninstall {
-		r, err := s.Uninstall(p)
+		r, err := s.Uninstall(p, checkconflicts, false)
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't uninstall selected candidate "+p.GetFingerPrint())
 		}
@@ -233,8 +490,8 @@ func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
 				return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't remove copy of package targetted for removal")
 			}
 		}
-
 	}
+
 	r, e := s2.Install(toInstall)
 	return toUninstall, r, e
 	// To that tree, ask to install the versions that should be upgraded, and try to solve
@@ -244,8 +501,8 @@ func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
 
 // Uninstall takes a candidate package and return a list of packages that would be removed
 // in order to purge the candidate. Returns error if unsat.
-func (s *Solver) Uninstall(c pkg.Package) ([]pkg.Package, error) {
-	var res []pkg.Package
+func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packages, error) {
+	var res pkg.Packages
 	candidate, err := s.InstalledDatabase.FindPackage(c)
 	if err != nil {
 
@@ -255,13 +512,23 @@ func (s *Solver) Uninstall(c pkg.Package) ([]pkg.Package, error) {
 		if err != nil || len(packages) == 0 {
 			candidate = c
 		} else {
-			candidate = pkg.Best(packages)
+			candidate = packages.Best(nil)
 		}
 		//Relax search, otherwise we cannot compute solutions for packages not in definitions
 		//	return nil, errors.Wrap(err, "Package not found between installed")
 	}
 	// Build a fake "Installed" - Candidate and its requires tree
-	var InstalledMinusCandidate []pkg.Package
+	var InstalledMinusCandidate pkg.Packages
+
+	// We are asked to not perform a full uninstall (checking all the possible requires that could
+	// be removed). Let's only check if we can remove the selected package
+	if !full && checkconflicts {
+		if conflicts, err := s.Conflicts(candidate, s.Installed()); conflicts {
+			return nil, err
+		} else {
+			return pkg.Packages{candidate}, nil
+		}
+	}
 
 	// TODO: Can be optimized
 	for _, i := range s.Installed() {
@@ -276,17 +543,19 @@ func (s *Solver) Uninstall(c pkg.Package) ([]pkg.Package, error) {
 		}
 	}
 
+	s2 := NewSolver(Options{Type: SingleCoreSimple}, pkg.NewInMemoryDatabase(false), s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+	s2.SetResolver(s.Resolver)
 	// Get the requirements to install the candidate
-	saved := s.InstalledDatabase
-	s.InstalledDatabase = pkg.NewInMemoryDatabase(false)
-	asserts, err := s.Install([]pkg.Package{candidate})
+	asserts, err := s2.Install(pkg.Packages{candidate})
 	if err != nil {
 		return nil, err
 	}
-	s.InstalledDatabase = saved
-
 	for _, a := range asserts {
 		if a.Value {
+			if !checkconflicts {
+				res = append(res, a.Package)
+				continue
+			}
 
 			c, err := s.ConflictsWithInstalled(a.Package)
 			if err != nil {
@@ -295,7 +564,7 @@ func (s *Solver) Uninstall(c pkg.Package) ([]pkg.Package, error) {
 
 			// If doesn't conflict with installed we just consider it for removal and look for the next one
 			if !c {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 				continue
 			}
 
@@ -305,7 +574,7 @@ func (s *Solver) Uninstall(c pkg.Package) ([]pkg.Package, error) {
 				return nil, err
 			}
 			if !c {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 			}
 
 		}
@@ -361,13 +630,20 @@ func (s *Solver) solve(f bf.Formula) (map[string]bool, bf.Formula, error) {
 
 // Solve builds the formula given the current state and returns package assertions
 func (s *Solver) Solve() (PackagesAssertions, error) {
+	var model map[string]bool
+	var err error
+
 	f, err := s.BuildFormula()
 
 	if err != nil {
 		return nil, err
 	}
 
-	model, _, err := s.solve(f)
+	model, _, err = s.solve(f)
+	if err != nil && s.Resolver != nil {
+		return s.Resolver.Solve(f, s)
+	}
+
 	if err != nil {
 		return nil, err
 	}
@@ -377,7 +653,7 @@ func (s *Solver) Solve() (PackagesAssertions, error) {
 
 // Install given a list of packages, returns package assertions to indicate the packages that must be installed in the system in order
 // to statisfy all the constraints
-func (s *Solver) Install(c []pkg.Package) (PackagesAssertions, error) {
+func (s *Solver) Install(c pkg.Packages) (PackagesAssertions, error) {
 
 	coll, err := s.getList(s.DefinitionDatabase, c)
 	if err != nil {

pkg/solver/solver_test.go

@@ -28,13 +28,13 @@ var _ = Describe("Solver", func() {
 	db := pkg.NewInMemoryDatabase(false)
 	dbInstalled := pkg.NewInMemoryDatabase(false)
 	dbDefinitions := pkg.NewInMemoryDatabase(false)
-	s := NewSolver(dbInstalled, dbDefinitions, db)
+	s := NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 	BeforeEach(func() {
 		db = pkg.NewInMemoryDatabase(false)
 		dbInstalled = pkg.NewInMemoryDatabase(false)
 		dbDefinitions = pkg.NewInMemoryDatabase(false)
-		s = NewSolver(dbInstalled, dbDefinitions, db)
+		s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 	})
 	Context("Simple set", func() {
 		It("Solves correctly if the selected package has no requirements or conflicts and we have nothing installed yet", func() {
@@ -52,7 +52,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
@@ -75,7 +75,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{B})
 			Expect(err).ToNot(HaveOccurred())
@@ -101,7 +101,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
@@ -130,7 +130,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
@@ -156,7 +156,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
@@ -181,7 +181,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -209,7 +209,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -236,7 +236,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -263,7 +263,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{C})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -291,7 +291,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{&pkg.DefaultPackage{Name: "c", Version: ">1.0", Category: "test"}})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -317,7 +317,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -345,7 +345,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A, B})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -360,15 +360,67 @@ var _ = Describe("Solver", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 
+		It("Install only package requires", func() {
+
+			E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			C := pkg.NewPackage("C", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{
+				Name:    "A",
+				Version: ">=1.0",
+			}}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("D", "1.9", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B := pkg.NewPackage("B", "1.1", []*pkg.DefaultPackage{
+				&pkg.DefaultPackage{
+					Name:    "D",
+					Version: ">=0",
+				},
+			}, []*pkg.DefaultPackage{})
+
+			A := pkg.NewPackage("A", "1.2", []*pkg.DefaultPackage{
+				&pkg.DefaultPackage{
+					Name:    "D",
+					Version: ">=1.0",
+				},
+			}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{A, B, C, D, E} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+
+			solution, err := s.Install([]pkg.Package{C})
+			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: false}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
+
+			Expect(len(solution)).To(Equal(4))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
 		It("Selects best version", func() {
 
 			E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			E.SetCategory("test")
 			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
 			D2 := pkg.NewPackage("D", "1.9", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D2.SetCategory("test")
 			D := pkg.NewPackage("D", "1.8", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D.SetCategory("test")
 			D1 := pkg.NewPackage("D", "1.4", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+			D1.SetCategory("test")
+			B := pkg.NewPackage("B", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
 
 			for _, p := range []pkg.Package{A, B, C, D, D1, D2, E} {
 				_, err := dbDefinitions.CreatePackage(p)
@@ -379,7 +431,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A, B})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -397,16 +449,23 @@ var _ = Describe("Solver", func() {
 		It("Support provides", func() {
 
 			E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-
+			E.SetCategory("test")
 			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
 			D2 := pkg.NewPackage("D", "1.9", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D2.SetCategory("test")
 			D := pkg.NewPackage("D", "1.8", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D.SetCategory("test")
 			D1 := pkg.NewPackage("D", "1.4", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0"}}, []*pkg.DefaultPackage{})
+			D1.SetCategory("test")
+			B := pkg.NewPackage("B", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "D", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
 
-			D2.SetProvides([]*pkg.DefaultPackage{{Name: "E"}})
-			A2 := pkg.NewPackage("A", "1.3", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "E", Version: ""}}, []*pkg.DefaultPackage{})
+			D2.SetProvides([]*pkg.DefaultPackage{{Name: "E", Category: "test"}})
+			A2 := pkg.NewPackage("A", "1.3", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "E", Version: "", Category: "test"}}, []*pkg.DefaultPackage{})
+			A2.SetCategory("test")
 
 			for _, p := range []pkg.Package{A, B, C, D, D1, D2, A2, E} {
 				_, err := dbDefinitions.CreatePackage(p)
@@ -417,7 +476,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A2, B})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
@@ -429,7 +488,7 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(5))
+			Expect(len(solution)).To(Equal(6))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -455,7 +514,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A2})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
@@ -470,7 +529,7 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(4))
+			Expect(len(solution)).To(Equal(6))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -496,7 +555,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A2})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
@@ -511,61 +570,370 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(4))
+			Expect(len(solution)).To(Equal(6))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
-		It("Uninstalls simple package correctly", func() {
+		Context("Uninstall", func() {
+			It("Uninstalls simple package correctly", func() {
 
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+
+				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("Uninstalls simple package expanded correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1.2", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+
+				solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"}, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
 
-			solution, err := s.Uninstall(A)
-			Expect(err).ToNot(HaveOccurred())
+				Expect(solution).To(ContainElement(A))
 
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("Uninstalls simple packages not in world correctly", func() {
 
-			//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
-			Expect(len(solution)).To(Equal(1))
-		})
-		It("Uninstalls simple package expanded correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "1.2", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("Uninstalls complex packages not in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
 
-			solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"})
-			Expect(err).ToNot(HaveOccurred())
+				Expect(solution).To(ContainElement(A))
 
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("Uninstalls complex packages correctly, even if shared deps are required by system packages", func() {
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.Uninstall(A, true, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).ToNot(ContainElement(B))
+
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("Uninstalls complex packages in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{C}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Uninstall(A, true, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(C))
+
+				Expect(len(solution)).To(Equal(2))
+			})
+
+			It("Uninstalls complex package correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				//	C // installed
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Uninstall(A, true, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(D))
+
+				Expect(len(solution)).To(Equal(3))
+
+			})
+
+			It("UninstallUniverse simple package correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("UninstallUniverse simple package expanded correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1.2", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+
+				solution, err := s.UninstallUniverse(pkg.Packages{
+					&pkg.DefaultPackage{Name: "A", Version: ">1.0"}})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("UninstallUniverse simple packages not in world correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("UninstallUniverse complex packages not in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+
+				Expect(len(solution)).To(Equal(2))
+			})
+
+			It("UninstallUniverse complex packages correctly, even if shared deps are required by system packages", func() {
+				// Here we diff a lot from standard Uninstall:
+				// all the packages that has reverse deps will be removed (aka --full)
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(C))
+
+				Expect(len(solution)).To(Equal(3))
+			})
+
+			It("UninstallUniverse complex packages in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{C}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(C))
+
+				Expect(len(solution)).To(Equal(2))
+			})
+
+			It("UninstallUniverse complex package correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				//	C // installed
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(D))
+
+				Expect(len(solution)).To(Equal(3))
+
+			})
 
-			//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
-			Expect(len(solution)).To(Equal(1))
 		})
 		It("Find conflicts", func() {
 
@@ -658,14 +1026,16 @@ var _ = Describe("Solver", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val).ToNot(BeTrue())
 		})
-		It("Uninstalls simple packages not in world correctly", func() {
+
+		It("Find conflicts using revdeps", func() {
 
 			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 
-			for _, p := range []pkg.Package{B, C, D} {
+			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{A}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{A, B, C, D} {
 				_, err := dbDefinitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
@@ -674,43 +1044,20 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			solution, err := s.Uninstall(A)
-			Expect(err).ToNot(HaveOccurred())
 
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
+			val, err := s.Conflicts(A, dbInstalled.World())
+			Expect(err.Error()).To(Equal("\n/B-"))
+			Expect(val).To(BeTrue())
 
-			//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
-			Expect(len(solution)).To(Equal(1))
 		})
 
-		It("Uninstalls complex packages not in world correctly", func() {
+		It("Find nested conflicts with revdeps", func() {
+
 			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
 
-			for _, p := range []pkg.Package{B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-			solution, err := s.Uninstall(A)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(1))
-		})
-
-		It("Uninstalls complex packages correctly, even if shared deps are required by system packages", func() {
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{A}, []*pkg.DefaultPackage{})
 
 			for _, p := range []pkg.Package{A, B, C, D} {
 				_, err := dbDefinitions.CreatePackage(p)
@@ -721,46 +1068,42 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			solution, err := s.Uninstall(A)
-			Expect(err).ToNot(HaveOccurred())
 
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-			Expect(solution).ToNot(ContainElement(B.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(1))
+			val, err := s.Conflicts(D, dbInstalled.World())
+			Expect(err.Error()).To(Equal("\n/A-\n/B-"))
+			Expect(val).To(BeTrue())
 		})
 
-		It("Uninstalls complex packages in world correctly", func() {
+		It("Doesn't find nested conflicts with revdeps", func() {
+
 			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{C}, []*pkg.DefaultPackage{})
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+
+			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{A}, []*pkg.DefaultPackage{})
 
 			for _, p := range []pkg.Package{A, B, C, D} {
 				_, err := dbDefinitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
 
-			for _, p := range []pkg.Package{A, C, D} {
+			for _, p := range []pkg.Package{A, B, C, D} {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
 
-			solution, err := s.Uninstall(A)
+			val, err := s.Conflicts(C, dbInstalled.World())
 			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-			Expect(solution).To(ContainElement(C.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(2))
+			Expect(val).ToNot(BeTrue())
 		})
 
-		It("Uninstalls complex package correctly", func() {
+		It("Doesn't find conflicts with revdeps", func() {
+
 			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
-			//	C // installed
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 
 			for _, p := range []pkg.Package{A, B, C, D} {
 				_, err := dbDefinitions.CreatePackage(p)
@@ -771,16 +1114,9 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-
-			solution, err := s.Uninstall(A)
+			val, err := s.Conflicts(C, dbInstalled.World())
 			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-			Expect(solution).To(ContainElement(B.IsFlagged(false)))
-			Expect(solution).To(ContainElement(D.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(3))
-
+			Expect(val).ToNot(BeTrue())
 		})
 
 	})
@@ -868,7 +1204,7 @@ var _ = Describe("Solver", func() {
 			Expect(lst).To(ContainElement(a03))
 			Expect(lst).ToNot(ContainElement(old))
 			Expect(len(lst)).To(Equal(5))
-			p := pkg.Best(lst)
+			p := lst.Best(nil)
 			Expect(p).To(Equal(a03))
 		})
 	})
@@ -893,7 +1229,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			uninstall, solution, err := s.Upgrade()
+			uninstall, solution, err := s.Upgrade(true, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(uninstall)).To(Equal(1))
@@ -906,5 +1242,30 @@ var _ = Describe("Solver", func() {
 			Expect(len(solution)).To(Equal(3))
 
 		})
+
+		It("UpgradeUniverse upgrades correctly", func() {
+			for _, p := range []pkg.Package{A1, B, C} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.UpgradeUniverse(true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(1))
+			Expect(uninstall[0].GetName()).To(Equal("a"))
+			Expect(uninstall[0].GetVersion()).To(Equal("1.1"))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+
+			Expect(len(solution)).To(Equal(4))
+		})
 	})
 })

pkg/spectooling/definition.go

@@ -0,0 +1,112 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>,
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package spectooling
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"gopkg.in/yaml.v2"
+)
+
+type DefaultPackageSanitized struct {
+	Name             string                     `json:"name" yaml:"name"`
+	Version          string                     `json:"version" yaml:"version"`
+	Category         string                     `json:"category" yaml:"category"`
+	UseFlags         []string                   `json:"use_flags,omitempty" yaml:"use_flags,omitempty"`
+	PackageRequires  []*DefaultPackageSanitized `json:"requires,omitempty" yaml:"requires,omitempty"`
+	PackageConflicts []*DefaultPackageSanitized `json:"conflicts,omitempty" yaml:"conflicts,omitempty"`
+	Provides         []*DefaultPackageSanitized `json:"provides,omitempty" yaml:"provides,omitempty"`
+
+	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
+
+	// Path is set only internally when tree is loaded from disk
+	Path string `json:"path,omitempty" yaml:"path,omitempty"`
+
+	Description string   `json:"description,omitempty" yaml:"description,omitempty"`
+	Uri         []string `json:"uri,omitempty" yaml:"uri,omitempty"`
+	License     string   `json:"license,omitempty" yaml:"license,omitempty"`
+	Hidden      bool     `json:"hidden,omitempty" yaml:"hidden,omitempty"`
+
+	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
+}
+
+func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
+	ans := &DefaultPackageSanitized{
+		Name:        p.GetName(),
+		Version:     p.GetVersion(),
+		Category:    p.GetCategory(),
+		UseFlags:    p.GetUses(),
+		Hidden:      p.IsHidden(),
+		Path:        p.GetPath(),
+		Description: p.GetDescription(),
+		Uri:         p.GetURI(),
+		License:     p.GetLicense(),
+		Labels:      p.GetLabels(),
+		Annotations: p.GetAnnotations(),
+	}
+
+	if p.GetRequires() != nil && len(p.GetRequires()) > 0 {
+		ans.PackageRequires = []*DefaultPackageSanitized{}
+		for _, r := range p.GetRequires() {
+			// I avoid recursive call of NewDefaultPackageSanitized
+			ans.PackageRequires = append(ans.PackageRequires,
+				&DefaultPackageSanitized{
+					Name:     r.Name,
+					Version:  r.Version,
+					Category: r.Category,
+					Hidden:   r.IsHidden(),
+				},
+			)
+		}
+	}
+
+	if p.GetConflicts() != nil && len(p.GetConflicts()) > 0 {
+		ans.PackageConflicts = []*DefaultPackageSanitized{}
+		for _, c := range p.GetConflicts() {
+			// I avoid recursive call of NewDefaultPackageSanitized
+			ans.PackageConflicts = append(ans.PackageConflicts,
+				&DefaultPackageSanitized{
+					Name:     c.Name,
+					Version:  c.Version,
+					Category: c.Category,
+					Hidden:   c.IsHidden(),
+				},
+			)
+		}
+	}
+
+	if p.GetProvides() != nil && len(p.GetProvides()) > 0 {
+		ans.Provides = []*DefaultPackageSanitized{}
+		for _, prov := range p.GetProvides() {
+			// I avoid recursive call of NewDefaultPackageSanitized
+			ans.Provides = append(ans.Provides,
+				&DefaultPackageSanitized{
+					Name:     prov.Name,
+					Version:  prov.Version,
+					Category: prov.Category,
+					Hidden:   prov.IsHidden(),
+				},
+			)
+		}
+	}
+
+	return ans
+}
+
+func (p *DefaultPackageSanitized) Yaml() ([]byte, error) {
+	return yaml.Marshal(p)
+}

pkg/spectooling/package_test.go

@@ -0,0 +1,87 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>,
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package spectooling_test
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+	. "github.com/mudler/luet/pkg/spectooling"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Spec Tooling", func() {
+	Context("Conversion1", func() {
+
+		b := pkg.NewPackage("B", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		c := pkg.NewPackage("C", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		d := pkg.NewPackage("D", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		p1 := pkg.NewPackage("A", "1.0", []*pkg.DefaultPackage{b, c}, []*pkg.DefaultPackage{d})
+		virtual := pkg.NewPackage("E", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		virtual.SetCategory("virtual")
+		p1.Provides = []*pkg.DefaultPackage{virtual}
+		p1.AddLabel("label1", "value1")
+		p1.AddLabel("label2", "value2")
+		p1.SetDescription("Package1")
+		p1.SetCategory("cat1")
+		p1.SetLicense("GPL")
+		p1.AddURI("https://github.com/mudler/luet")
+		p1.AddUse("systemd")
+		It("Convert pkg1", func() {
+			res := NewDefaultPackageSanitized(p1)
+			expected_res := &DefaultPackageSanitized{
+				Name:     "A",
+				Version:  "1.0",
+				Category: "cat1",
+				PackageRequires: []*DefaultPackageSanitized{
+					&DefaultPackageSanitized{
+						Name:    "B",
+						Version: "1.0",
+					},
+					&DefaultPackageSanitized{
+						Name:    "C",
+						Version: "1.0",
+					},
+				},
+				PackageConflicts: []*DefaultPackageSanitized{
+					&DefaultPackageSanitized{
+						Name:    "D",
+						Version: "1.0",
+					},
+				},
+				Provides: []*DefaultPackageSanitized{
+					&DefaultPackageSanitized{
+						Name:     "E",
+						Category: "virtual",
+						Version:  "1.0",
+					},
+				},
+				Labels: map[string]string{
+					"label1": "value1",
+					"label2": "value2",
+				},
+				Description: "Package1",
+				License:     "GPL",
+				Uri:         []string{"https://github.com/mudler/luet"},
+				UseFlags:    []string{"systemd"},
+			}
+
+			Expect(res).To(Equal(expected_res))
+		})
+
+	})
+})

pkg/spectooling/spectooling_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>,
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package spectooling_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Spec Tooling Suite")
+}

pkg/tree/builder.go

@@ -26,5 +26,5 @@ type Builder interface {
 	GetDatabase() pkg.PackageDatabase
 	WithDatabase(d pkg.PackageDatabase)
 
-	GetSourcePath() string
+	GetSourcePath() []string
 }

pkg/tree/builder/gentoo/gentoo.go

@@ -50,7 +50,7 @@ type GentooBuilder struct {
 }
 
 type EbuildParser interface {
-	ScanEbuild(string) ([]pkg.Package, error)
+	ScanEbuild(string) (pkg.Packages, error)
 }
 
 func (gb *GentooBuilder) scanEbuild(path string, db pkg.PackageDatabase) error {

pkg/tree/builder/gentoo/gentoo_test.go

@@ -27,8 +27,8 @@ import (
 type FakeParser struct {
 }
 
-func (f *FakeParser) ScanEbuild(path string) ([]pkg.Package, error) {
-	return []pkg.Package{&pkg.DefaultPackage{Name: path}}, nil
+func (f *FakeParser) ScanEbuild(path string) (pkg.Packages, error) {
+	return pkg.Packages{&pkg.DefaultPackage{Name: path}}, nil
 }
 
 var _ = Describe("GentooBuilder", func() {

pkg/tree/builder/gentoo/simpleparser.go

@@ -323,7 +323,7 @@ func SourceFile(ctx context.Context, path string, pkg *_gentoo.GentooPackage) (m
 }
 
 // ScanEbuild returns a list of packages (always one with SimpleEbuildParser) decoded from an ebuild.
-func (ep *SimpleEbuildParser) ScanEbuild(path string) ([]pkg.Package, error) {
+func (ep *SimpleEbuildParser) ScanEbuild(path string) (pkg.Packages, error) {
 	Debug("Starting parsing of ebuild", path)
 
 	pkgstr := filepath.Base(path)
@@ -332,7 +332,7 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) ([]pkg.Package, error) {
 
 	gp, err := _gentoo.ParsePackageStr(pkgstr)
 	if err != nil {
-		return []pkg.Package{}, errors.New("Error on parsing package string")
+		return pkg.Packages{}, errors.New("Error on parsing package string")
 	}
 
 	pack := &pkg.DefaultPackage{
@@ -350,7 +350,13 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) ([]pkg.Package, error) {
 	vars, err := SourceFile(timeout, path, gp)
 	if err != nil {
 		Error("Error on source file ", pack.Name, ": ", err)
-		return []pkg.Package{}, err
+		return pkg.Packages{}, err
+	}
+
+	// Retrieve slot
+	slot, ok := vars["SLOT"]
+	if ok && slot.String() != "0" {
+		pack.SetCategory(fmt.Sprintf("%s-%s", gp.Category, slot.String()))
 	}
 
 	// TODO: Handle this a bit better
@@ -405,8 +411,8 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) ([]pkg.Package, error) {
 		gRDEPEND, err := ParseRDEPEND(rdepend.String())
 		if err != nil {
 			Warning("Error on parsing RDEPEND for package ", pack.Category+"/"+pack.Name, err)
-			return []pkg.Package{pack}, nil
-			//	return []pkg.Package{}, err
+			return pkg.Packages{pack}, nil
+			//	return pkg.Packages{}, err
 		}
 
 		pack.PackageConflicts = []*pkg.DefaultPackage{}
@@ -417,6 +423,7 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) ([]pkg.Package, error) {
 		for _, d := range gRDEPEND.GetDependencies() {
 
 			//TODO: Resolve to db or create a new one.
+			//TODO: handle SLOT too.
 			dep := &pkg.DefaultPackage{
 				Name:     d.Dep.Name,
 				Version:  d.Dep.Version + d.Dep.VersionSuffix,
@@ -436,5 +443,5 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) ([]pkg.Package, error) {
 	Debug("Finished processing ebuild", path, "deps ", len(pack.PackageRequires))
 
 	//TODO: Deps and conflicts
-	return []pkg.Package{pack}, nil
+	return pkg.Packages{pack}, nil
 }

pkg/tree/compiler_recipe.go

@@ -38,6 +38,20 @@ func NewCompilerRecipe(d pkg.PackageDatabase) Builder {
 	return &CompilerRecipe{Recipe: Recipe{Database: d}}
 }
 
+func ReadDefinitionFile(path string) (pkg.DefaultPackage, error) {
+	empty := pkg.DefaultPackage{}
+	dat, err := ioutil.ReadFile(path)
+	if err != nil {
+		return empty, errors.Wrap(err, "Error reading file "+path)
+	}
+	pack, err := pkg.DefaultPackageFromYaml(dat)
+	if err != nil {
+		return empty, errors.Wrap(err, "Error reading yaml "+path)
+	}
+
+	return pack, nil
+}
+
 // Recipe is the "general" reciper for Trees
 type CompilerRecipe struct {
 	Recipe
@@ -45,7 +59,7 @@ type CompilerRecipe struct {
 
 func (r *CompilerRecipe) Load(path string) error {
 
-	r.SourcePath = path
+	r.SourcePath = append(r.SourcePath, path)
 	//tmpfile, err := ioutil.TempFile("", "luet")
 	//if err != nil {
 	//	return err
@@ -56,17 +70,17 @@ func (r *CompilerRecipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
+		if err != nil {
+			return errors.Wrap(err, "Error on walk path "+currentpath)
+		}
+
 		if info.Name() != DefinitionFile {
 			return nil // Skip with no errors
 		}
 
-		dat, err := ioutil.ReadFile(currentpath)
+		pack, err := ReadDefinitionFile(currentpath)
 		if err != nil {
-			return errors.Wrap(err, "Error reading file "+currentpath)
-		}
-		pack, err := pkg.DefaultPackageFromYaml(dat)
-		if err != nil {
-			return errors.Wrap(err, "Error reading yaml "+currentpath)
+			return err
 		}
 		// Path is set only internally when tree is loaded from disk
 		pack.SetPath(filepath.Dir(currentpath))
@@ -74,13 +88,17 @@ func (r *CompilerRecipe) Load(path string) error {
 		// Instead of rdeps, have a different tree for build deps.
 		compileDefPath := pack.Rel(CompilerDefinitionFile)
 		if helpers.Exists(compileDefPath) {
-			dat, err = ioutil.ReadFile(compileDefPath)
+			dat, err := ioutil.ReadFile(compileDefPath)
 			if err != nil {
-				return errors.Wrap(err, "Error reading file "+currentpath)
+				return errors.Wrap(err,
+					"Error reading file "+CompilerDefinitionFile+" from "+
+						filepath.Dir(currentpath))
 			}
 			packbuild, err := pkg.DefaultPackageFromYaml(dat)
 			if err != nil {
-				return errors.Wrap(err, "Error reading yaml "+currentpath)
+				return errors.Wrap(err,
+					"Error reading yaml "+CompilerDefinitionFile+" from "+
+						filepath.Dir(currentpath))
 			}
 			pack.Requires(packbuild.GetRequires())
 			pack.Conflicts(packbuild.GetConflicts())
@@ -103,4 +121,4 @@ func (r *CompilerRecipe) Load(path string) error {
 
 func (r *CompilerRecipe) GetDatabase() pkg.PackageDatabase   { return r.Database }
 func (r *CompilerRecipe) WithDatabase(d pkg.PackageDatabase) { r.Database = d }
-func (r *CompilerRecipe) GetSourcePath() string              { return r.SourcePath }
+func (r *CompilerRecipe) GetSourcePath() []string            { return r.SourcePath }