Tag 0.9.3

Collections, similarly to packages, have a `build.yaml` and
a `finalize.yaml` that are templated for each package.
They have a `collection.yaml` containing a list of
packages that are part of the tree.

.github/workflows/release.yml

@@ -0,0 +1,23 @@
+on: push
+name: Build and release on push
+jobs:
+  release:
+    name: Test and Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Tests
+        run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage
+      - name: Build
+        run: sudo -E env "PATH=$PATH" make multiarch-build && sudo chmod -R 777 release/
+      - name: Release
+        uses: fnkr/github-action-ghr@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GHR_PATH: release/
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+
+on: pull_request
+name: Build and Test
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage

.travis.yml

@@ -1,19 +1,35 @@
+dist: bionic
 language: go
-services:
-  - docker
 go:
   - "1.14"
+
 env:
-  - "GO15VENDOREXPERIMENT=1"
+  global:
+    - "GO15VENDOREXPERIMENT=1"
+  jobs:
+    - "DOCKER_BUILDKIT=0"
+    - "DOCKER_BUILDKIT=1"
 before_install:
-  - make deps
-  - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && mkdir -p $HOME/bin && export PATH=$PATH:$HOME/bin && mv container-diff-linux-amd64 $HOME/bin/container-diff
+  - sudo rm -rf /var/lib/apt/lists/*
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) edge"
+  - sudo apt-get update
+  - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
+  - export DOCKER_CLI_EXPERIMENTAL=enabled
+  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  - mkdir -vp ~/.docker/cli-plugins/
+  - curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+  - chmod a+x ~/.docker/cli-plugins/docker-buildx
+  - docker buildx version
+  - sudo -E env "PATH=$PATH" apt-get install -y libcap2-bin
+  - sudo -E env "PATH=$PATH" make deps
 script:
-  - make multiarch-build test-integration test-coverage
-after_success:
-  - |
-      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
-        git config --global user.name "Deployer" && git config --global user.email foo@bar.com
-        go get github.com/tcnksm/ghr
-        ghr -u mudler -r luet --replace $TRAVIS_TAG release/
-      fi
+  - sudo -E env "PATH=$PATH" make multiarch-build test-integration test-coverage
+
+#after_success:
+#  - |
+#      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+#        sudo -E env "PATH=$PATH" git config --global user.name "Deployer" && git config --global user.email foo@bar.com
+#        sudo -E env "PATH=$PATH" go get github.com/tcnksm/ghr
+#        sudo -E env "PATH=$PATH" ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+#      fi

Dockerfile

@@ -1,6 +1,7 @@
 FROM golang as builder
+RUN apt-get update && apt-get install -y upx
 ADD . /luet
-RUN cd /luet && make build
+RUN cd /luet && make build-small
 
 FROM scratch
 ENV LUET_NOLOCK=true

Makefile

@@ -1,7 +1,7 @@
 
 # go tool nm ./luet | grep Commit
-LDFLAGS += -X "github.com/mudler/luet/cmd.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S %Z')"
-LDFLAGS += -X "github.com/mudler/luet/cmd.BuildCommit=$(shell git rev-parse HEAD)"
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S %Z')"
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildCommit=$(shell git rev-parse HEAD)"
 
 NAME ?= luet
 PACKAGE_NAME ?= $(NAME)
@@ -9,7 +9,6 @@ PACKAGE_CONFLICT ?= $(PACKAGE_NAME)-beta
 REVISION := $(shell git rev-parse --short HEAD || echo dev)
 VERSION := $(shell git describe --tags || echo $(REVISION))
 VERSION := $(shell echo $(VERSION) | sed -e 's/^v//g')
-ITTERATION := $(shell date +%s)
 BUILD_PLATFORMS ?= -osarch="linux/amd64" -osarch="linux/386" -osarch="linux/arm"
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
@@ -66,14 +65,15 @@ deps:
 build:
 	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)'
 
+.PHONY: build-small
+build-small:
+	@$(MAKE) LDFLAGS+="-s -w" build
+	upx --brute -1 $(NAME)
+
 .PHONY: image
 image:
 	docker build --rm -t luet/base .
 
-.PHONY: gox-build
-gox-build:
-	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
-
 .PHONY: lint
 lint:
 	golint ./... | grep -v "be unexported"
@@ -90,4 +90,4 @@ test-docker:
 
 .PHONY: multiarch-build
 multiarch-build:
-	gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"

cmd/build.go

@@ -15,15 +15,18 @@
 package cmd
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/spf13/cobra"
@@ -78,7 +81,17 @@ var buildCmd = &cobra.Command{
 		nodeps := viper.GetBool("nodeps")
 		onlydeps := viper.GetBool("onlydeps")
 		keepExportedImages := viper.GetBool("keep-exported-images")
+		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
+		full, _ := cmd.Flags().GetBool("full")
+		skip, _ := cmd.Flags().GetBool("skip-if-metadata-exists")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		var results Results
 
+		out, _ := cmd.Flags().GetString("output")
+		if out != "terminal" {
+			LuetCfg.GetLogging().SetLogLevel("error")
+		}
+		pretend, _ := cmd.Flags().GetBool("pretend")
 		compilerSpecs := compiler.NewLuetCompilationspecs()
 		var compilerBackend compiler.CompilerBackend
 		var db pkg.PackageDatabase
@@ -142,11 +155,30 @@ var buildCmd = &cobra.Command{
 		opts.OnlyDeps = onlydeps
 		opts.NoDeps = nodeps
 		opts.KeepImageExport = keepExportedImages
+		opts.SkipIfMetadataExists = skip
+		opts.PackageTargetOnly = onlyTarget
 
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts)
+		var solverOpts solver.Options
+		if concurrent {
+			solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
+		} else {
+			solverOpts = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+		}
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts, solverOpts)
 		luetCompiler.SetConcurrency(concurrency)
 		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
-		if !all {
+		if full {
+			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
+			if err != nil {
+				Fatal(err.Error())
+			}
+			for _, spec := range specs {
+				Info(":package: Selecting ", spec.GetPackage().GetName(), spec.GetPackage().GetVersion())
+
+				compilerSpecs.Add(spec)
+			}
+		} else if !all {
 			for _, a := range args {
 
 				pack, err := helpers.ParsePackageStr(a)
@@ -181,9 +213,58 @@ var buildCmd = &cobra.Command{
 		if revdeps {
 			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
+		} else if pretend {
+			toCalculate := []compiler.CompilationSpec{}
+			if full {
+				var err error
+				toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...)
+				if err != nil {
+					errs = append(errs, err)
+				}
+			} else {
+				toCalculate = compilerSpecs.All()
+			}
+
+			for _, sp := range toCalculate {
+				packs, err := luetCompiler.ComputeDepTree(sp)
+				if err != nil {
+					errs = append(errs, err)
+				}
+				for _, p := range packs {
+					results.Packages = append(results.Packages,
+						PackageResult{
+							Name:       p.Package.GetName(),
+							Version:    p.Package.GetVersion(),
+							Category:   p.Package.GetCategory(),
+							Repository: "",
+							Hidden:     p.Package.IsHidden(),
+							Target:     sp.GetPackage().HumanReadableString(),
+						})
+				}
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			case "terminal":
+				for _, p := range results.Packages {
+					Info(p.String())
+				}
+			}
 		} else {
 			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
-
 		}
 		if len(errs) != 0 {
 			for _, e := range errs {
@@ -208,7 +289,9 @@ func init() {
 	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
 	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
-	buildCmd.Flags().Bool("all", false, "Build all packages in the tree")
+	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
+	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
+
 	buildCmd.Flags().String("destination", path, "Destination folder")
 	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
 	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
@@ -218,11 +301,17 @@ func init() {
 	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
 	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
 	buildCmd.Flags().Bool("keep-exported-images", false, "Keep exported images used during building")
-
+	buildCmd.Flags().Bool("skip-if-metadata-exists", false, "Skip package if metadata exists")
+	buildCmd.Flags().Bool("only-target-package", false, "Build packages of only the required target. Otherwise builds all the necessary ones not present in the destination")
 	buildCmd.Flags().String("solver-type", "", "Solver strategy")
 	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+
+	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
+
+	buildCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 
 	RootCmd.AddCommand(buildCmd)
 }

cmd/config.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	config "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
 
 	"github.com/spf13/cobra"
 )
@@ -52,6 +53,23 @@ var configCmd = &cobra.Command{
 			}
 		}
 
+		if len(config.LuetCfg.ConfigProtectConfDir) > 0 {
+
+			// Load config protect configs
+			installer.LoadConfigProtectConfs(config.LuetCfg)
+
+			fmt.Println("config_protect_confdir:")
+			for _, dir := range config.LuetCfg.ConfigProtectConfDir {
+				fmt.Println("  - ", dir)
+			}
+
+			if len(config.LuetCfg.GetConfigProtectConfFiles()) > 0 {
+				fmt.Println("protect_conf_files:")
+				for _, file := range config.LuetCfg.GetConfigProtectConfFiles() {
+					fmt.Println("  - ", file.String())
+				}
+			}
+		}
 	},
 }
 

cmd/convert.go

@@ -28,7 +28,7 @@ import (
 )
 
 var convertCmd = &cobra.Command{
-	Use:   "convert",
+	Use:   "convert [portage-tree] [luet-tree]",
 	Short: "convert other package manager tree into luet",
 	Long:  `Parses external PM and produces a luet parsable tree`,
 	PreRun: func(cmd *cobra.Command, args []string) {

cmd/database.go

@@ -0,0 +1,37 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/database"
+
+	"github.com/spf13/cobra"
+)
+
+var databaseGroupCmd = &cobra.Command{
+	Use:   "database [command] [OPTIONS]",
+	Short: "Manage system database (dangerous commands ahead!)",
+}
+
+func init() {
+	RootCmd.AddCommand(databaseGroupCmd)
+
+	databaseGroupCmd.AddCommand(
+		NewDatabaseCreateCommand(),
+		NewDatabaseRemoveCommand(),
+	)
+}

cmd/database/create.go

@@ -0,0 +1,78 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseCreateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "create <artifact_metadata1.yaml> <artifact_metadata1.yaml>",
+		Short: "Insert a package in the system DB",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			var systemDB pkg.PackageDatabase
+
+			for _, a := range args {
+				dat, err := ioutil.ReadFile(a)
+				if err != nil {
+					Fatal("Failed reading ", a, ": ", err.Error())
+				}
+				art, err := compiler.NewPackageArtifactFromYaml(dat)
+				if err != nil {
+					Fatal("Failed reading yaml ", a, ": ", err.Error())
+				}
+
+				if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+					systemDB = pkg.NewBoltDatabase(
+						filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+				} else {
+					systemDB = pkg.NewInMemoryDatabase(true)
+				}
+
+				files := art.GetFiles()
+
+				if _, err := systemDB.CreatePackage(art.GetCompileSpec().GetPackage()); err != nil {
+					Fatal("Failed to create ", a, ": ", err.Error())
+				}
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.GetCompileSpec().GetPackage().GetFingerPrint(), Files: files}); err != nil {
+					Fatal("Failed setting package files for ", a, ": ", err.Error())
+				}
+
+				Info(art.GetCompileSpec().GetPackage().HumanReadableString(), " created")
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/database/remove.go

@@ -0,0 +1,69 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"path/filepath"
+
+	. "github.com/mudler/luet/pkg/logger"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseRemoveCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "remove [package1] [package2] ...",
+		Short: "Remove a package from the system DB (forcefully - you normally don't want to do that)",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var systemDB pkg.PackageDatabase
+
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					Fatal("Invalid package string ", a, ": ", err.Error())
+				}
+
+				if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+					systemDB = pkg.NewBoltDatabase(
+						filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+				} else {
+					systemDB = pkg.NewInMemoryDatabase(true)
+				}
+
+				if err := systemDB.RemovePackage(pack); err != nil {
+					Fatal("Failed removing ", a, ": ", err.Error())
+				}
+
+				if err := systemDB.RemovePackageFiles(pack); err != nil {
+					Fatal("Failed removing files for ", a, ": ", err.Error())
+				}
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/helpers/cli.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package cmd_helpers
 
 import (
 	"errors"

cmd/install.go

@@ -19,9 +19,10 @@ import (
 	"path/filepath"
 
 	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -73,13 +74,24 @@ var installCmd = &cobra.Command{
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps := LuetCfg.Viper.GetBool("nodeps")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
 		LuetCfg.GetSolverOptions().MaxAttempts = attempts
 
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
 			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
 			SolverOptions:               *LuetCfg.GetSolverOptions(),
@@ -118,6 +130,7 @@ func init() {
 	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
 	installCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
 	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	installCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/pack.go

@@ -0,0 +1,89 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var packCmd = &cobra.Command{
+	Use:   "pack <package name>",
+	Short: "pack a custom package",
+	Long:  `pack and creates metadata directly from a source path`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
+		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+		viper.BindPFlag("source", cmd.Flags().Lookup("source"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		sourcePath := viper.GetString("source")
+
+		dst := viper.GetString("destination")
+		compressionType := viper.GetString("compression")
+		concurrency := LuetCfg.GetGeneral().Concurrency
+
+		if len(args) != 1 {
+			Fatal("You must specify a package name")
+		}
+
+		packageName := args[0]
+
+		p, err := helpers.ParsePackageStr(packageName)
+		if err != nil {
+			Fatal("Invalid package string ", packageName, ": ", err.Error())
+		}
+
+		spec := &compiler.LuetCompilationSpec{Package: p}
+		artifact := compiler.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
+		artifact.SetCompressionType(compiler.CompressionImplementation(compressionType))
+		err = artifact.Compress(sourcePath, concurrency)
+		if err != nil {
+			Fatal("failed compressing ", packageName, ": ", err.Error())
+		}
+		artifact.SetCompileSpec(spec)
+		filelist, err := artifact.FileList()
+		if err != nil {
+			Fatal("failed generating file list for ", packageName, ": ", err.Error())
+		}
+		artifact.SetFiles(filelist)
+		artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
+		err = artifact.WriteYaml(dst)
+		if err != nil {
+			Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	packCmd.Flags().String("source", path, "Source folder")
+	packCmd.Flags().String("destination", path, "Destination folder")
+	packCmd.Flags().String("compression", "gzip", "Compression alg: none, gzip")
+
+	RootCmd.AddCommand(packCmd)
+}

cmd/root.go

@@ -24,11 +24,13 @@ import (
 	"strings"
 
 	"github.com/marcsauter/single"
+	bus "github.com/mudler/luet/pkg/bus"
+
+	extensions "github.com/mudler/cobra-extensions"
 	config "github.com/mudler/luet/pkg/config"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	repo "github.com/mudler/luet/pkg/repository"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -38,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.7.8"
+	LuetCLIVersion = "0.9.3"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -68,6 +70,18 @@ var RootCmd = &cobra.Command{
 		if err != nil {
 			Fatal("failed on init tmp basedir:", err.Error())
 		}
+
+		viper.BindPFlag("plugin", cmd.Flags().Lookup("plugin"))
+
+		plugin := viper.GetStringSlice("plugin")
+
+		bus.Manager.Load(plugin...).Register()
+		if len(bus.Manager.Plugins) != 0 {
+			Info(":lollipop:Enabled plugins:")
+			for _, p := range bus.Manager.Plugins {
+				Info("\t:arrow_right:", p.Name)
+			}
+		}
 	},
 	PersistentPostRun: func(cmd *cobra.Command, args []string) {
 		// Cleanup all tmp directories used by luet
@@ -88,8 +102,12 @@ func LoadConfig(c *config.LuetConfig) error {
 		return err
 	}
 
+	noSpinner := c.Viper.GetBool("no_spinner")
+
 	InitAurora()
-	NewSpinner()
+	if !noSpinner {
+		NewSpinner()
+	}
 
 	Debug("Using config file:", c.Viper.ConfigFileUsed())
 
@@ -133,15 +151,6 @@ func Execute() {
 	}
 
 	if err := RootCmd.Execute(); err != nil {
-		if len(os.Args) > 0 {
-			for _, c := range RootCmd.Commands() {
-				if c.Name() == os.Args[1] {
-					os.Exit(-1) // Something failed
-				}
-			}
-			// Try to load a bin from path.
-			helpers.Exec("luet-"+os.Args[1], os.Args[1:], os.Environ())
-		}
 		fmt.Println(err)
 		os.Exit(-1)
 	}
@@ -154,22 +163,22 @@ func init() {
 	pflags.BoolP("debug", "d", false, "verbose output")
 	pflags.Bool("fatal", false, "Enables Warnings to exit")
 	pflags.Bool("enable-logfile", false, "Enable log to file")
+	pflags.Bool("no-spinner", false, "Disable spinner.")
 	pflags.Bool("color", config.LuetCfg.GetLogging().Color, "Enable/Disable color.")
 	pflags.Bool("emoji", config.LuetCfg.GetLogging().EnableEmoji, "Enable/Disable emoji.")
+	pflags.Bool("skip-config-protect", config.LuetCfg.ConfigProtectSkip,
+		"Disable config protect analysis.")
 	pflags.StringP("logfile", "l", config.LuetCfg.GetLogging().Path,
 		"Logfile path. Empty value disable log to file.")
+	pflags.StringSlice("plugin", []string{}, "A list of runtime plugins to load")
 
-	sameOwner := false
-	u, err := user.Current()
-	// os/user doesn't work in from scratch environments
+	// os/user doesn't work in from scratch environments.
+	// Check if i can retrieve user informations.
+	_, err := user.Current()
 	if err != nil {
 		Warning("failed to retrieve user identity:", err.Error())
-		sameOwner = true
 	}
-	if u != nil && u.Uid == "0" {
-		sameOwner = true
-	}
-	pflags.Bool("same-owner", sameOwner, "Maintain same owner on uncompress.")
+	pflags.Bool("same-owner", config.LuetCfg.GetGeneral().SameOwner, "Maintain same owner on uncompress.")
 	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
 
 	config.LuetCfg.Viper.BindPFlag("logging.color", pflags.Lookup("color"))
@@ -181,26 +190,56 @@ func init() {
 	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("debug"))
 	config.LuetCfg.Viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
 	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
+	config.LuetCfg.Viper.BindPFlag("plugin", pflags.Lookup("plugin"))
+
+	// Currently I maintain this only from cli.
+	config.LuetCfg.Viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))
+	config.LuetCfg.Viper.BindPFlag("config_protect_skip", pflags.Lookup("skip-config-protect"))
+
+	// Extensions must be binary with the "luet-" prefix to be able to be shown in the help.
+	// we also accept extensions in the relative path where luet is being started, "extensions/"
+	exts := extensions.Discover("luet", "extensions")
+	for _, ex := range exts {
+		cobraCmd := ex.CobraCommand()
+		RootCmd.AddCommand(cobraCmd)
+	}
+
 }
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
+	// Luet support these priorities on read configuration file:
+	// - command line option (if available)
+	// - $PWD/.luet.yaml
+	// - $HOME/.luet.yaml
+	// - /etc/luet/luet.yaml
+	//
+	// Note: currently a single viper instance support only one config name.
 
-	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
-	if err != nil {
-		Error(err)
-		os.Exit(1)
-	}
 	viper.SetEnvPrefix(LuetEnvPrefix)
 	viper.SetConfigType("yaml")
-	viper.SetConfigName(".luet") // name of config file (without extension)
-	if cfgFile != "" {           // enable ability to specify config file via flag
+
+	if cfgFile != "" { // enable ability to specify config file via flag
 		viper.SetConfigFile(cfgFile)
 	} else {
-		viper.AddConfigPath(dir)
-		viper.AddConfigPath(".")
-		viper.AddConfigPath("$HOME")
-		viper.AddConfigPath("/etc/luet")
+		// Retrieve pwd directory
+		pwdDir, err := os.Getwd()
+		if err != nil {
+			Error(err)
+			os.Exit(1)
+		}
+		homeDir := helpers.GetHomeDir()
+
+		if helpers.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && helpers.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
+			viper.AddConfigPath(".")
+			if homeDir != "" {
+				viper.AddConfigPath(homeDir)
+			}
+			viper.SetConfigName(".luet")
+		} else {
+			viper.SetConfigName("luet")
+			viper.AddConfigPath("/etc/luet")
+		}
 	}
 
 	viper.AutomaticEnv() // read in environment variables that match

cmd/search.go

@@ -32,12 +32,18 @@ type PackageResult struct {
 	Category   string `json:"category"`
 	Version    string `json:"version"`
 	Repository string `json:"repository"`
+	Target     string `json:"target"`
+	Hidden     bool   `json:"hidden"`
 }
 
 type Results struct {
 	Packages []PackageResult `json:"packages"`
 }
 
+func (r PackageResult) String() string {
+	return fmt.Sprintf("%s/%s-%s required for %s", r.Category, r.Name, r.Version, r.Target)
+}
+
 var searchCmd = &cobra.Command{
 	Use:     "search <term>",
 	Short:   "Search packages",
@@ -58,6 +64,9 @@ var searchCmd = &cobra.Command{
 		if len(args) != 1 {
 			Fatal("Wrong number of arguments (expected 1)")
 		}
+
+		hidden, _ := cmd.Flags().GetBool("hidden")
+
 		installed := LuetCfg.Viper.GetBool("installed")
 		stype := LuetCfg.Viper.GetString("solver.type")
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
@@ -114,25 +123,31 @@ var searchCmd = &cobra.Command{
 			}
 			for _, m := range matches {
 				if !revdeps {
-					Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
-					results.Packages = append(results.Packages,
-						PackageResult{
-							Name:       m.Package.GetName(),
-							Version:    m.Package.GetVersion(),
-							Category:   m.Package.GetCategory(),
-							Repository: m.Repo.GetName(),
-						})
+					if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
+						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
+						results.Packages = append(results.Packages,
+							PackageResult{
+								Name:       m.Package.GetName(),
+								Version:    m.Package.GetVersion(),
+								Category:   m.Package.GetCategory(),
+								Repository: m.Repo.GetName(),
+								Hidden:     m.Package.IsHidden(),
+							})
+					}
 				} else {
 					visited := make(map[string]interface{})
 					for _, revdep := range m.Package.ExpandedRevdeps(m.Repo.GetTree().GetDatabase(), visited) {
-						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
-						results.Packages = append(results.Packages,
-							PackageResult{
-								Name:       revdep.GetName(),
-								Version:    revdep.GetVersion(),
-								Category:   revdep.GetCategory(),
-								Repository: m.Repo.GetName(),
-							})
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: m.Repo.GetName(),
+									Hidden:     revdep.IsHidden(),
+								})
+						}
 					}
 				}
 			}
@@ -162,26 +177,32 @@ var searchCmd = &cobra.Command{
 
 			for _, pack := range iMatches {
 				if !revdeps {
-					Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
-					results.Packages = append(results.Packages,
-						PackageResult{
-							Name:       pack.GetName(),
-							Version:    pack.GetVersion(),
-							Category:   pack.GetCategory(),
-							Repository: "system",
-						})
+					if !pack.IsHidden() || pack.IsHidden() && hidden {
+						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+						results.Packages = append(results.Packages,
+							PackageResult{
+								Name:       pack.GetName(),
+								Version:    pack.GetVersion(),
+								Category:   pack.GetCategory(),
+								Repository: "system",
+								Hidden:     pack.IsHidden(),
+							})
+					}
 				} else {
 					visited := make(map[string]interface{})
 
 					for _, revdep := range pack.ExpandedRevdeps(system.Database, visited) {
-						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
-						results.Packages = append(results.Packages,
-							PackageResult{
-								Name:       revdep.GetName(),
-								Version:    revdep.GetVersion(),
-								Category:   revdep.GetCategory(),
-								Repository: "system",
-							})
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: "system",
+									Hidden:     revdep.IsHidden(),
+								})
+						}
 					}
 				}
 			}
@@ -223,5 +244,7 @@ func init() {
 	searchCmd.Flags().Bool("by-label", false, "Search packages through label")
 	searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex")
 	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
+
 	RootCmd.AddCommand(searchCmd)
 }

cmd/tree.go

@@ -34,5 +34,6 @@ func init() {
 		NewTreePkglistCommand(),
 		NewTreeValidateCommand(),
 		NewTreeBumpCommand(),
+		NewTreeImageCommand(),
 	)
 }

cmd/tree/bump.go

@@ -18,11 +18,11 @@ package cmd_tree
 
 import (
 	"fmt"
-	//"os"
-	//"sort"
 
 	. "github.com/mudler/luet/pkg/logger"
+	spectooling "github.com/mudler/luet/pkg/spectooling"
 	tree "github.com/mudler/luet/pkg/tree"
+	version "github.com/mudler/luet/pkg/versioner"
 
 	"github.com/spf13/cobra"
 )
@@ -42,19 +42,28 @@ func NewTreeBumpCommand() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			spec, _ := cmd.Flags().GetString("definition-file")
 			toStdout, _ := cmd.Flags().GetBool("to-stdout")
+			pkgVersion, _ := cmd.Flags().GetString("pkg-version")
 			pack, err := tree.ReadDefinitionFile(spec)
 			if err != nil {
 				Fatal(err.Error())
 			}
 
-			// Retrieve version build section with Gentoo parser
-			err = pack.BumpBuildVersion()
-			if err != nil {
-				Fatal("Error on increment build version: " + err.Error())
+			if pkgVersion != "" {
+				validator := &version.WrappedVersioner{}
+				err := validator.Validate(pkgVersion)
+				if err != nil {
+					Fatal("Invalid version string: " + err.Error())
+				}
+				pack.SetVersion(pkgVersion)
+			} else {
+				// Retrieve version build section with Gentoo parser
+				err = pack.BumpBuildVersion()
+				if err != nil {
+					Fatal("Error on increment build version: " + err.Error())
+				}
 			}
-
 			if toStdout {
-				data, err := pack.Yaml()
+				data, err := spectooling.NewDefaultPackageSanitized(&pack).Yaml()
 				if err != nil {
 					Fatal("Error on yaml conversion: " + err.Error())
 				}
@@ -71,6 +80,7 @@ func NewTreeBumpCommand() *cobra.Command {
 		},
 	}
 
+	ans.Flags().StringP("pkg-version", "p", "", "Set a specific package version")
 	ans.Flags().StringP("definition-file", "f", "", "Path of the definition to bump.")
 	ans.Flags().BoolP("to-stdout", "o", false, "Bump package to output.")
 

cmd/tree/images.go

@@ -0,0 +1,136 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"fmt"
+
+	//. "github.com/mudler/luet/pkg/config"
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func NewTreeImageCommand() *cobra.Command {
+
+	var ans = &cobra.Command{
+		Use:   "images [OPTIONS]",
+		Short: "List of the images of a package",
+		PreRun: func(cmd *cobra.Command, args []string) {
+			t, _ := cmd.Flags().GetStringArray("tree")
+			if len(t) == 0 {
+				Fatal("Mandatory tree param missing.")
+			}
+
+			if len(args) != 1 {
+				Fatal("Expects one package as parameter")
+			}
+			viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var results TreeResults
+
+			treePath, _ := cmd.Flags().GetStringArray("tree")
+			imageRepository := viper.GetString("image-repository")
+
+			out, _ := cmd.Flags().GetString("output")
+			if out != "terminal" {
+				LuetCfg.GetLogging().SetLogLevel("error")
+			}
+
+			reciper := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			for _, t := range treePath {
+				err := reciper.Load(t)
+				if err != nil {
+					Fatal("Error on load tree ", err)
+				}
+			}
+			compilerBackend := backend.NewSimpleDockerBackend()
+
+			opts := compiler.NewDefaultCompilerOptions()
+			opts.SolverOptions = *LuetCfg.GetSolverOptions()
+			opts.ImageRepository = imageRepository
+
+			solverOpts := solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
+			luetCompiler := compiler.NewLuetCompiler(compilerBackend, reciper.GetDatabase(), opts, solverOpts)
+
+			a := args[0]
+
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+
+			spec, err := luetCompiler.FromPackage(pack)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+			asserts, err := luetCompiler.ComputeDepTree(spec)
+
+			for _, assertion := range asserts { //highly dependent on the order
+
+				//buildImageHash := imageRepository + ":" + assertion.Hash.BuildHash
+				currentPackageImageHash := imageRepository + ":" + assertion.Hash.PackageHash
+
+				results.Packages = append(results.Packages, TreePackageResult{
+					Name:     assertion.Package.GetName(),
+					Version:  assertion.Package.GetVersion(),
+					Category: assertion.Package.GetCategory(),
+					Image:    currentPackageImageHash,
+				})
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			default:
+				for _, p := range results.Packages {
+					fmt.Println(fmt.Sprintf("%s/%s-%s: %s", p.Category, p.Name, p.Version, p.Image))
+				}
+			}
+		},
+	}
+
+	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+	ans.Flags().StringArrayP("tree", "t", []string{}, "Path of the tree to use.")
+	ans.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+
+	return ans
+}

cmd/tree/pkglist.go

@@ -22,10 +22,11 @@ import (
 
 	//. "github.com/mudler/luet/pkg/config"
 	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/spf13/cobra"
@@ -36,6 +37,7 @@ type TreePackageResult struct {
 	Category string `json:"category"`
 	Version  string `json:"version"`
 	Path     string `json:"path"`
+	Image    string `json:"image"`
 }
 
 type TreeResults struct {
@@ -74,15 +76,24 @@ func NewTreePkglistCommand() *cobra.Command {
 			if len(t) == 0 {
 				Fatal("Mandatory tree param missing.")
 			}
+
+			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
+			if revdeps && deps {
+				Fatal("Both revdeps and deps option used. Choice only one.")
+			}
+
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			var results TreeResults
+			var depSolver solver.PackageSolver
 
 			treePath, _ := cmd.Flags().GetStringArray("tree")
 			verbose, _ := cmd.Flags().GetBool("verbose")
 			buildtime, _ := cmd.Flags().GetBool("buildtime")
 			full, _ := cmd.Flags().GetBool("full")
 			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
 
 			out, _ := cmd.Flags().GetString("output")
 			if out != "terminal" {
@@ -95,6 +106,7 @@ func NewTreePkglistCommand() *cobra.Command {
 			} else {
 				reciper = tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
 			}
+
 			for _, t := range treePath {
 				err := reciper.Load(t)
 				if err != nil {
@@ -102,6 +114,15 @@ func NewTreePkglistCommand() *cobra.Command {
 				}
 			}
 
+			if deps {
+				emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+
+				depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
+					reciper.GetDatabase(),
+					emptyInstallationDb)
+
+			}
+
 			regExcludes, err := helpers.CreateRegexArray(excludes)
 			if err != nil {
 				Fatal(err.Error())
@@ -146,15 +167,8 @@ func NewTreePkglistCommand() *cobra.Command {
 				}
 
 				if addPkg {
-					if !revdeps {
-						plist = append(plist, pkgstr)
-						results.Packages = append(results.Packages, TreePackageResult{
-							Name:     p.GetName(),
-							Version:  p.GetVersion(),
-							Category: p.GetCategory(),
-							Path:     p.GetPath(),
-						})
-					} else {
+					if revdeps {
+
 						visited := make(map[string]interface{})
 						for _, revdep := range p.ExpandedRevdeps(reciper.GetDatabase(), visited) {
 							if full {
@@ -172,7 +186,58 @@ func NewTreePkglistCommand() *cobra.Command {
 								Path:     revdep.GetPath(),
 							})
 						}
+					} else if deps {
+
+						Spinner(32)
+						solution, err := depSolver.Install(pkg.Packages{p})
+						if err != nil {
+							Fatal(err.Error())
+						}
+						ass := solution.SearchByName(p.GetPackageName())
+						solution, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+						if err != nil {
+							Fatal(err.Error())
+						}
+						SpinnerStop()
+
+						for _, pa := range solution {
+
+							if pa.Value {
+								// Exclude itself
+								if pa.Package.GetName() == p.GetName() && pa.Package.GetCategory() == p.GetCategory() {
+									continue
+								}
+
+								if full {
+									pkgstr = pkgDetail(pa.Package)
+								} else if verbose {
+									pkgstr = pa.Package.HumanReadableString()
+								} else {
+									pkgstr = fmt.Sprintf("%s/%s", pa.Package.GetCategory(), pa.Package.GetName())
+								}
+								plist = append(plist, pkgstr)
+								results.Packages = append(results.Packages, TreePackageResult{
+									Name:     pa.Package.GetName(),
+									Version:  pa.Package.GetVersion(),
+									Category: pa.Package.GetCategory(),
+									Path:     pa.Package.GetPath(),
+								})
+							}
+
+						}
+
+					} else {
+
+						plist = append(plist, pkgstr)
+						results.Packages = append(results.Packages, TreePackageResult{
+							Name:     p.GetName(),
+							Version:  p.GetVersion(),
+							Category: p.GetCategory(),
+							Path:     p.GetPath(),
+						})
+
 					}
+
 				}
 			}
 
@@ -192,7 +257,9 @@ func NewTreePkglistCommand() *cobra.Command {
 				}
 				fmt.Println(string(j2))
 			default:
-				sort.Strings(plist)
+				if !deps {
+					sort.Strings(plist)
+				}
 				for _, p := range plist {
 					fmt.Println(p)
 				}
@@ -204,6 +271,7 @@ func NewTreePkglistCommand() *cobra.Command {
 	ans.Flags().BoolP("buildtime", "b", false, "Build time match")
 	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 	ans.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	ans.Flags().Bool("deps", false, "Search package dependencies")
 
 	ans.Flags().BoolP("verbose", "v", false, "Add package version")
 	ans.Flags().BoolP("full", "f", false, "Show package detail")

cmd/tree/validate.go

@@ -25,8 +25,8 @@ import (
 	"strconv"
 	"sync"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
@@ -35,248 +35,398 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func validateWorker(i int,
-	wg *sync.WaitGroup,
-	c <-chan pkg.Package,
-	reciper tree.Builder,
-	withSolver bool,
-	regExcludes, regMatches []*regexp.Regexp,
-	excludes, matches []string,
-	errs chan error) {
+type ValidateOpts struct {
+	WithSolver    bool
+	OnlyRuntime   bool
+	OnlyBuildtime bool
+	RegExcludes   []*regexp.Regexp
+	RegMatches    []*regexp.Regexp
+	Excludes      []string
+	Matches       []string
 
-	defer wg.Done()
+	// Runtime validate stuff
+	RuntimeCacheDeps *pkg.InMemoryDatabase
+	RuntimeReciper   *tree.InstallerRecipe
+
+	// Buildtime validate stuff
+	BuildtimeCacheDeps *pkg.InMemoryDatabase
+	BuildtimeReciper   *tree.CompilerRecipe
+
+	Mutex      sync.Mutex
+	BrokenPkgs int
+	BrokenDeps int
+
+	Errors []error
+}
+
+func (o *ValidateOpts) IncrBrokenPkgs() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenPkgs++
+}
+
+func (o *ValidateOpts) IncrBrokenDeps() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenDeps++
+}
+
+func (o *ValidateOpts) AddError(err error) {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.Errors = append(o.Errors, err)
+}
+
+func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, reciper tree.Builder, cacheDeps *pkg.InMemoryDatabase) error {
+	var errstr string
+	var ans error
 
 	var depSolver solver.PackageSolver
-	var cacheDeps *pkg.InMemoryDatabase
-	brokenPkgs := 0
-	brokenDeps := 0
-	var errstr string
 
-	emptyInstallationDb := pkg.NewInMemoryDatabase(false)
-	if withSolver {
-		depSolver = solver.NewSolver(pkg.NewInMemoryDatabase(false),
+	if opts.WithSolver {
+		emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+		depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
 			reciper.GetDatabase(),
 			emptyInstallationDb)
-
-		// Use Singleton in memory cache for speedup dependencies
-		// analysis
-		cacheDeps = pkg.NewInMemoryDatabase(true).(*pkg.InMemoryDatabase)
 	}
 
-	for p := range c {
+	found, err := reciper.GetDatabase().FindPackages(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  ">=0",
+		},
+	)
 
-		found, err := reciper.GetDatabase().FindPackages(
-			&pkg.DefaultPackage{
-				Name:     p.GetName(),
-				Category: p.GetCategory(),
-				Version:  ">=0",
-			},
+	if err != nil || len(found) < 1 {
+		if err != nil {
+			errstr = err.Error()
+		} else {
+			errstr = "No packages"
+		}
+		Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			errstr,
+		))
+
+		opts.IncrBrokenDeps()
+
+		return errors.New(
+			fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+				checkType,
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				errstr,
+			))
+	}
+
+	// Ensure that we use the right package from right recipier for deps
+	pReciper, err := reciper.GetDatabase().FindPackage(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  p.GetVersion(),
+		},
+	)
+	if err != nil {
+		errstr = fmt.Sprintf("[%9s] %s/%s-%s: Error on retrieve package - %s.",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			err.Error(),
 		)
+		Error(errstr)
 
-		if err != nil || len(found) < 1 {
+		return errors.New(errstr)
+	}
+	p = pReciper
+
+	pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
+		p.GetVersion())
+
+	validpkg := true
+
+	if len(opts.Matches) > 0 {
+		matched := false
+		for _, rgx := range opts.RegMatches {
+			if rgx.MatchString(pkgstr) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return nil
+		}
+	}
+
+	if len(opts.Excludes) > 0 {
+		excluded := false
+		for _, rgx := range opts.RegExcludes {
+			if rgx.MatchString(pkgstr) {
+				excluded = true
+				break
+			}
+		}
+
+		if excluded {
+			return nil
+		}
+	}
+
+	Info(fmt.Sprintf("[%9s] Checking package ", checkType)+
+		fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
+		"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
+
+	all := p.GetRequires()
+	all = append(all, p.GetConflicts()...)
+	for idx, r := range all {
+
+		var deps pkg.Packages
+		var err error
+		if r.IsSelector() {
+			deps, err = reciper.GetDatabase().FindPackages(
+				&pkg.DefaultPackage{
+					Name:     r.GetName(),
+					Category: r.GetCategory(),
+					Version:  r.GetVersion(),
+				},
+			)
+		} else {
+			deps = append(deps, r)
+		}
+
+		if err != nil || len(deps) < 1 {
 			if err != nil {
 				errstr = err.Error()
 			} else {
 				errstr = "No packages"
 			}
-			Error(fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
+			Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+				checkType,
 				p.GetCategory(), p.GetName(), p.GetVersion(),
+				r.GetCategory(), r.GetName(), r.GetVersion(),
 				errstr,
 			))
 
-			errs <- errors.New(
-				fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
-					p.GetCategory(), p.GetName(), p.GetVersion(),
-					errstr,
-				))
+			opts.IncrBrokenDeps()
 
-			brokenPkgs++
-		}
-
-		pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
-			p.GetVersion())
-
-		validpkg := true
-
-		if len(matches) > 0 {
-			matched := false
-			for _, rgx := range regMatches {
-				if rgx.MatchString(pkgstr) {
-					matched = true
-					break
-				}
-			}
-
-			if !matched {
-				continue
-			}
-		}
-
-		if len(excludes) > 0 {
-			excluded := false
-			for _, rgx := range regExcludes {
-				if rgx.MatchString(pkgstr) {
-					excluded = true
-					break
-				}
-			}
-
-			if excluded {
-				continue
-			}
-		}
-
-		Info("Checking package "+
-			fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
-			"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
-
-		all := p.GetRequires()
-		all = append(all, p.GetConflicts()...)
-		for idx, r := range all {
-
-			var deps pkg.Packages
-			var err error
-			if r.IsSelector() {
-				deps, err = reciper.GetDatabase().FindPackages(
-					&pkg.DefaultPackage{
-						Name:     r.GetName(),
-						Category: r.GetCategory(),
-						Version:  r.GetVersion(),
-					},
-				)
-			} else {
-				deps = append(deps, r)
-			}
-
-			if err != nil || len(deps) < 1 {
-				if err != nil {
-					errstr = err.Error()
-				} else {
-					errstr = "No packages"
-				}
-				Error(fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
+			ans = errors.New(
+				fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+					checkType,
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					r.GetCategory(), r.GetName(), r.GetVersion(),
+					errstr))
+
+			validpkg = false
+
+		} else {
+
+			Debug(fmt.Sprintf("[%9s] Find packages for dep", checkType),
+				fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+
+			if opts.WithSolver {
+
+				Info(fmt.Sprintf("[%9s]  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
+					checkType,
+					idx+1, len(all),
 					p.GetCategory(), p.GetName(), p.GetVersion(),
 					r.GetCategory(), r.GetName(), r.GetVersion(),
-					errstr,
 				))
 
-				errs <- errors.New(
-					fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
-						p.GetCategory(), p.GetName(), p.GetVersion(),
-						r.GetCategory(), r.GetName(), r.GetVersion(),
-						errstr))
-
-				brokenDeps++
-
-				validpkg = false
-
-			} else {
-
-				Debug("Find packages for dep",
-					fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
-
-				if withSolver {
-
-					Info(fmt.Sprintf("  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
-						idx+1, len(all),
+				// Check if the solver is already been done for the deep
+				_, err := cacheDeps.Get(r.HashFingerprint(""))
+				if err == nil {
+					Debug(fmt.Sprintf("[%9s]  :direct_hit: Cache Hit for dep", checkType),
+						fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+					continue
+				}
+
+				Spinner(32)
+				solution, err := depSolver.Install(pkg.Packages{r})
+				ass := solution.SearchByName(r.GetPackageName())
+				if err == nil {
+					_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+				}
+				SpinnerStop()
+
+				if err != nil {
+
+					Error(fmt.Sprintf("[%9s] %s/%s-%s: solver broken for dep %s/%s-%s - %s",
+						checkType,
 						p.GetCategory(), p.GetName(), p.GetVersion(),
 						r.GetCategory(), r.GetName(), r.GetVersion(),
+						err.Error(),
 					))
 
-					// Check if the solver is already been done for the deep
-					_, err := cacheDeps.Get(r.HashFingerprint())
-					if err == nil {
-						Debug("  :direct_hit: Cache Hit for dep",
-							fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
-						continue
-					}
-
-					Spinner(32)
-					solution, err := depSolver.Install(pkg.Packages{r})
-					ass := solution.SearchByName(r.GetPackageName())
-					if err == nil {
-						_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
-					}
-					SpinnerStop()
-
-					if err != nil {
-
-						Error(fmt.Sprintf("%s/%s-%s: solver broken for dep %s/%s-%s - %s",
+					ans = errors.New(
+						fmt.Sprintf("[%9s] %s/%s-%s: solver broken for Dep %s/%s-%s - %s",
+							checkType,
 							p.GetCategory(), p.GetName(), p.GetVersion(),
 							r.GetCategory(), r.GetName(), r.GetVersion(),
-							err.Error(),
-						))
-
-						errs <- errors.New(
-							fmt.Sprintf("%s/%s-%s: solver broken for Dep %s/%s-%s - %s",
-								p.GetCategory(), p.GetName(), p.GetVersion(),
-								r.GetCategory(), r.GetName(), r.GetVersion(),
-								err.Error()))
-
-						brokenDeps++
-						validpkg = false
-					}
-
-					// Register the key
-					cacheDeps.Set(r.HashFingerprint(), "1")
+							err.Error()))
 
+					opts.IncrBrokenDeps()
+					validpkg = false
 				}
+
+				// Register the key
+				cacheDeps.Set(r.HashFingerprint(""), "1")
+
+			}
+		}
+
+	}
+
+	if !validpkg {
+		opts.IncrBrokenPkgs()
+	}
+
+	return ans
+}
+
+func validateWorker(i int,
+	wg *sync.WaitGroup,
+	c <-chan pkg.Package,
+	opts *ValidateOpts) {
+
+	defer wg.Done()
+
+	for p := range c {
+
+		if opts.OnlyBuildtime {
+			// Check buildtime compiler/deps
+			err := validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+		} else if opts.OnlyRuntime {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+		} else {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+			// Check buildtime compiler/deps
+			err = validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
 			}
 
 		}
 
-		if !validpkg {
-			brokenPkgs++
+	}
+}
+
+func initOpts(opts *ValidateOpts, onlyRuntime, onlyBuildtime, withSolver bool, treePaths []string) {
+	var err error
+
+	opts.OnlyBuildtime = onlyBuildtime
+	opts.OnlyRuntime = onlyRuntime
+	opts.WithSolver = withSolver
+	opts.RuntimeReciper = nil
+	opts.BuildtimeReciper = nil
+	opts.BrokenPkgs = 0
+	opts.BrokenDeps = 0
+
+	if onlyBuildtime {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+	} else if onlyRuntime {
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	} else {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	}
+
+	opts.RuntimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+	opts.BuildtimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+
+	for _, treePath := range treePaths {
+		Info(fmt.Sprintf("Loading :deciduous_tree: %s...", treePath))
+		if opts.BuildtimeReciper != nil {
+			err = opts.BuildtimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
+		}
+		if opts.RuntimeReciper != nil {
+			err = opts.RuntimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
 		}
 	}
+
+	opts.RegExcludes, err = helpers.CreateRegexArray(opts.Excludes)
+	if err != nil {
+		Fatal(err.Error())
+	}
+	opts.RegMatches, err = helpers.CreateRegexArray(opts.Matches)
+	if err != nil {
+		Fatal(err.Error())
+	}
+
 }
 
 func NewTreeValidateCommand() *cobra.Command {
 	var excludes []string
 	var matches []string
 	var treePaths []string
+	var opts ValidateOpts
 
 	var ans = &cobra.Command{
 		Use:   "validate [OPTIONS]",
 		Short: "Validate a tree or a list of packages",
 		Args:  cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
 			if len(treePaths) < 1 {
 				Fatal("Mandatory tree param missing.")
 			}
+			if onlyRuntime && onlyBuildtime {
+				Fatal("Both --only-runtime and --only-buildtime options are not possibile.")
+			}
 		},
 		Run: func(cmd *cobra.Command, args []string) {
+			var reciper tree.Builder
+
 			concurrency := LuetCfg.GetGeneral().Concurrency
 
 			withSolver, _ := cmd.Flags().GetBool("with-solver")
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
 
-			reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
-			for _, treePath := range treePaths {
-				err := reciper.Load(treePath)
-				if err != nil {
-					Fatal("Error on load tree ", err)
-				}
-			}
+			opts.Excludes = excludes
+			opts.Matches = matches
+			initOpts(&opts, onlyRuntime, onlyBuildtime, withSolver, treePaths)
 
-			regExcludes, err := helpers.CreateRegexArray(excludes)
-			if err != nil {
-				Fatal(err.Error())
-			}
-			regMatches, err := helpers.CreateRegexArray(matches)
-			if err != nil {
-				Fatal(err.Error())
+			// We need at least one valid reciper for get list of the packages.
+			if onlyBuildtime {
+				reciper = opts.BuildtimeReciper
+			} else {
+				reciper = opts.RuntimeReciper
 			}
 
 			all := make(chan pkg.Package)
-			errs := make(chan error)
 
 			var wg = new(sync.WaitGroup)
 
 			for i := 0; i < concurrency; i++ {
 				wg.Add(1)
-				go validateWorker(i, wg, all,
-					reciper, withSolver, regExcludes, regMatches, excludes, matches,
-					errs)
+				go validateWorker(i, wg, all, &opts)
 			}
 			for _, p := range reciper.GetDatabase().World() {
 				all <- p
@@ -286,11 +436,10 @@ func NewTreeValidateCommand() *cobra.Command {
 			// Wait separately and once done close the channel
 			go func() {
 				wg.Wait()
-				close(errs)
 			}()
 
 			stringerrs := []string{}
-			for e := range errs {
+			for _, e := range opts.Errors {
 				stringerrs = append(stringerrs, e.Error())
 			}
 			sort.Strings(stringerrs)
@@ -300,9 +449,9 @@ func NewTreeValidateCommand() *cobra.Command {
 
 			// fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
 			if len(stringerrs) != 0 {
+				Error(fmt.Sprintf("Found %d broken packages and %d broken deps.",
+					opts.BrokenPkgs, opts.BrokenDeps))
 				Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
-				//	if brokenPkgs > 0 {
-				//os.Exit(1)
 			} else {
 				Info("All good! :white_check_mark:")
 				os.Exit(0)
@@ -310,6 +459,8 @@ func NewTreeValidateCommand() *cobra.Command {
 		},
 	}
 
+	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
+	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")
 	ans.Flags().BoolP("with-solver", "s", false,
 		"Enable check of requires also with solver.")
 	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{},

cmd/uninstall.go

@@ -18,11 +18,12 @@ import (
 	"os"
 	"path/filepath"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/spf13/cobra"
 )
@@ -57,24 +58,34 @@ var uninstallCmd = &cobra.Command{
 			rate := LuetCfg.Viper.GetFloat64("solver.rate")
 			attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 			force := LuetCfg.Viper.GetBool("force")
-			nodeps := LuetCfg.Viper.GetBool("nodeps")
+			nodeps, _ := cmd.Flags().GetBool("nodeps")
 			full, _ := cmd.Flags().GetBool("full")
 			checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
+			fullClean, _ := cmd.Flags().GetBool("full-clean")
+			concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 
 			LuetCfg.GetSolverOptions().Type = stype
 			LuetCfg.GetSolverOptions().LearnRate = float32(rate)
 			LuetCfg.GetSolverOptions().Discount = float32(discount)
 			LuetCfg.GetSolverOptions().MaxAttempts = attempts
-
+			if concurrent {
+				LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+			} else {
+				LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+			}
 			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
+			// Load config protect configs
+			installer.LoadConfigProtectConfs(LuetCfg)
+
 			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-				Concurrency:    LuetCfg.GetGeneral().Concurrency,
-				SolverOptions:  *LuetCfg.GetSolverOptions(),
-				NoDeps:         nodeps,
-				Force:          force,
-				FullUninstall:  full,
-				CheckConflicts: checkconflicts,
+				Concurrency:        LuetCfg.GetGeneral().Concurrency,
+				SolverOptions:      *LuetCfg.GetSolverOptions(),
+				NoDeps:             nodeps,
+				Force:              force,
+				FullUninstall:      full,
+				FullCleanUninstall: fullClean,
+				CheckConflicts:     checkconflicts,
 			})
 
 			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
@@ -107,6 +118,8 @@ func init() {
 	uninstallCmd.Flags().Bool("force", false, "Force uninstall")
 	uninstallCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
 	uninstallCmd.Flags().Bool("conflictscheck", true, "Check if the package marked for deletion is required by other packages")
+	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
+	uninstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -22,6 +22,7 @@ import (
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/spf13/cobra"
 )
@@ -58,18 +59,37 @@ var upgradeCmd = &cobra.Command{
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
 		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
+		nodeps, _ := cmd.Flags().GetBool("nodeps")
+		full, _ := cmd.Flags().GetBool("full")
+		universe, _ := cmd.Flags().GetBool("universe")
+		clean, _ := cmd.Flags().GetBool("clean")
+		sync, _ := cmd.Flags().GetBool("sync")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
 		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
 
 		Debug("Solver", LuetCfg.GetSolverOptions().String())
 
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:   LuetCfg.GetGeneral().Concurrency,
-			SolverOptions: *LuetCfg.GetSolverOptions(),
-			Force:         force,
+			Concurrency:                LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:              *LuetCfg.GetSolverOptions(),
+			Force:                      force,
+			FullUninstall:              full,
+			NoDeps:                     nodeps,
+			SolverUpgrade:              universe,
+			RemoveUnavailableOnUpgrade: clean,
+			UpgradeNewRevisions:        sync,
 		})
 		inst.Repositories(repos)
 		_, err := inst.SyncRepositories(false)
@@ -103,6 +123,12 @@ func init() {
 	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
+	upgradeCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
+	upgradeCmd.Flags().Bool("full", true, "Attempts to remove as much packages as possible which aren't required (slow)")
+	upgradeCmd.Flags().Bool("universe", false, "Use ONLY the SAT solver to compute upgrades (experimental)")
+	upgradeCmd.Flags().Bool("clean", false, "Try to drop removed packages (experimental, only when --universe is enabled)")
+	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
+	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

contrib/config/config.protect.d/01_etc.yml.example

@@ -0,0 +1,3 @@
+name: "etc_conf"
+dirs:
+  - "/etc/"

contrib/config/get_luet_root.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -ex
+export LUET_NOLOCK=true
+
+LUET_VERSION=0.8.6
+LUET_ROOTFS=${LUET_ROOTFS:-/}
+LUET_DATABASE_PATH=${LUET_DATABASE_PATH:-/var/luet/db}
+LUET_DATABASE_ENGINE=${LUET_DATABASE_ENGINE:-boltdb}
+LUET_CONFIG_PROTECT=${LUET_CONFIG_PROTECT:-1}
+
+wget -q https://github.com/mudler/luet/releases/download/0.8.6/luet-0.8.6-linux-amd64 -O luet
+chmod +x luet
+
+mkdir -p /etc/luet/repos.conf.d || true
+mkdir -p $LUET_DATABASE_PATH || true
+mkdir -p /var/tmp/luet || true
+
+if [ "${LUET_CONFIG_PROTECT}" = "1" ] ; then
+  mkdir -p /etc/luet/config.protect.d || true
+  wget -q  https://raw.githubusercontent.com/mudler/luet/master/contrib/config/config.protect.d/01_etc.yml.example -O /etc/luet/config.protect.d/01_etc.yml
+fi
+wget -q https://raw.githubusercontent.com/mocaccinoOS/repository-index/master/packages/mocaccino-repository-index/mocaccino-repository-index.yml -O /etc/luet/repos.conf.d/mocaccino-repository-index.yml
+
+cat > /etc/luet/luet.yaml <<EOF
+general:
+  debug: false
+system:
+  rootfs: ${LUET_ROOTFS}
+  database_path: "${LUET_DATABASE_PATH}"
+  database_engine: "${LUET_DATABASE_ENGINE}"
+  tmpdir_base: "/var/tmp/luet"
+EOF
+
+./luet install repository/luet repository/mocaccino-repository-index
+./luet install system/luet system/luet-extensions
+
+rm -rf luet

contrib/config/luet.yaml

@@ -16,6 +16,12 @@
 #   Enable JSON log format instead of console mode.
 #   json_format: false.
 #
+#   Disable/Enable color
+#   color: true
+#
+#   Enable/Disable emoji
+#   enable_emoji: true
+#
 # ---------------------------------------------
 # General configuration section:
 # ---------------------------------------------
@@ -63,6 +69,7 @@
 #   Default $TMPDIR/tmpluet
 #   tmpdir_base: "/tmp/tmpluet"
 #
+#
 # ---------------------------------------------
 # Repositories configurations directories.
 # ---------------------------------------------
@@ -73,7 +80,25 @@
 #   - /etc/luet/repos.conf.d
 #
 #
-# ---------------------------------------------
+# ------------------------------------------------
+# Config protect configuration files directories.
+# -----------------------------------------------
+# Define the list of directories where load
+# configuration files with the list of config
+# protect paths.
+# config_protect_confdir:
+#   - /etc/luet/config.protect.d
+#
+# Permit to ignore rules defined on
+# config protect confdir and packages
+# annotation.
+# config_protect_skip: false
+#
+# The paths used for load repositories and config
+# protects are based on host rootfs.
+# If set to false rootfs path is used as prefix.
+# config_from_host: true
+#
 # System repositories
 # ---------------------------------------------
 # In alternative to define repositories files

go.mod

@@ -4,12 +4,13 @@ go 1.12
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/Sabayon/pkgs-checker v0.6.2-0.20200404093625-076438c31739
+	github.com/Sabayon/pkgs-checker v0.7.2
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/briandowns/spinner v1.7.0
 	github.com/cavaliercoder/grab v2.0.0+incompatible
-	github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3
+	github.com/crillab/gophersat v1.3.2-0.20201023142334-3fc2ac466765
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200417035958-130b0bc6032c+incompatible
+	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
 	github.com/ghodss/yaml v1.0.0
 	github.com/hashicorp/go-version v1.2.0
@@ -22,28 +23,27 @@ require (
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
 	github.com/mattn/go-isatty v0.0.10 // indirect
 	github.com/moby/sys/mount v0.1.1-0.20200320164225-6154f11e6840 // indirect
+	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.0
-	github.com/otiai10/copy v1.0.2
+	github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
+	github.com/onsi/ginkgo v1.14.2
+	github.com/onsi/gomega v1.10.3
+	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
 	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
-	github.com/pkg/errors v0.8.1
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.5.0
-	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
+	github.com/pkg/errors v0.9.1
+	github.com/spf13/cobra v1.0.0
+	github.com/spf13/viper v1.6.3
 	go.etcd.io/bbolt v1.3.4
 	go.uber.org/atomic v1.5.1 // indirect
 	go.uber.org/multierr v1.4.0 // indirect
 	go.uber.org/zap v1.13.0
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
-	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
-	golang.org/x/tools v0.0.0-20200102200121-6de373a2766c // indirect
-	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
-	gopkg.in/yaml.v2 v2.2.7
+	gopkg.in/yaml.v2 v2.3.0
+	gotest.tools/v3 v3.0.2 // indirect
+	helm.sh/helm/v3 v3.3.4
 	mvdan.cc/sh/v3 v3.0.0-beta1
 )
 
-replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200418093736-b2b0766ef22c+incompatible
+replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200605210607-749178b8f80d+incompatible

pkg/bus/events.go

@@ -0,0 +1,46 @@
+package bus
+
+import (
+	"github.com/mudler/go-pluggable"
+)
+
+var (
+	// Package events
+
+	// EventPackageInstall is the event fired when a new package is being installed
+	EventPackageInstall pluggable.EventType = "package.install"
+	// EventPackageUnInstall is the event fired when a new package is being uninstalled
+	EventPackageUnInstall pluggable.EventType = "package.uninstall"
+
+	// Package build
+
+	// EventPackagePreBuild is the event fired before a package is being built
+	EventPackagePreBuild pluggable.EventType = "package.pre.build"
+	// EventPackagePreBuildArtifact is the event fired before a package artifact is being built
+	EventPackagePreBuildArtifact pluggable.EventType = "package.pre.build_artifact"
+	// EventPackagePostBuildArtifact is the event fired after a package artifact was built
+	EventPackagePostBuildArtifact pluggable.EventType = "package.post.build_artifact"
+	// EventPackagePostBuild is the event fired after a package was built
+	EventPackagePostBuild pluggable.EventType = "package.post.build"
+
+	// Repository events
+
+	// EventRepositoryPreBuild is the event fired before a repository is being built
+	EventRepositoryPreBuild pluggable.EventType = "repository.pre.build"
+	// EventRepositoryPostBuild is the event fired after a repository was built
+	EventRepositoryPostBuild pluggable.EventType = "repository.post.build"
+)
+
+// Manager is the bus instance manager, which subscribes plugins to events emitted by Luet
+var Manager *pluggable.Manager = pluggable.NewManager(
+	[]pluggable.EventType{
+		EventPackageInstall,
+		EventPackageUnInstall,
+		EventPackagePreBuild,
+		EventPackagePreBuildArtifact,
+		EventPackagePostBuildArtifact,
+		EventPackagePostBuild,
+		EventRepositoryPreBuild,
+		EventRepositoryPostBuild,
+	},
+)

pkg/compiler/artifact.go

@@ -18,6 +18,8 @@ package compiler
 import (
 	"archive/tar"
 	"bufio"
+	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -25,15 +27,18 @@ import (
 	"path/filepath"
 	"regexp"
 
+	system "github.com/docker/docker/pkg/system"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
 	"strings"
 	"sync"
 
+	bus "github.com/mudler/luet/pkg/bus"
 	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v2"
@@ -166,6 +171,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
 	}
 
+	bus.Manager.Publish(bus.EventPackagePreBuildArtifact, a)
+
 	mangle, err := NewPackageArtifactFromYaml(data)
 	if err != nil {
 		return errors.Wrap(err, "Generated invalid artifact")
@@ -187,6 +194,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 		return errors.Wrap(err, "While writing PackageArtifact YAML")
 	}
 	//a.CompileSpec.GetPackage().SetPath(p)
+	bus.Manager.Publish(bus.EventPackagePostBuildArtifact, a)
 
 	return nil
 }
@@ -277,8 +285,87 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	return errors.New("Compression type must be supplied")
 }
 
+func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	// If the destination path already exists I rename target file name with postfix.
+	var destPath string
+
+	// Read data. TODO: We need change archive callback to permit to return a Reader
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	// If file is not present on archive but is defined on mods
+	// I receive the callback. Prevent nil exception.
+	if header != nil {
+		switch header.Typeflag {
+		case tar.TypeReg:
+			destPath = filepath.Join(dst, path)
+		default:
+			// Nothing to do. I return original reader
+			return header, buffer.Bytes(), nil
+		}
+
+		// Check if exists
+		if helpers.Exists(destPath) {
+			for i := 1; i < 1000; i++ {
+				name := filepath.Join(filepath.Join(filepath.Dir(path),
+					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
+
+				if helpers.Exists(name) {
+					continue
+				}
+				Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
+					filepath.Join(dst, name)))
+				return &tar.Header{
+					Mode:       header.Mode,
+					Typeflag:   header.Typeflag,
+					PAXRecords: header.PAXRecords,
+					Name:       name,
+				}, buffer.Bytes(), nil
+			}
+		}
+	}
+
+	return header, buffer.Bytes(), nil
+}
+
+func (a *PackageArtifact) GetProtectFiles() []string {
+	ans := []string{}
+	annotationDir := ""
+
+	if !LuetCfg.ConfigProtectSkip {
+
+		// a.CompileSpec could be nil when artifact.Unpack is used for tree tarball
+		if a.CompileSpec != nil &&
+			a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
+			}
+		}
+		// TODO: check if skip this if we have a.CompileSpec nil
+
+		cp := NewConfigProtect(annotationDir)
+		cp.Map(a.Files)
+
+		// NOTE: for unpack we need files path without initial /
+		ans = cp.GetProtectFiles(false)
+	}
+
+	return ans
+}
+
 // Unpack Untar and decompress (TODO) to the given path
 func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+
+	// Create
+	protectedFiles := a.GetProtectFiles()
+
+	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
+
 	switch a.CompressionType {
 	case GZip:
 		// Create the uncompressed archive
@@ -307,15 +394,16 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
 		}
 
-		err = helpers.Untar(a.GetPath()+".uncompressed", dst,
-			LuetCfg.GetGeneral().SameOwner)
+		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
 		}
 		return nil
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.Untar(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner)
+		return helpers.UntarProtect(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner,
+			protectedFiles, tarModifier)
 	}
 	return errors.New("Compression type must be supplied")
 }
@@ -386,6 +474,27 @@ type CopyJob struct {
 	Artifact string
 }
 
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
@@ -399,16 +508,19 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 		// 	continue
 		// }
 
-		if !helpers.Exists(job.Dst) {
+		_, err := os.Lstat(job.Dst)
+		if err != nil {
+			Debug("Copying ", job.Src)
 			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
+			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
 
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, t CompressionImplementation) (Artifact, error) {
+func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t CompressionImplementation) (Artifact, error) {
 
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
@@ -438,7 +550,8 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 	}
 
 	// Handle includes in spec. If specified they filter what gets in the package
-	if len(includes) > 0 {
+
+	if len(includes) > 0 && len(excludes) == 0 {
 		var includeRegexp []*regexp.Regexp
 		for _, i := range includes {
 			r, e := regexp.Compile(i)
@@ -459,19 +572,108 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 					}
 				}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
+
+	} else if len(includes) == 0 && len(excludes) != 0 {
+		var excludeRegexp []*regexp.Regexp
+		for _, i := range excludes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			excludeRegexp = append(excludeRegexp, r)
+		}
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		ADD:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range excludeRegexp {
+					if i.MatchString(a.Name) {
+						continue ADD
+					}
+				}
+				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
+		}
+
+	} else if len(includes) != 0 && len(excludes) != 0 {
+
+		var includeRegexp []*regexp.Regexp
+		for _, i := range includes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			includeRegexp = append(includeRegexp, r)
+		}
+		var excludeRegexp []*regexp.Regexp
+		for _, i := range excludes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			excludeRegexp = append(excludeRegexp, r)
+		}
+
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		EXCLUDES:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) {
+						for _, e := range excludeRegexp {
+							if e.MatchString(a.Name) {
+								continue EXCLUDES
+							}
+						}
+						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+						continue EXCLUDES
+					}
+				}
+			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
+		}
+
 	} else {
 		// Otherwise just grab all
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 			for _, a := range l.Diffs.Additions {
+				Debug("File ", a.Name, " added")
 				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	}
 
 	close(toCopy)
 	wg.Wait()
+
 	a := NewPackageArtifact(dst)
 	a.SetCompressionType(t)
 	err = a.Compress(archive, concurrency)

pkg/compiler/artifact_test.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/solver"
 
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
@@ -41,7 +42,7 @@ var _ = Describe("Artifact", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -111,21 +112,25 @@ RUN echo bar > /test2`))
 			diffs, err := b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))
 			Expect(err).ToNot(HaveOccurred())
 
+			artifacts := []ArtifactNode{}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
 			Expect(diffs).To(Equal(
 				[]ArtifactLayer{{
 					FromImage: filepath.Join(tmpdir2, "output1.tar"),
 					ToImage:   filepath.Join(tmpdir, "output2.tar"),
 					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+						Additions: artifacts,
 					},
 				}}))
 			err = b.ExtractRootfs(CompilerBackendOptions{SourcePath: filepath.Join(tmpdir, "output2.tar"), Destination: rootfs}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, None)
+			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, None)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
 			err = helpers.Untar(artifact.GetPath(), unpacked, false)

pkg/compiler/backend/diff.go

@@ -0,0 +1,199 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/pkg/errors"
+)
+
+// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
+// example of json return: [
+//   {
+//     "Image1": "luet/base",
+//     "Image2": "alpine",
+//     "DiffType": "File",
+//     "Diff": {
+//       "Adds": null,
+//       "Dels": [
+//         {
+//           "Name": "/luetbuild",
+//           "Size": 5830706
+//         },
+//         {
+//           "Name": "/luetbuild/Dockerfile",
+//           "Size": 50
+//         },
+//         {
+//           "Name": "/luetbuild/output1",
+//           "Size": 5830656
+//         }
+//       ],
+//       "Mods": null
+//     }
+//   }
+// ]
+func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]compiler.ArtifactLayer, error) {
+
+	res := compiler.ArtifactLayer{FromImage: srcImage, ToImage: dstImage}
+
+	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(srcRootFS) // clean up
+
+	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(dstRootFS) // clean up
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(srcImage, "/") {
+		srcImageTar, err := ioutil.TempFile(tmpdiffs, "srctar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(srcImageTar.Name()) // clean up
+		srcImageExport := compiler.CompilerBackendOptions{
+			ImageName:   srcImage,
+			Destination: srcImageTar.Name(),
+		}
+		err = b.ExportImage(srcImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting src image "+srcImage)
+		}
+		srcImage = srcImageTar.Name()
+	}
+
+	srcImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  srcImage,
+		Destination: srcRootFS,
+	}
+	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+srcImage)
+	}
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(dstImage, "/") {
+		dstImageTar, err := ioutil.TempFile(tmpdiffs, "dsttar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(dstImageTar.Name()) // clean up
+		dstImageExport := compiler.CompilerBackendOptions{
+			ImageName:   dstImage,
+			Destination: dstImageTar.Name(),
+		}
+		err = b.ExportImage(dstImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting dst image "+dstImage)
+		}
+		dstImage = dstImageTar.Name()
+	}
+
+	dstImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  dstImage,
+		Destination: dstRootFS,
+	}
+	err = b.ExtractRootfs(dstImageExtract, false)
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+dstImage)
+	}
+
+	// Get Additions/Changes. dst -> src
+	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, dstRootFS, "", -1)
+		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
+		if err == nil {
+			var sizeA, sizeB int64
+			sizeA = fileInfo.Size()
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+
+			if sizeA != sizeB {
+				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
+				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
+					Name: filepath.Join("/", realpath),
+					Size: int(sizeB),
+				})
+			} else {
+				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
+			}
+		} else {
+			var sizeB int64
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+				Size: int(sizeB),
+			})
+
+			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+	}
+
+	// Get deletions. src -> dst
+	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, srcRootFS, "", -1)
+		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
+			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
+			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+			})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+	}
+
+	return []compiler.ArtifactLayer{res}, nil
+}

pkg/compiler/backend/diff_test.go

@@ -0,0 +1,73 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend_test
+
+import (
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Docker image diffs", func() {
+	var b compiler.CompilerBackend
+
+	BeforeEach(func() {
+		b = NewSimpleDockerBackend()
+	})
+
+	Context("Generate diffs from docker images", func() {
+		It("Detect no changes", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "alpine:latest",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, "alpine:latest", "alpine:latest")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+
+		It("Detects additions and changed files", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/micro",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			err = b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/extra",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, "quay.io/mocaccino/micro", "quay.io/mocaccino/extra")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+
+			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+	})
+})

pkg/compiler/backend/simpledocker.go

@@ -16,7 +16,6 @@
 package backend
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -90,6 +89,19 @@ func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
+func (*SimpleDocker) ImageExists(imagename string) bool {
+	buildarg := []string{"inspect", "--type=image", imagename}
+	Debug(":whale: Checking existance of docker image: " + imagename)
+	cmd := exec.Command("docker", buildarg...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		Debug("Image not present")
+		Debug(string(out))
+		return false
+	}
+	return true
+}
+
 func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
@@ -209,64 +221,9 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 	return nil
 }
 
-// 	container-diff diff daemon://luet/base alpine --type=file -j
-// [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("tmpdiffs")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-	var errorBuffer bytes.Buffer
-
-	diffargs := []string{"diff", fromImage, toImage, "-v", "error", "-q", "--type=file", "-j", "-n", "-c", tmpdiffs}
-	cmd := exec.Command("container-diff", diffargs...)
-	cmd.Stderr = &errorBuffer
-	out, err := cmd.Output()
-
-	if string(errorBuffer.Bytes()) != "" {
-		Warning("container-diff errored with: " + string(errorBuffer.Bytes()))
-	}
-
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
-	}
-
-	if config.LuetCfg.GetGeneral().ShowBuildOutput {
-		Info(string(out))
-	}
-
-	var diffs []compiler.ArtifactLayer
-
-	err = json.Unmarshal(out, &diffs)
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
-	}
+// Changes retrieves changes between image layers
+func (d *SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+	diffs, err := GenerateChanges(d, fromImage, toImage)
 
 	if config.LuetCfg.GetGeneral().Debug {
 		summary := compiler.ComputeArtifactLayerSummary(diffs)
@@ -279,5 +236,5 @@ func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLaye
 		}
 	}
 
-	return diffs, nil
+	return diffs, err
 }

pkg/compiler/backend/simpledocker_test.go

@@ -18,6 +18,7 @@ package backend_test
 import (
 	. "github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/solver"
 
 	"io/ioutil"
 	"os"
@@ -40,7 +41,7 @@ var _ = Describe("Docker backend", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -100,15 +101,19 @@ RUN echo bar > /test2`))
 			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
+			artifacts := []ArtifactNode{}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
 			Expect(b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))).To(Equal(
 				[]ArtifactLayer{{
 					FromImage: filepath.Join(tmpdir2, "output1.tar"),
 					ToImage:   filepath.Join(tmpdir, "output2.tar"),
 					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+						Additions: artifacts,
 					},
 				}}))
 

pkg/compiler/backend/simpleimg.go

@@ -97,6 +97,13 @@ func (*SimpleImg) CopyImage(src, dst string) error {
 	return nil
 }
 
+func (*SimpleImg) ImageExists(imagename string) bool {
+	// NOOP: not implemented
+	// TODO: Since img doesn't have an inspect command,
+	// we need to parse the ls output manually
+	return false
+}
+
 func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
@@ -142,8 +149,8 @@ func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms
 
 // TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
 // Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	return NewSimpleDockerBackend().Changes(fromImage, toImage)
+func (i *SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+	return GenerateChanges(i, fromImage, toImage)
 }
 
 func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {

pkg/compiler/compiler.go

@@ -20,9 +20,13 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+
 	"regexp"
 	"strings"
 	"sync"
+	"time"
+
+	bus "github.com/mudler/luet/pkg/bus"
 
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -33,6 +37,8 @@ import (
 )
 
 const BuildFile = "build.yaml"
+const DefinitionFile = "definition.yaml"
+const CollectionFile = "collection.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
@@ -43,9 +49,10 @@ type LuetCompiler struct {
 	Concurrency               int
 	CompressionType           CompressionImplementation
 	Options                   CompilerOptions
+	SolverOptions             solver.Options
 }
 
-func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions) Compiler {
+func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions, solvopts solver.Options) Compiler {
 	// The CompilerRecipe will gives us a tree with only build deps listed.
 	return &LuetCompiler{
 		Backend: backend,
@@ -60,6 +67,7 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *Compi
 		Concurrency:     opt.Concurrency,
 		Clean:           opt.Clean,
 		Options:         *opt,
+		SolverOptions:   solvopts,
 	}
 }
 
@@ -174,7 +182,7 @@ func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpec
 	return artifacts, allErrors
 }
 
-func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string) error {
+func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, include bool) error {
 	var includeRegexp []*regexp.Regexp
 	for _, i := range includes {
 		r, e := regexp.Compile(i)
@@ -206,7 +214,7 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 			}
 		}
 
-		if !match {
+		if include && !match || !include && match {
 			toRemove = append(toRemove, currentpath)
 		}
 
@@ -228,33 +236,99 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 	return nil
 }
 
-func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec) (Artifact, error) {
-	if !cs.Clean {
-		if art, err := LoadArtifactFromYaml(p); err == nil {
-			Debug("Artifact reloaded. Skipping build")
-			return art, err
-		}
+func (cs *LuetCompiler) unpackFs(rootfs string, concurrency int, p CompilationSpec) (Artifact, error) {
+	if p.GetPackageDir() != "" {
+		Info(":tophat: Packing from output dir", p.GetPackageDir())
+		rootfs = filepath.Join(rootfs, p.GetPackageDir())
+	}
+
+	if len(p.GetIncludes()) > 0 {
+		// strip from includes
+		cs.stripFromRootfs(p.GetIncludes(), rootfs, true)
+	}
+	if len(p.GetExcludes()) > 0 {
+		// strip from includes
+		cs.stripFromRootfs(p.GetExcludes(), rootfs, false)
+	}
+	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
+	artifact.SetCompressionType(cs.CompressionType)
+
+	if err := artifact.Compress(rootfs, concurrency); err != nil {
+		return nil, errors.Wrap(err, "Error met while creating package archive")
+	}
+
+	artifact.SetCompileSpec(p)
+	return artifact, nil
+}
+
+func (cs *LuetCompiler) unpackDelta(rootfs string, concurrency int, keepPermissions bool, p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions) (Artifact, error) {
+	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
+	if err := cs.Backend.ExportImage(builderOpts); err != nil {
+		return nil, errors.Wrap(err, "Could not export image")
+	}
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(builderOpts.Destination)
+	}
+	Info(pkgTag, ":hammer: Generating delta")
+	diffs, err := cs.Backend.Changes(builderOpts.Destination, runnerOpts.Destination)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not generate changes from layers")
+	}
+	artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.CompressionType)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not generate deltas")
+	}
+
+	artifact.SetCompileSpec(p)
+	return artifact, nil
+}
+
+func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
+	concurrency int, keepPermissions bool,
+	p CompilationSpec) (CompilerBackendOptions, CompilerBackendOptions, error) {
+
+	var runnerOpts, builderOpts CompilerBackendOptions
+
+	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
+
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	// TODO: As the salt contains the packageImage ( in registry/organization/imagename:tag format)
+	// the images hashes are broken with registry mirrors.
+	// We should use the image tag, or pass by the package assertion hash which is unique
+	// and identifies the deptree of the package.
+
+	fp := p.GetPackage().HashFingerprint(packageImage)
+
+	if buildertaggedImage == "" {
+		buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder"
+		Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image)
+	}
+
+	// TODO:  Cleanup, not actually hit
+	if packageImage == "" {
+		packageImage = cs.ImageRepository + "-" + fp
 	}
-	pkgTag := ":package:  " + p.GetPackage().GetName()
 
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
 	// and we build all the images first.
 
 	err := os.MkdirAll(p.Rel("build"), os.ModePerm)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for building")
+		return builderOpts, runnerOpts, errors.Wrap(err, "Error met while creating tempdir for building")
 	}
 	buildDir, err := ioutil.TempDir(p.Rel("build"), "pack")
 	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for building")
+		return builderOpts, runnerOpts, errors.Wrap(err, "Error met while creating tempdir for building")
 	}
 	defer os.RemoveAll(buildDir) // clean up
 
 	// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
 	err = helpers.CopyDir(p.GetPackage().GetPath(), buildDir)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not copy package sources")
-
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not copy package sources")
 	}
 
 	// Copy file into the build context, the compilespec might have requested to do so.
@@ -265,80 +339,78 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 	}
 
-	fp := p.GetPackage().HashFingerprint()
-	if buildertaggedImage == "" {
-		buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder"
-	}
-	if packageImage == "" {
-		packageImage = cs.ImageRepository + "-" + fp
-	}
-
-	Info(pkgTag, "Generating :whale: definition for builder image from", image)
+	Info(pkgTag, ":whale: Generating 'builder' image definition from", image)
 
 	// First we create the builder image
-	p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile"))
-	builderOpts := CompilerBackendOptions{
+	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")); err != nil {
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
+	}
+
+	// Then we write the step image, which uses the builder one
+	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")); err != nil {
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
+	}
+
+	builderOpts = CompilerBackendOptions{
 		ImageName:      buildertaggedImage,
 		SourcePath:     buildDir,
 		DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 	}
-
-	buildBuilderImage := true
-	if cs.Options.PullFirst {
-		if err := cs.Backend.DownloadImage(builderOpts); err == nil {
-			buildBuilderImage = false
-		}
-	}
-
-	if buildBuilderImage {
-		if err = cs.Backend.BuildImage(builderOpts); err != nil {
-			return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
-		}
-	}
-
-	if err = cs.Backend.ExportImage(builderOpts); err != nil {
-		return nil, errors.Wrap(err, "Could not export image")
-	}
-
-	if !cs.Options.KeepImageExport {
-		defer os.Remove(builderOpts.Destination)
-	}
-
-	if cs.Options.Push && buildBuilderImage {
-		if err = cs.Backend.Push(builderOpts); err != nil {
-			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
-		}
-	}
-	// Then we write the step image, which uses the builder one
-	p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
-	runnerOpts := CompilerBackendOptions{
+	runnerOpts = CompilerBackendOptions{
 		ImageName:      packageImage,
 		SourcePath:     buildDir,
 		DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 	}
 
-	// if !keepPackageImg {
-	// 	err = cs.Backend.ImageDefinitionToTar(runnerOpts)
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "Could not export image to tar")
-	// 	}
-	// } else {
-	buildPackageImage := true
-	if cs.Options.PullFirst {
-		//Best effort pull
-		if err := cs.Backend.DownloadImage(runnerOpts); err == nil {
-			buildPackageImage = false
+	buildAndPush := func(opts CompilerBackendOptions) error {
+		buildImage := true
+		if cs.Options.PullFirst {
+			if err := cs.Backend.DownloadImage(opts); err == nil {
+				buildImage = false
+			}
 		}
+		if buildImage {
+			if err := cs.Backend.BuildImage(opts); err != nil {
+				return errors.Wrap(err, "Could not build image: "+image+" "+opts.DockerFileName)
+			}
+			if cs.Options.Push {
+				if err = cs.Backend.Push(opts); err != nil {
+					return errors.Wrap(err, "Could not push image: "+image+" "+opts.DockerFileName)
+				}
+			}
+		}
+		return nil
 	}
 
-	if buildPackageImage {
-		if err := cs.Backend.BuildImage(runnerOpts); err != nil {
-			return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
-		}
+	if err := buildAndPush(builderOpts); err != nil {
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
 	}
 
+	if err := buildAndPush(runnerOpts); err != nil {
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
+	}
+
+	return builderOpts, runnerOpts, nil
+}
+
+func (cs *LuetCompiler) genArtifact(p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions, concurrency int, keepPermissions bool) (Artifact, error) {
+
+	// generate Artifact
+	var artifact Artifact
+	var rootfs string
+	var err error
+	unpack := p.ImageUnpack()
+	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
+
+	// If package_dir was specified in the spec, we want to treat the content of the directory
+	// as the root of our archive.  ImageUnpack is implied to be true. override it
+	if p.GetPackageDir() != "" {
+		unpack = true
+	}
+
+	// prepare folder content of the image with the package compiled inside
 	if err := cs.Backend.ExportImage(runnerOpts); err != nil {
 		return nil, errors.Wrap(err, "Failed exporting image")
 	}
@@ -347,33 +419,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		defer os.Remove(runnerOpts.Destination)
 	}
 
-	if cs.Options.Push && buildPackageImage {
-		err = cs.Backend.Push(runnerOpts)
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
-		}
-	}
-	//	}
-
-	var diffs []ArtifactLayer
-	var artifact Artifact
-	unpack := p.ImageUnpack()
-
-	// If package_dir was specified in the spec, we want to treat the content of the directory
-	// as the root of our archive.  ImageUnpack is implied to be true. override it
-	if p.GetPackageDir() != "" {
-		unpack = true
-	}
-
-	if !unpack {
-		// we have to get diffs only if spec is not unpacked
-		diffs, err = cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate changes from layers")
-		}
-	}
-
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	rootfs, err = ioutil.TempDir(p.GetOutputPath(), "rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
 	}
@@ -381,55 +427,24 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 
 	// TODO: Compression and such
 	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
-		ImageName:  packageImage,
+		ImageName:  runnerOpts.ImageName,
 		SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions)
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not extract rootfs")
 	}
 
-	if !keepImg {
-		// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
-		// TODO: Handle caching and optionally do not remove things
-		err = cs.Backend.RemoveImage(builderOpts)
-		if err != nil {
-			Warning("Could not remove image ", builderOpts.ImageName)
-			//	return nil, errors.Wrap(err, "Could not remove image")
-		}
-		err = cs.Backend.RemoveImage(runnerOpts)
-		if err != nil {
-			Warning("Could not remove image ", builderOpts.ImageName)
-			//	return nil, errors.Wrap(err, "Could not remove image")
-		}
-	}
-
 	if unpack {
-
-		if p.GetPackageDir() != "" {
-			Info(":tophat: Packing from output dir", p.GetPackageDir())
-			rootfs = filepath.Join(rootfs, p.GetPackageDir())
-		}
-
-		if len(p.GetIncludes()) > 0 {
-			// strip from includes
-			cs.stripIncludesFromRootfs(p.GetIncludes(), rootfs)
-		}
-		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
-		artifact.SetCompressionType(cs.CompressionType)
-		err = artifact.Compress(rootfs, concurrency)
+		// Take content of container as a base for our package files
+		artifact, err = cs.unpackFs(rootfs, concurrency, p)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating package archive")
 		}
-
-		artifact.SetCompileSpec(p)
 	} else {
-		Info(pkgTag, "Generating delta")
-
-		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
+		// Generate delta between the two images
+		artifact, err = cs.unpackDelta(rootfs, concurrency, keepPermissions, p, builderOpts, runnerOpts)
 		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate deltas")
+			return nil, errors.Wrap(err, "Error met while creating package archive")
 		}
-
-		artifact.SetCompileSpec(p)
 	}
 
 	filelist, err := artifact.FileList()
@@ -438,6 +453,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	}
 
 	artifact.SetFiles(filelist)
+	artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
 
 	err = artifact.WriteYaml(p.GetOutputPath())
 	if err != nil {
@@ -448,9 +464,93 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	return artifact, nil
 }
 
+func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string,
+	concurrency int,
+	keepPermissions, keepImg bool,
+	p CompilationSpec, generateArtifact bool) (Artifact, error) {
+
+	if !cs.Clean {
+		exists := cs.Backend.ImageExists(buildertaggedImage) && cs.Backend.ImageExists(packageImage)
+		if art, err := LoadArtifactFromYaml(p); err == nil && (cs.Options.SkipIfMetadataExists || exists) {
+			Debug("Artifact reloaded. Skipping build")
+			return art, err
+		}
+	}
+
+	builderOpts, runnerOpts, err := cs.buildPackageImage(image, buildertaggedImage, packageImage, concurrency, keepPermissions, p)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed building package image")
+	}
+
+	if !keepImg {
+		defer func() {
+			// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
+			if err := cs.Backend.RemoveImage(builderOpts); err != nil {
+				Warning("Could not remove image ", builderOpts.ImageName)
+			}
+			if err := cs.Backend.RemoveImage(runnerOpts); err != nil {
+				Warning("Could not remove image ", runnerOpts.ImageName)
+			}
+		}()
+	}
+
+	if !generateArtifact {
+		return &PackageArtifact{}, nil
+	}
+
+	return cs.genArtifact(p, builderOpts, runnerOpts, concurrency, keepPermissions)
+}
+
+func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) {
+	compilerSpecs := NewLuetCompilationspecs()
+
+	w := db.World()
+
+	for _, p := range w {
+		spec, err := cs.FromPackage(p)
+		if err != nil {
+			return nil, err
+		}
+		if dst != "" {
+			spec.SetOutputPath(dst)
+		}
+		compilerSpecs.Add(spec)
+	}
+
+	switch minimum {
+	case true:
+		return cs.ComputeMinimumCompilableSet(compilerSpecs.Unique().All()...)
+	default:
+		return compilerSpecs.Unique().All(), nil
+	}
+}
+
+// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
+func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error) {
+	// Generate a set with all the deps of the provided specs
+	// we will use that set to remove the deps from the list of provided compilation specs
+	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
+	result := []CompilationSpec{}
+	for _, spec := range p {
+		ass, err := cs.ComputeDepTree(spec)
+		if err != nil {
+			return result, errors.Wrap(err, "computin specs deptree")
+		}
+
+		allDependencies = append(allDependencies, ass.Drop(spec.GetPackage())...)
+	}
+
+	for _, spec := range p {
+		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
+			result = append(result, spec)
+		}
+	}
+	return result, nil
+}
+
 func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
 
-	s := solver.NewResolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
+	s := solver.NewResolver(cs.SolverOptions, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
 
 	solution, err := s.Install(pkg.Packages{p.GetPackage()})
 	if err != nil {
@@ -466,9 +566,8 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
 	for _, assertion := range dependencies { //highly dependent on the order
 		if assertion.Value {
 			nthsolution := dependencies.Cut(assertion.Package)
-
 			assertion.Hash = solver.PackageHash{
-				BuildHash:   nthsolution.Drop(assertion.Package).AssertionHash(),
+				BuildHash:   nthsolution.HashFrom(assertion.Package),
 				PackageHash: nthsolution.AssertionHash(),
 			}
 			assertions = append(assertions, assertion)
@@ -493,16 +592,25 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 
 	if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" {
 		Error("Package with no deps and no seed image supplied, bailing out")
-		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() + "with no deps and no seed image supplied, bailing out")
+		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() +
+			" with no deps and no seed image supplied, bailing out")
 	}
 
 	targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
 	targetPackageHash := cs.ImageRepository + ":" + targetAssertion.Hash.PackageHash
 
+	bus.Manager.Publish(bus.EventPackagePreBuild, struct {
+		CompileSpec CompilationSpec
+		Assert      solver.PackageAssert
+	}{
+		CompileSpec: p,
+		Assert:      *targetAssertion,
+	})
+
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p)
+		return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@@ -516,6 +624,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	depsN := 0
 	currentN := 0
 
+	packageDeps := !cs.Options.PackageTargetOnly
 	if !cs.Options.NoDeps {
 		Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
 		for _, assertion := range dependencies { //highly dependent on the order
@@ -525,8 +634,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 
 		for _, assertion := range dependencies { //highly dependent on the order
 			currentN++
-			pkgTag := fmt.Sprintf(":package:  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
-			Info(pkgTag, "   :zap:  Building dependency")
+			pkgTag := fmt.Sprintf(":package: %d/%d %s ⤑ :hammer: build %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
+			Info(pkgTag, " starts")
 			compileSpec, err := cs.FromPackage(assertion.Package)
 			if err != nil {
 				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
@@ -538,10 +647,18 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from", buildImageHash)
 			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
 
+			bus.Manager.Publish(bus.EventPackagePreBuild, struct {
+				CompileSpec CompilationSpec
+				Assert      solver.PackageAssert
+			}{
+				CompileSpec: compileSpec,
+				Assert:      assertion,
+			})
+
 			lastHash = currentPackageImageHash
 			if compileSpec.GetImage() != "" {
 				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
-				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 				if err != nil {
 					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				}
@@ -551,14 +668,23 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 			}
 
 			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
-			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 			if err != nil {
 				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				//	deperrs = append(deperrs, err)
 				//		break // stop at first error
 			}
+
+			bus.Manager.Publish(bus.EventPackagePostBuild, struct {
+				CompileSpec CompilationSpec
+				Artifact    Artifact
+			}{
+				CompileSpec: compileSpec,
+				Artifact:    artifact,
+			})
+
 			departifacts = append(departifacts, artifact)
-			Info(pkgTag, ":collision: Done")
+			Info(pkgTag, ":white_check_mark: Done")
 		}
 
 	} else if len(dependencies) > 0 {
@@ -566,20 +692,31 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	}
 
 	if !cs.Options.OnlyDeps {
-		Info(":package:", p.GetPackage().HumanReadableString(), ":cyclone:  Building package target from:", lastHash)
-		artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p)
+		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
+		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", lastHash)
+		artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 		if err != nil {
 			return artifact, err
 		}
 		artifact.SetDependencies(departifacts)
 		artifact.SetSourceAssertion(p.GetSourceAssertion())
 
+		bus.Manager.Publish(bus.EventPackagePostBuild, struct {
+			CompileSpec CompilationSpec
+			Artifact    Artifact
+		}{
+			CompileSpec: p,
+			Artifact:    artifact,
+		})
+
 		return artifact, err
 	} else {
 		return departifacts[len(departifacts)-1], nil
 	}
 }
 
+type templatedata map[string]interface{}
+
 func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 
 	pack, err := cs.Database.FindPackageCandidate(p)
@@ -587,16 +724,39 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 		return nil, err
 	}
 
-	buildFile := pack.Rel(BuildFile)
-	if !helpers.Exists(buildFile) {
-		return nil, errors.New("No build file present for " + p.GetFingerPrint())
+	var dataresult []byte
+
+	val := pack.Rel(DefinitionFile)
+	if _, err := os.Stat(pack.Rel(CollectionFile)); err == nil {
+		val = pack.Rel(CollectionFile)
+
+		data, err := ioutil.ReadFile(val)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+val)
+		}
+
+		dataBuild, err := ioutil.ReadFile(pack.Rel(BuildFile))
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+val)
+		}
+		packsRaw, err := pkg.GetRawPackages(data)
+
+		raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
+
+		dat, err := helpers.RenderHelm(string(dataBuild), raw)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
+		}
+		dataresult = []byte(dat)
+	} else {
+		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
+		}
+		dataresult = []byte(out)
 	}
 
-	dat, err := ioutil.ReadFile(buildFile)
-	if err != nil {
-		return nil, err
-	}
-	return NewLuetCompilationSpec(dat, pack)
+	return NewLuetCompilationSpec(dataresult, pack)
 }
 
 func (cs *LuetCompiler) GetBackend() CompilerBackend {

pkg/compiler/compiler_test.go

@@ -23,6 +23,7 @@ import (
 	sd "github.com/mudler/luet/pkg/compiler/backend"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -38,7 +39,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -82,7 +83,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -108,6 +109,27 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Templated packages", func() {
+		It("Renders", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/templates")
+			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+			pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec, err := compiler.FromPackage(pkg)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetImage()).To(Equal("b:bar"))
+		})
+	})
+
 	Context("Reconstruct image tree", func() {
 		It("Compiles it", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -120,7 +142,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -177,7 +199,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -219,7 +241,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -243,6 +265,146 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
+		It("Compiles and excludes files", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludes")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles includes and excludes files", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludesincludes")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+		})
+
+		It("Compiles and excludes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles includes and excludes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludeincludeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
 		It("Compiles and includes ony wanted files also from unpacked packages", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
@@ -254,7 +416,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -292,7 +454,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -333,7 +495,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -377,7 +539,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -419,7 +581,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -457,7 +619,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
 			Expect(err).ToNot(HaveOccurred())
@@ -496,7 +658,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
@@ -548,7 +710,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -591,7 +753,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -624,7 +786,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{
 				Name:     "dironly",
@@ -679,7 +841,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -706,6 +868,25 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Compilation of whole tree", func() {
+		It("doesn't include dependencies that would be compiled anyway", func() {
+			// As some specs are dependent from each other, don't pull it in if they would
+			// be eventually
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/includeimage")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(specs)).To(Equal(1))
+
+			Expect(specs[0].GetPackage().GetFingerPrint()).To(Equal("b-test-1.0"))
+		})
+	})
+
 	Context("File list", func() {
 		It("is generated after the compilation process and annotated in the metadata", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -715,7 +896,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())

pkg/compiler/interface.go

@@ -28,9 +28,10 @@ type Compiler interface {
 	CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
+	ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error)
 	SetConcurrency(i int)
 	FromPackage(pkg.Package) (CompilationSpec, error)
-
+	FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error)
 	SetBackend(CompilerBackend)
 	GetBackend() CompilerBackend
 	SetCompressionType(t CompressionImplementation)
@@ -51,9 +52,12 @@ type CompilerOptions struct {
 	Clean                    bool
 	KeepImageExport          bool
 
-	OnlyDeps      bool
-	NoDeps        bool
-	SolverOptions config.LuetSolverOptions
+	OnlyDeps             bool
+	NoDeps               bool
+	SolverOptions        config.LuetSolverOptions
+	SkipIfMetadataExists bool
+
+	PackageTargetOnly bool
 }
 
 func NewDefaultCompilerOptions() *CompilerOptions {
@@ -82,6 +86,8 @@ type CompilerBackend interface {
 	DownloadImage(opts CompilerBackendOptions) error
 
 	Push(opts CompilerBackendOptions) error
+
+	ImageExists(string) bool
 }
 
 type Artifact interface {
@@ -141,6 +147,7 @@ type ArtifactLayersSummary struct {
 type CompilationSpec interface {
 	ImageUnpack() bool // tells if the definition is just an image
 	GetIncludes() []string
+	GetExcludes() []string
 
 	RenderBuildImage() (string, error)
 	WriteBuildImageDefinition(string) error

pkg/compiler/spec.go

@@ -102,6 +102,7 @@ type LuetCompilationSpec struct {
 	OutputPath string   `json:"-"` // Where the build processfiles go
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
+	Excludes   []string `json:"excludes"`
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
@@ -148,6 +149,10 @@ func (cs *LuetCompilationSpec) GetIncludes() []string {
 	return cs.Includes
 }
 
+func (cs *LuetCompilationSpec) GetExcludes() []string {
+	return cs.Excludes
+}
+
 func (cs *LuetCompilationSpec) GetRetrieve() []string {
 	return cs.Retrieve
 }

pkg/compiler/spec_test.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -61,7 +62,7 @@ var _ = Describe("Spec", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -114,7 +115,7 @@ RUN echo bar > /test2`))
 
 		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 		spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
 		Expect(err).ToNot(HaveOccurred())
 

pkg/config/config.go

@@ -64,10 +64,11 @@ type LuetGeneralConfig struct {
 }
 
 type LuetSolverOptions struct {
-	Type        string  `mapstructure:"type"`
-	LearnRate   float32 `mapstructure:"rate"`
-	Discount    float32 `mapstructure:"discount"`
-	MaxAttempts int     `mapstructure:"max_attempts"`
+	Type           string            `mapstructure:"type"`
+	LearnRate      float32           `mapstructure:"rate"`
+	Discount       float32           `mapstructure:"discount"`
+	MaxAttempts    int               `mapstructure:"max_attempts"`
+	Implementation solver.SolverType `mapstructure:"implementation"`
 }
 
 func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
@@ -96,7 +97,7 @@ type LuetSystemConfig struct {
 	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
 }
 
-func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
 	dbpath = filepath.Join(dbpath, "repos/"+name)
 	err := os.MkdirAll(dbpath, os.ModePerm)
@@ -106,7 +107,7 @@ func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	return dbpath
 }
 
-func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+func (sc *LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
 	dbpath := filepath.Join(sc.Rootfs,
 		sc.DatabasePath)
 	err := os.MkdirAll(dbpath, os.ModePerm)
@@ -116,7 +117,7 @@ func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
 	return dbpath
 }
 
-func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
 	var cachepath string
 	if sc.PkgsCachePath != "" {
 		cachepath = sc.PkgsCachePath
@@ -134,6 +135,10 @@ func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
 	return
 }
 
+func (sc *LuetSystemConfig) GetRootFsAbs() (string, error) {
+	return filepath.Abs(sc.Rootfs)
+}
+
 type LuetRepository struct {
 	Name           string            `json:"name" yaml:"name" mapstructure:"name"`
 	Description    string            `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
@@ -200,9 +205,14 @@ type LuetConfig struct {
 	System  LuetSystemConfig  `mapstructure:"system"`
 	Solver  LuetSolverOptions `mapstructure:"solver"`
 
-	RepositoriesConfDir []string         `mapstructure:"repos_confdir"`
-	CacheRepositories   []LuetRepository `mapstructure:"repetitors"`
-	SystemRepositories  []LuetRepository `mapstructure:"repositories"`
+	RepositoriesConfDir  []string         `mapstructure:"repos_confdir"`
+	ConfigProtectConfDir []string         `mapstructure:"config_protect_confdir"`
+	ConfigProtectSkip    bool             `mapstructure:"config_protect_skip"`
+	ConfigFromHost       bool             `mapstructure:"config_from_host"`
+	CacheRepositories    []LuetRepository `mapstructure:"repetitors"`
+	SystemRepositories   []LuetRepository `mapstructure:"repositories"`
+
+	ConfigProtectConfFiles []ConfigProtectConfFile
 }
 
 func NewLuetConfig(viper *v.Viper) *LuetConfig {
@@ -211,7 +221,7 @@ func NewLuetConfig(viper *v.Viper) *LuetConfig {
 	}
 
 	GenDefault(viper)
-	return &LuetConfig{Viper: viper}
+	return &LuetConfig{Viper: viper, ConfigProtectConfFiles: nil}
 }
 
 func GenDefault(viper *v.Viper) {
@@ -231,7 +241,7 @@ func GenDefault(viper *v.Viper) {
 
 	u, err := user.Current()
 	// os/user doesn't work in from scratch environments
-	if err != nil || u.Uid == "0" {
+	if err != nil || (u != nil && u.Uid == "0") {
 		viper.SetDefault("general.same_owner", true)
 	} else {
 		viper.SetDefault("general.same_owner", false)
@@ -244,6 +254,10 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("system.pkgs_cache_path", "packages")
 
 	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
+	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
+	viper.SetDefault("config_protect_skip", false)
+	// TODO: Set default to false when we are ready for migration.
+	viper.SetDefault("config_from_host", true)
 	viper.SetDefault("cache_repositories", []string{})
 	viper.SetDefault("system_repositories", []string{})
 
@@ -273,6 +287,18 @@ func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
 	return &c.Solver
 }
 
+func (c *LuetConfig) GetConfigProtectConfFiles() []ConfigProtectConfFile {
+	return c.ConfigProtectConfFiles
+}
+
+func (c *LuetConfig) AddConfigProtectConfFile(file *ConfigProtectConfFile) {
+	if c.ConfigProtectConfFiles == nil {
+		c.ConfigProtectConfFiles = []ConfigProtectConfFile{*file}
+	} else {
+		c.ConfigProtectConfFiles = append(c.ConfigProtectConfFiles, *file)
+	}
+}
+
 func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
 	var ans *LuetRepository = nil
 

pkg/config/config_protect.go

@@ -0,0 +1,119 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+type ConfigProtectConfFile struct {
+	Filename string
+
+	Name        string   `mapstructure:"name" yaml:"name" json:"name"`
+	Directories []string `mapstructure:"dirs" yaml:"dirs" json:"dirs"`
+}
+
+func NewConfigProtectConfFile(filename string) *ConfigProtectConfFile {
+	return &ConfigProtectConfFile{
+		Filename:    filename,
+		Name:        "",
+		Directories: []string{},
+	}
+}
+
+func (c *ConfigProtectConfFile) String() string {
+	return fmt.Sprintf("[%s] filename: %s, dirs: %s", c.Name, c.Filename,
+		c.Directories)
+}
+
+type ConfigProtect struct {
+	AnnotationDir string
+	MapProtected  map[string]bool
+}
+
+func NewConfigProtect(annotationDir string) *ConfigProtect {
+	if len(annotationDir) > 0 && annotationDir[0:1] != "/" {
+		annotationDir = "/" + annotationDir
+	}
+	return &ConfigProtect{
+		AnnotationDir: annotationDir,
+		MapProtected:  make(map[string]bool, 0),
+	}
+}
+
+func (c *ConfigProtect) AddAnnotationDir(d string) {
+	c.AnnotationDir = d
+}
+
+func (c *ConfigProtect) GetAnnotationDir() string {
+	return c.AnnotationDir
+}
+
+func (c *ConfigProtect) Map(files []string) {
+	if LuetCfg.ConfigProtectSkip {
+		return
+	}
+
+	for _, file := range files {
+
+		if file[0:1] != "/" {
+			file = "/" + file
+		}
+
+		if len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
+				for _, dir := range conf.Directories {
+					// Note file is without / at begin (on unpack)
+					if strings.HasPrefix(file, filepath.Clean(dir)) {
+						// docker archive modifier works with path without / at begin.
+						c.MapProtected[file] = true
+						goto nextFile
+					}
+				}
+			}
+		}
+
+		if c.AnnotationDir != "" && strings.HasPrefix(file, filepath.Clean(c.AnnotationDir)) {
+			c.MapProtected[file] = true
+		}
+	nextFile:
+	}
+
+}
+
+func (c *ConfigProtect) Protected(file string) bool {
+	if file[0:1] != "/" {
+		file = "/" + file
+	}
+	_, ans := c.MapProtected[file]
+	return ans
+}
+
+func (c *ConfigProtect) GetProtectFiles(withSlash bool) []string {
+	ans := []string{}
+
+	for key, _ := range c.MapProtected {
+		if withSlash {
+			ans = append(ans, key)
+		} else {
+			ans = append(ans, key[1:])
+		}
+	}
+	return ans
+}

pkg/config/config_protect_test.go

@@ -0,0 +1,118 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+
+	Context("Test config protect", func() {
+
+		It("Protect1", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("/etc")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+		It("Protect2", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{},
+			))
+		})
+
+		It("Protect3: Annotation dir without initial slash", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("etc")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+	})
+
+})

pkg/helpers/archive.go

@@ -17,6 +17,7 @@ package helpers
 
 import (
 	"archive/tar"
+	"bytes"
 	"io"
 	"os"
 	"path/filepath"
@@ -49,6 +50,146 @@ func Tar(src, dest string) error {
 	return err
 }
 
+type TarModifierWrapperFunc func(path, dst string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
+type TarModifierWrapper struct {
+	DestinationPath string
+	Modifier        TarModifierWrapperFunc
+}
+
+func NewTarModifierWrapper(dst string, modifier TarModifierWrapperFunc) *TarModifierWrapper {
+	return &TarModifierWrapper{
+		DestinationPath: dst,
+		Modifier:        modifier,
+	}
+}
+
+func (m *TarModifierWrapper) GetModifier() archive.TarModifierFunc {
+	return func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+		return m.Modifier(m.DestinationPath, path, header, content)
+	}
+}
+
+func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modifier *TarModifierWrapper) error {
+	var ans error
+
+	if len(protectedFiles) <= 0 {
+		return Untar(src, dst, sameOwner)
+	}
+
+	// POST: we have files to protect. I create a ReplaceFileTarWrapper
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	// Create modifier map
+	mods := make(map[string]archive.TarModifierFunc)
+	for _, file := range protectedFiles {
+		mods[file] = modifier.GetModifier()
+	}
+
+	if sameOwner {
+		// PRE: i have root privileged.
+
+		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
+
+		opts := &archive.TarOptions{
+			// NOTE: NoLchown boolean is used for chmod of the symlink
+			// Probably it's needed set this always to true.
+			NoLchown:        true,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+			ContinueOnError: true,
+		}
+
+		ans = archive.Untar(replacerArchive, dst, opts)
+	} else {
+		ans = unTarIgnoreOwner(dst, in, mods)
+	}
+
+	return ans
+}
+
+func unTarIgnoreOwner(dest string, in io.ReadCloser, mods map[string]archive.TarModifierFunc) error {
+	tr := tar.NewReader(in)
+	for {
+		header, err := tr.Next()
+
+		var data []byte
+		var headerReplaced = false
+
+		switch {
+		case err == io.EOF:
+			goto tarEof
+		case err != nil:
+			return err
+		case header == nil:
+			continue
+		}
+
+		// the target location where the dir/file should be created
+		target := filepath.Join(dest, header.Name)
+		if mods != nil {
+			modifier, ok := mods[header.Name]
+			if ok {
+				header, data, err = modifier(header.Name, header, tr)
+				if err != nil {
+					return err
+				}
+
+				// Override target path
+				target = filepath.Join(dest, header.Name)
+				headerReplaced = true
+			}
+
+		}
+
+		// Check the file type
+		switch header.Typeflag {
+
+		// if its a dir and it doesn't exist create it
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return err
+				}
+			}
+
+			// handle creation of file
+		case tar.TypeReg:
+
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+
+			// copy over contents
+			if headerReplaced {
+				_, err = io.Copy(f, bytes.NewReader(data))
+			} else {
+				_, err = io.Copy(f, tr)
+			}
+			if err != nil {
+				return err
+			}
+
+			// manually close here after each file operation; defering would cause each
+			// file close to wait until all operations have completed.
+			f.Close()
+
+		case tar.TypeSymlink:
+			source := header.Linkname
+			err := os.Symlink(source, target)
+			if err != nil {
+				return err
+			}
+		}
+	}
+tarEof:
+
+	return nil
+}
+
 // Untar just a wrapper around the docker functions
 func Untar(src, dest string, sameOwner bool) error {
 	var ans error
@@ -72,62 +213,7 @@ func Untar(src, dest string, sameOwner bool) error {
 
 		ans = archive.Untar(in, dest, opts)
 	} else {
-
-		// TODO: replace with https://github.com/mholt/archiver ?
-		var fileReader io.ReadCloser = in
-
-		tr := tar.NewReader(fileReader)
-		for {
-			header, err := tr.Next()
-
-			switch {
-			case err == io.EOF:
-				goto tarEof
-			case err != nil:
-				return err
-			case header == nil:
-				continue
-			}
-
-			// the target location where the dir/file should be created
-			target := filepath.Join(dest, header.Name)
-
-			// Check the file type
-			switch header.Typeflag {
-
-			// if its a dir and it doesn't exist create it
-			case tar.TypeDir:
-				if _, err := os.Stat(target); err != nil {
-					if err := os.MkdirAll(target, 0755); err != nil {
-						return err
-					}
-				}
-
-				// handle creation of file
-			case tar.TypeReg:
-				f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
-				if err != nil {
-					return err
-				}
-
-				// copy over contents
-				if _, err := io.Copy(f, tr); err != nil {
-					return err
-				}
-
-				// manually close here after each file operation; defering would cause each
-				// file close to wait until all operations have completed.
-				f.Close()
-
-			case tar.TypeSymlink:
-				source := header.Linkname
-				err := os.Symlink(source, target)
-				if err != nil {
-					return err
-				}
-			}
-		}
-	tarEof:
+		ans = unTarIgnoreOwner(dest, in, nil)
 	}
 
 	return ans

pkg/helpers/archive_test.go

@@ -0,0 +1,134 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/archive"
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// Code from moby/moby pkg/archive/archive_test
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(filepath.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+		if makeLinks {
+			if err := os.Link(filepath.Join(targetPath, fileName), filepath.Join(targetPath, fileName+"-link")); err != nil {
+				return 0, err
+			}
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}
+
+func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	// If the destination path already exists I rename target file name with postfix.
+	var basePath string
+
+	// Read data. TODO: We need change archive callback to permit to return a Reader
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	if header != nil {
+
+		switch header.Typeflag {
+		case tar.TypeReg:
+			basePath = filepath.Base(path)
+		default:
+			// Nothing to do. I return original reader
+			return header, buffer.Bytes(), nil
+		}
+
+		if basePath == "file-0" {
+			name := filepath.Join(filepath.Join(filepath.Dir(path), fmt.Sprintf("._cfg%04d_%s", 1, basePath)))
+			return &tar.Header{
+				Mode:       header.Mode,
+				Typeflag:   header.Typeflag,
+				PAXRecords: header.PAXRecords,
+				Name:       name,
+			}, buffer.Bytes(), nil
+		} else if basePath == "file-1" {
+			return header, []byte("newcontent"), nil
+		}
+
+		// else file not present
+	}
+
+	return header, buffer.Bytes(), nil
+}
+
+var _ = Describe("Helpers Archive", func() {
+	Context("Untar Protect", func() {
+
+		It("Detect existing and not-existing files", func() {
+
+			archiveSourceDir, err := ioutil.TempDir("", "archive-source")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(archiveSourceDir)
+
+			_, err = prepareUntarSourceDirectory(10, archiveSourceDir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			targetDir, err := ioutil.TempDir("", "archive-target")
+			Expect(err).ToNot(HaveOccurred())
+			//	defer os.RemoveAll(targetDir)
+
+			sourceArchive, err := archive.TarWithOptions(archiveSourceDir, &archive.TarOptions{})
+			Expect(err).ToNot(HaveOccurred())
+			defer sourceArchive.Close()
+
+			tarModifier := NewTarModifierWrapper(targetDir, tarModifierWrapperFunc)
+			mods := make(map[string]archive.TarModifierFunc)
+			mods["file-0"] = tarModifier.GetModifier()
+			mods["file-1"] = tarModifier.GetModifier()
+			mods["file-9999"] = tarModifier.GetModifier()
+
+			replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
+			//replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
+			opts := &archive.TarOptions{
+				// NOTE: NoLchown boolean is used for chmod of the symlink
+				// Probably it's needed set this always to true.
+				NoLchown:        true,
+				ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+				ContinueOnError: true,
+			}
+
+			err = archive.Untar(replacerArchive, targetDir, opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
+		})
+	})
+})

pkg/helpers/file.go

@@ -24,6 +24,37 @@ import (
 	copy "github.com/otiai10/copy"
 )
 
+func OrderFiles(target string, files []string) ([]string, []string) {
+
+	var newFiles []string
+	var notPresent []string
+
+	for _, f := range files {
+		target := filepath.Join(target, f)
+		fi, err := os.Lstat(target)
+		if err != nil {
+			notPresent = append(notPresent, f)
+			continue
+		}
+		if m := fi.Mode(); !m.IsDir() {
+			newFiles = append(newFiles, f)
+		}
+	}
+
+	for _, f := range files {
+		target := filepath.Join(target, f)
+		fi, err := os.Lstat(target)
+		if err != nil {
+			continue
+		}
+		if m := fi.Mode(); m.IsDir() {
+			newFiles = append(newFiles, f)
+		}
+	}
+
+	return newFiles, notPresent
+}
+
 func ListDir(dir string) ([]string, error) {
 	content := []string{}
 
@@ -93,7 +124,7 @@ func ensureDir(fileName string) {
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
 func CopyFile(src, dst string) (err error) {
-	return copy.Copy(src, dst)
+	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }
 
 func IsDirectory(path string) (bool, error) {
@@ -110,5 +141,5 @@ func IsDirectory(path string) (bool, error) {
 func CopyDir(src string, dst string) (err error) {
 	src = filepath.Clean(src)
 	dst = filepath.Clean(dst)
-	return copy.Copy(src, dst)
+	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }

pkg/helpers/file_test.go

@@ -16,6 +16,10 @@
 package helpers_test
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -28,4 +32,33 @@ var _ = Describe("Helpers", func() {
 			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
 		})
 	})
+
+	Context("Orders dir and files correctly", func() {
+		It("puts files first and folders at end", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			err = ioutil.WriteFile(filepath.Join(testDir, "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(testDir, "baz"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "bar"), 0755)
+			Expect(err).ToNot(HaveOccurred())
+			err = ioutil.WriteFile(filepath.Join(testDir, "bar", "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "baz2"), 0755)
+			Expect(err).ToNot(HaveOccurred())
+			err = ioutil.WriteFile(filepath.Join(testDir, "baz2", "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			ordered, notExisting := OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
+
+			Expect(ordered).To(Equal([]string{"baz", "bar/foo", "foo", "baz2/foo", "bar", "baz2"}))
+			Expect(notExisting).To(Equal([]string{"notexisting"}))
+		})
+	})
 })

pkg/helpers/helm.go

@@ -0,0 +1,59 @@
+package helpers
+
+import (
+	"io/ioutil"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+)
+
+// RenderHelm renders the template string with helm
+func RenderHelm(template string, values map[string]interface{}) (string, error) {
+	c := &chart.Chart{
+		Metadata: &chart.Metadata{
+			Name:    "",
+			Version: "",
+		},
+		Templates: []*chart.File{
+			{Name: "templates", Data: []byte(template)},
+		},
+		Values: map[string]interface{}{"Values": values},
+	}
+
+	v, err := chartutil.CoalesceValues(c, map[string]interface{}{})
+	if err != nil {
+		return "", errors.Wrap(err, "while rendering template")
+	}
+	out, err := engine.Render(c, v)
+	if err != nil {
+		return "", errors.Wrap(err, "while rendering template")
+	}
+
+	return out["templates"], nil
+}
+
+type templatedata map[string]interface{}
+
+func RenderFiles(toTemplate, valuesFile string) (string, error) {
+	raw, err := ioutil.ReadFile(toTemplate)
+	if err != nil {
+		return "", errors.Wrap(err, "reading file "+toTemplate)
+	}
+
+	if !Exists(valuesFile) {
+		return "", errors.Wrap(err, "file not existing "+valuesFile)
+	}
+	def, err := ioutil.ReadFile(valuesFile)
+	if err != nil {
+		return "", errors.Wrap(err, "reading file "+valuesFile)
+	}
+
+	var values templatedata
+	if err = yaml.Unmarshal(def, &values); err != nil {
+		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
+	}
+	return RenderHelm(string(raw), values)
+}

pkg/helpers/helm_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Helpers", func() {
+	Context("RenderHelm", func() {
+		It("Renders templates", func() {
+			out, err := RenderHelm("{{.Values.Test}}", map[string]interface{}{"Test": "foo"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foo"))
+		})
+	})
+})

pkg/helpers/match.go

@@ -0,0 +1,49 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"regexp"
+)
+
+func MapMatchRegex(m *map[string]string, r *regexp.Regexp) bool {
+	ans := false
+
+	if m != nil {
+		for k, v := range *m {
+			if r.MatchString(k + "=" + v) {
+				ans = true
+				break
+			}
+		}
+	}
+
+	return ans
+}
+
+func MapHasKey(m *map[string]string, label string) bool {
+	ans := false
+	if m != nil {
+		for k, _ := range *m {
+			if k == label {
+				ans = true
+				break
+			}
+		}
+	}
+	return ans
+}

pkg/helpers/sys.go

@@ -16,7 +16,9 @@
 package helpers
 
 import (
+	"os"
 	"os/exec"
+	"os/user"
 	"syscall"
 
 	"github.com/pkg/errors"
@@ -30,3 +32,17 @@ func Exec(cmd string, args []string, env []string) error {
 	}
 	return syscall.Exec(path, args, env)
 }
+
+func GetHomeDir() (ans string) {
+	// os/user doesn't work in from scratch environments
+	u, err := user.Current()
+	if err == nil {
+		ans = u.HomeDir
+	} else {
+		ans = ""
+	}
+	if os.Getenv("HOME") != "" {
+		ans = os.Getenv("HOME")
+	}
+	return ans
+}

pkg/installer/client/local.go

@@ -38,15 +38,26 @@ func NewLocalClient(r RepoData) *LocalClient {
 func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
 	var err error
 
+	rootfs := ""
 	artifactName := path.Base(artifact.GetPath())
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 
+	if !config.LuetCfg.ConfigFromHost {
+		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Check if file is already in cache
 	if helpers.Exists(cacheFile) {
 		Info("Use artifact", artifactName, "from cache.")
 	} else {
 		ok := false
 		for _, uri := range c.RepoData.Urls {
+
+			uri = filepath.Join(rootfs, uri)
+
 			Info("Downloading artifact", artifactName, "from", uri)
 
 			//defer os.Remove(file.Name())
@@ -72,8 +83,20 @@ func (c *LocalClient) DownloadFile(name string) (string, error) {
 	var err error
 	var file *os.File = nil
 
+	rootfs := ""
+
+	if !config.LuetCfg.ConfigFromHost {
+		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return "", err
+		}
+	}
+
 	ok := false
 	for _, uri := range c.RepoData.Urls {
+
+		uri = filepath.Join(rootfs, uri)
+
 		Info("Downloading file", name, "from", uri)
 		file, err = config.LuetCfg.GetSystem().TempFile("localclient")
 		if err != nil {

pkg/installer/config_protect.go

@@ -0,0 +1,100 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"io/ioutil"
+	"path"
+	"path/filepath"
+	"regexp"
+
+	"github.com/ghodss/yaml"
+
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+func LoadConfigProtectConfs(c *LuetConfig) error {
+	var regexConfs = regexp.MustCompile(`.yml$`)
+	var err error
+
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, cdir := range c.ConfigProtectConfDir {
+		cdir = filepath.Join(rootfs, cdir)
+
+		Debug("Parsing Config Protect Directory", cdir, "...")
+
+		files, err := ioutil.ReadDir(cdir)
+		if err != nil {
+			Debug("Skip dir", cdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexConfs.MatchString(file.Name()) {
+				Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(cdir, file.Name()))
+			if err != nil {
+				Warning("On read file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := LoadConfigProtectConFile(file.Name(), content)
+			if err != nil {
+				Warning("On parse file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Directories) == 0 {
+				Warning("Invalid config protect file", file.Name())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddConfigProtectConfFile(r)
+		}
+	}
+	return nil
+
+}
+
+func LoadConfigProtectConFile(filename string, data []byte) (*ConfigProtectConfFile, error) {
+	ans := NewConfigProtectConfFile(filename)
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}

pkg/installer/finalizer.go

@@ -47,7 +47,7 @@ func (f *LuetFinalizer) RunInstall(s *System) error {
 
 	for _, c := range f.Install {
 		toRun := append(args, c)
-		Info("Executing finalizer on ", s.Target, cmd, toRun)
+		Info(":shell: Executing finalizer on ", s.Target, cmd, toRun)
 		if s.Target == "/" {
 			cmd := exec.Command(cmd, toRun...)
 			stdoutStderr, err := cmd.CombinedOutput()

pkg/installer/installer.go

@@ -16,6 +16,7 @@
 package installer
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -23,26 +24,27 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/mudler/luet/pkg/bus"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
-	"github.com/mudler/luet/pkg/tree"
 
 	"github.com/pkg/errors"
 )
 
 type LuetInstallerOptions struct {
-	SolverOptions               config.LuetSolverOptions
-	Concurrency                 int
-	NoDeps                      bool
-	OnlyDeps                    bool
-	Force                       bool
-	PreserveSystemEssentialData bool
-	FullUninstall               bool
-	CheckConflicts              bool
+	SolverOptions                                                  config.LuetSolverOptions
+	Concurrency                                                    int
+	NoDeps                                                         bool
+	OnlyDeps                                                       bool
+	Force                                                          bool
+	PreserveSystemEssentialData                                    bool
+	FullUninstall, FullCleanUninstall                              bool
+	CheckConflicts                                                 bool
+	SolverUpgrade, RemoveUnavailableOnUpgrade, UpgradeNewRevisions bool
 }
 
 type LuetInstaller struct {
@@ -66,24 +68,88 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 	if err != nil {
 		return err
 	}
+
+	Info(":thinking: Computing upgrade, please hang tight... :zzz:")
+	if l.Options.UpgradeNewRevisions {
+		Info(":memo: note: will consider new build revisions while upgrading")
+	}
+	Spinner(32)
+	defer SpinnerStop()
 	// First match packages against repositories by priority
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	// compute a "big" world
-	solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
-	uninstall, solution, err := solv.Upgrade(false)
-	if err != nil {
-		return errors.Wrap(err, "Failed solving solution for upgrade")
+	solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+	var uninstall pkg.Packages
+	var solution solver.PackagesAssertions
+
+	if l.Options.SolverUpgrade {
+
+		uninstall, solution, err = solv.UpgradeUniverse(l.Options.RemoveUnavailableOnUpgrade)
+		if err != nil {
+			return errors.Wrap(err, "Failed solving solution for upgrade")
+		}
+	} else {
+		uninstall, solution, err = solv.Upgrade(!l.Options.FullUninstall, l.Options.NoDeps)
+		if err != nil {
+			return errors.Wrap(err, "Failed solving solution for upgrade")
+		}
+
+	}
+	SpinnerStop()
+
+	if len(uninstall) > 0 {
+		Info(":recycle: Packages marked for uninstall:")
+	}
+
+	for _, p := range uninstall {
+		Info(fmt.Sprintf("- %s", p.HumanReadableString()))
+	}
+
+	if len(solution) > 0 {
+		Info(":zap: Packages marked for upgrade:")
 	}
 
 	toInstall := pkg.Packages{}
 	for _, assertion := range solution {
 		// Be sure to filter from solutions packages already installed in the system
 		if _, err := s.Database.FindPackage(assertion.Package); err != nil && assertion.Value {
+			Info(fmt.Sprintf("- %s", assertion.Package.HumanReadableString()))
 			toInstall = append(toInstall, assertion.Package)
 		}
 	}
 
+	if l.Options.UpgradeNewRevisions {
+		Info(":mag: Checking packages with new revisions available")
+		for _, p := range s.Database.World() {
+			matches := syncedRepos.PackageMatches(pkg.Packages{p})
+			if len(matches) == 0 {
+				// Package missing. the user should run luet upgrade --universe
+				Info(":warning: Installed packages seems to be missing from remote repositories.")
+				Info(":warning: It is suggested to run 'luet upgrade --universe'")
+				continue
+			}
+			for _, artefact := range matches[0].Repo.GetIndex() {
+				if artefact.GetCompileSpec().GetPackage() == nil {
+					return errors.New("Package in compilespec empty")
+
+				}
+				if artefact.GetCompileSpec().GetPackage().Matches(p) && artefact.GetCompileSpec().GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
+					toInstall = append(toInstall, matches[0].Package).Unique()
+					uninstall = append(uninstall, p).Unique()
+					Info(
+						fmt.Sprintf("- %s ( %s vs %s ) repo: %s (date: %s)",
+							p.HumanReadableString(),
+							artefact.GetCompileSpec().GetPackage().GetBuildTimestamp(),
+							p.GetBuildTimestamp(),
+							matches[0].Repo.GetName(),
+							matches[0].Repo.GetLastUpdate(),
+						))
+				}
+			}
+		}
+	}
+
 	return l.swap(syncedRepos, uninstall, toInstall, s)
 }
 
@@ -229,7 +295,7 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 					if err != nil {
 						return err
 					}
-					Info("Found package:", p.HumanReadableString())
+					Info(":mag: Found package:", p.HumanReadableString())
 					toMerge = append(toMerge, ArtifactMatch{Artifact: artefact, Package: p})
 					break FILES
 				}
@@ -250,7 +316,7 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 			return errors.Wrap(err, "Failed creating package")
 		}
 		s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.GetFiles()})
-		Info("Reclaimed package:", pack.HumanReadableString())
+		Info(":zap: Reclaimed package:", pack.HumanReadableString())
 	}
 	Info("Done!")
 
@@ -293,30 +359,39 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 	var solution solver.PackagesAssertions
 
 	if !l.Options.NoDeps {
-		solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		Info(":deciduous_tree: Computing installation, hang tight")
+		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		solution, err = solv.Install(p)
+		/// TODO: PackageAssertions needs to be a map[fingerprint]pack so lookup is in O(1)
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Failed solving solution for package")
 		}
+		Info(":deciduous_tree: Finished calculating dependencies")
 		// Gathers things to install
+		Info(":deciduous_tree: Checking for packages already installed, and prepare for installation")
 		for _, assertion := range solution {
 			if assertion.Value {
+				if _, err := s.Database.FindPackage(assertion.Package); err == nil {
+					// skip matching if it is installed already
+					continue
+				}
 				packagesToInstall = append(packagesToInstall, assertion.Package)
 			}
 		}
 	} else if !l.Options.OnlyDeps {
 		for _, currentPack := range p {
+			if _, err := s.Database.FindPackage(currentPack); err == nil {
+				// skip matching if it is installed already
+				continue
+			}
 			packagesToInstall = append(packagesToInstall, currentPack)
 		}
 	}
-	Info(":deciduous_tree: Finding packages to install")
+	Info(":deciduous_tree: Finding packages to install from :cloud:")
 	// Gathers things to install
 	for _, currentPack := range packagesToInstall {
 		// Check if package is already installed.
-		if _, err := s.Database.FindPackage(currentPack); err == nil {
-			// skip matching if it is installed already
-			continue
-		}
+
 		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
 		if len(matches) == 0 {
 			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
@@ -328,10 +403,11 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 
 			}
 			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+				currentPack.SetBuildTimestamp(artefact.GetCompileSpec().GetPackage().GetBuildTimestamp())
 				// Filter out already installed
 				if _, err := s.Database.FindPackage(currentPack); err != nil {
 					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
-					Info("\t:package:", currentPack.HumanReadableString(), "from repository", matches[0].Repo.GetName())
+					Info("\t:package:", currentPack.HumanReadableString(), ":cloud:", matches[0].Repo.GetName())
 				}
 				break A
 			}
@@ -376,9 +452,9 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Failed creating package")
 		}
+		bus.Manager.Publish(bus.EventPackageInstall, c)
 	}
-	executedFinalizer := map[string]bool{}
-
+	var toFinalize []pkg.Package
 	if !l.Options.NoDeps {
 		// TODO: Lower those errors as warning
 		for _, w := range p {
@@ -390,36 +466,17 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 		ORDER:
 			for _, ass := range ordered {
 				if ass.Value {
-
 					installed, ok := toInstall[ass.Package.GetFingerPrint()]
 					if !ok {
 						// It was a dep already installed in the system, so we can skip it safely
 						continue ORDER
 					}
-
 					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
 					if err != nil {
 						return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
 					}
-					if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
-						Info("Executing finalizer for " + ass.Package.HumanReadableString())
-						finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
-						if err != nil && !l.Options.Force {
-							return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
-						}
-						if _, exists := executedFinalizer[ass.Package.GetFingerPrint()]; !exists {
-							finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
-							if err != nil && !l.Options.Force {
-								return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
-							}
-							err = finalizer.RunInstall(s)
-							if err != nil && !l.Options.Force {
-								return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
-							}
-							executedFinalizer[ass.Package.GetFingerPrint()] = true
-						}
-					}
 
+					toFinalize = append(toFinalize, treePackage)
 				}
 			}
 
@@ -430,29 +487,11 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 			if err != nil {
 				return errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
 			}
-			if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
-				Info("Executing finalizer for " + c.Package.HumanReadableString())
-				finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
-				if err != nil && !l.Options.Force {
-					return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
-				}
-				if _, exists := executedFinalizer[c.Package.GetFingerPrint()]; !exists {
-					finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
-					if err != nil && !l.Options.Force {
-						return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
-					}
-					err = finalizer.RunInstall(s)
-					if err != nil && !l.Options.Force {
-						return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
-					}
-					executedFinalizer[c.Package.GetFingerPrint()] = true
-				}
-			}
+			toFinalize = append(toFinalize, treePackage)
 		}
 	}
 
-	return nil
-
+	return s.ExecuteFinalizers(toFinalize, l.Options.Force)
 }
 
 func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) {
@@ -534,16 +573,39 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 }
 
 func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
+	var cp *config.ConfigProtect
+	annotationDir := ""
+
 	files, err := s.Database.GetPackageFiles(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed getting installed files")
 	}
 
-	// Remove from target
-	for _, f := range files {
-		target := filepath.Join(s.Target, f)
-		Debug("Removing", target)
+	if !config.LuetCfg.ConfigProtectSkip {
 
+		if p.HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := p.GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
+			}
+		}
+
+		cp = config.NewConfigProtect(annotationDir)
+		cp.Map(files)
+	}
+
+	toRemove, notPresent := helpers.OrderFiles(s.Target, files)
+
+	// Remove from target
+	for _, f := range toRemove {
+		target := filepath.Join(s.Target, f)
+
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Preserving protected file:", f)
+			continue
+		}
+
+		Debug("Removing", target)
 		if l.Options.PreserveSystemEssentialData &&
 			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) ||
 			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath()) {
@@ -551,11 +613,41 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 			continue
 		}
 
-		err := os.Remove(target)
+		fi, err := os.Lstat(target)
 		if err != nil {
-			Warning("Failed removing file (not present in the system target ?)", target)
+			Warning("File not found (it was before?) ", err.Error())
+			continue
+		}
+		switch mode := fi.Mode(); {
+		case mode.IsDir():
+			files, err := ioutil.ReadDir(target)
+			if err != nil {
+				Warning("Failed reading folder", target, err.Error())
+			}
+			if len(files) != 0 {
+				Debug("Preserving not-empty folder", target)
+				continue
+			}
+		}
+
+		if err = os.Remove(target); err != nil {
+			Warning("Failed removing file (maybe not present in the system target anymore ?)", target, err.Error())
 		}
 	}
+
+	for _, f := range notPresent {
+		target := filepath.Join(s.Target, f)
+
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Preserving protected file:", f)
+			continue
+		}
+
+		if err = os.Remove(target); err != nil {
+			Debug("Failed removing file (not present in the system target)", target, err.Error())
+		}
+	}
+
 	err = s.Database.RemovePackageFiles(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed removing package files from database")
@@ -565,7 +657,9 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		return errors.Wrap(err, "Failed removing package from database")
 	}
 
-	Info(p.GetFingerPrint(), "Removed")
+	bus.Manager.Publish(bus.EventPackageUnInstall, p)
+
+	Info(":recycle:", p.GetFingerPrint(), "Removed :heavy_check_mark:")
 	return nil
 }
 
@@ -573,7 +667,7 @@ func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
 	Spinner(32)
 	defer SpinnerStop()
 
-	Info("Uninstalling :package:", p.HumanReadableString(), "hang tight")
+	Info(":recycle: Uninstalling :package:", p.HumanReadableString(), "hang tight")
 
 	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) TODO - mark the uninstallation in db
 	// Get installed definition
@@ -596,26 +690,36 @@ func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
 	}
 
 	if !l.Options.NoDeps {
-		Info("Finding :package:", p.HumanReadableString(), "dependency graph :deciduous_tree:")
-		solv := solver.NewResolver(installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
-		solution, err := solv.Uninstall(p, checkConflicts, full)
-		if err != nil && !l.Options.Force {
-			return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+		Info(":mag: Finding :package:", p.HumanReadableString(), "dependency graph :deciduous_tree:")
+		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		var solution pkg.Packages
+		var err error
+		if l.Options.FullCleanUninstall {
+			solution, err = solv.UninstallUniverse(pkg.Packages{p})
+			if err != nil {
+				return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+			}
+		} else {
+			solution, err = solv.Uninstall(p, checkConflicts, full)
+			if err != nil && !l.Options.Force {
+				return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+			}
 		}
+
 		for _, p := range solution {
-			Info("Uninstalling", p.HumanReadableString())
+			Info(":recycle: Uninstalling", p.HumanReadableString())
 			err := l.uninstall(p, s)
 			if err != nil && !l.Options.Force {
 				return errors.Wrap(err, "Uninstall failed")
 			}
 		}
 	} else {
-		Info("Uninstalling", p.HumanReadableString(), "without deps")
+		Info(":recycle: Uninstalling", p.HumanReadableString(), "without deps")
 		err := l.uninstall(p, s)
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Uninstall failed")
 		}
-		Info(":package:", p.HumanReadableString(), "uninstalled")
+		Info(":recycle: :package:", p.HumanReadableString(), "uninstalled :heavy_check_mark:")
 	}
 	return nil
 

pkg/installer/installer_test.go

@@ -24,6 +24,8 @@ import (
 	compiler "github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/helpers"
+	solver "github.com/mudler/luet/pkg/solver"
+
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
@@ -47,7 +49,7 @@ var _ = Describe("Installer", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -163,7 +165,7 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
 			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
-				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -281,7 +283,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -399,7 +401,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -486,7 +488,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -545,7 +547,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -663,8 +665,8 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 			Expect(len(generalRecipeNewRepo.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
-			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -795,7 +797,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -912,7 +914,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1017,7 +1019,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1109,7 +1111,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(3))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
 			Expect(err).ToNot(HaveOccurred())

pkg/installer/interface.go

@@ -70,6 +70,6 @@ type Repository interface {
 	SetPriority(int)
 	GetRepositoryFile(string) (LuetRepositoryFile, error)
 	SetRepositoryFile(string, LuetRepositoryFile)
-
+	SetName(p string)
 	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
 }

pkg/installer/repository.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mudler/luet/pkg/bus"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
@@ -265,6 +266,8 @@ func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact
 		// We want to include packages that are ONLY referenced in the tree.
 		// the ones which aren't should be deleted. (TODO: by another cli command?)
 		if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
+			Info(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				artifact.GetCompileSpec().GetPackage().HumanReadableString()))
 			return nil
 		}
 
@@ -302,6 +305,11 @@ func (r *LuetSystemRepository) GetType() string {
 func (r *LuetSystemRepository) SetType(p string) {
 	r.LuetRepository.Type = p
 }
+
+func (r *LuetSystemRepository) SetName(p string) {
+	r.LuetRepository.Name = p
+}
+
 func (r *LuetSystemRepository) AddUrl(p string) {
 	r.LuetRepository.Urls = append(r.LuetRepository.Urls, p)
 }
@@ -419,6 +427,14 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 		r.Name, r.Revision, r.LastUpdate,
 	))
 
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
+
 	// Create tree and repository file
 	archive, err := config.LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
@@ -499,6 +515,14 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 		return err
 	}
 
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
+
 	return nil
 }
 
@@ -685,6 +709,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	repo.SetAuthentication(r.GetAuthentication())
 	repo.SetType(r.GetType())
 	repo.SetPriority(r.GetPriority())
+	repo.SetName(r.GetName())
 	InfoC(
 		aurora.Bold(
 			aurora.Yellow(":information_source: Repository "+repo.GetName()+" priority: ")).String() +

pkg/installer/repository_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -48,7 +49,7 @@ var _ = Describe("Repository", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -116,11 +117,11 @@ var _ = Describe("Repository", func() {
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())

pkg/installer/system.go

@@ -1,7 +1,12 @@
 package installer
 
 import (
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	"github.com/pkg/errors"
 )
 
 type System struct {
@@ -12,3 +17,31 @@ type System struct {
 func (s *System) World() (pkg.Packages, error) {
 	return s.Database.World(), nil
 }
+
+type templatedata map[string]interface{}
+
+func (s *System) ExecuteFinalizers(packs []pkg.Package, force bool) error {
+	executedFinalizer := map[string]bool{}
+	for _, p := range packs {
+		if helpers.Exists(p.Rel(tree.FinalizerFile)) {
+			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile))
+			if err != nil && !force {
+				return errors.Wrap(err, "reading file "+p.Rel(tree.FinalizerFile))
+			}
+
+			if _, exists := executedFinalizer[p.GetFingerPrint()]; !exists {
+				Info("Executing finalizer for " + p.HumanReadableString())
+				finalizer, err := NewLuetFinalizerFromYaml([]byte(out))
+				if err != nil && !force {
+					return errors.Wrap(err, "Error reading finalizer "+p.Rel(tree.FinalizerFile))
+				}
+				err = finalizer.RunInstall(s)
+				if err != nil && !force {
+					return errors.Wrap(err, "Error executing install finalizer "+p.Rel(tree.FinalizerFile))
+				}
+				executedFinalizer[p.GetFingerPrint()] = true
+			}
+		}
+	}
+	return nil
+}

pkg/logger/logger.go

@@ -78,23 +78,25 @@ func Spinner(i int) {
 		i = 43
 	}
 
-	if !s.Active() {
+	if s != nil && !s.Active() {
 		//	s.UpdateCharSet(spinner.CharSets[i])
 		s.Start() // Start the spinner
 	}
 }
 
 func SpinnerText(suffix, prefix string) {
-	s.Lock()
-	defer s.Unlock()
-	if LuetCfg.GetGeneral().Debug {
-		fmt.Println(fmt.Sprintf("%s %s",
-			Bold(Cyan(prefix)).String(),
-			Bold(Magenta(suffix)).BgBlack().String(),
-		))
-	} else {
-		s.Suffix = Bold(Magenta(suffix)).BgBlack().String()
-		s.Prefix = Bold(Cyan(prefix)).String()
+	if s != nil {
+		s.Lock()
+		defer s.Unlock()
+		if LuetCfg.GetGeneral().Debug {
+			fmt.Println(fmt.Sprintf("%s %s",
+				Bold(Cyan(prefix)).String(),
+				Bold(Magenta(suffix)).BgBlack().String(),
+			))
+		} else {
+			s.Suffix = Bold(Magenta(suffix)).BgBlack().String()
+			s.Prefix = Bold(Cyan(prefix)).String()
+		}
 	}
 }
 
@@ -108,7 +110,9 @@ func SpinnerStop() {
 	if 2 > confLevel {
 		return
 	}
-	s.Stop()
+	if s != nil {
+		s.Stop()
+	}
 }
 
 func level2Number(level string) int {

pkg/package/annotations.go

@@ -0,0 +1,23 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+type AnnotationKey string
+
+const (
+	ConfigProtectAnnnotation AnnotationKey = "config_protect"
+)

pkg/package/database_bench_test.go

@@ -0,0 +1,116 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	"io/ioutil"
+	"os"
+	"strconv"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Database  benchmark", func() {
+
+	Context("BoltDB", func() {
+
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		var db PackageSet
+
+		BeforeEach(func() {
+
+			tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			db = NewBoltDatabase(tmpfile.Name())
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+
+		Measure("it should be fast in computing world from a 50000 dataset", func(b Benchmarker) {
+			for i := 0; i < 50000; i++ {
+				a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			runtime := b.Time("runtime", func() {
+				packs := db.World()
+				Expect(len(packs)).To(Equal(50000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 30), "World() shouldn't take too long.")
+
+		}, 1)
+
+		Measure("it should be fast in computing world from a 100000 dataset", func(b Benchmarker) {
+			for i := 0; i < 100000; i++ {
+				a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			runtime := b.Time("runtime", func() {
+				packs := db.World()
+				Expect(len(packs)).To(Equal(100000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 30), "World() shouldn't take too long.")
+
+		}, 1)
+	})
+
+	Context("InMemory", func() {
+
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		var db PackageSet
+
+		BeforeEach(func() {
+
+			tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			db = NewInMemoryDatabase(false)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+
+		Measure("it should be fast in computing world from a 100000 dataset", func(b Benchmarker) {
+
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 100000; i++ {
+					a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					_, err := db.CreatePackage(a)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				packs := db.World()
+				Expect(len(packs)).To(Equal(100000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 10), "World() shouldn't take too long.")
+
+		}, 2)
+	})
+})

pkg/package/database_boltdb.go

@@ -162,26 +162,17 @@ func (db *BoltDatabase) GetAllPackages(packages chan Package) error {
 		return err
 	}
 	defer bolt.Close()
-	// Fetching records one by one (useful when the bucket contains a lot of records)
-	//query := bolt.Select()
-
-	var packs []Package
+	var packs []DefaultPackage
 	err = bolt.All(&packs)
 	if err != nil {
 		return err
 	}
 
 	for _, r := range packs {
-		packages <- r
+		packages <- &r
 	}
 
 	return nil
-
-	// return query.Each(new(DefaultPackage), func(record interface{}) error {
-	// 	u := record.(*DefaultPackage)
-	// 	packages <- u
-	// 	return err
-	// })
 }
 
 // Encode encodes the package to string.
@@ -316,16 +307,23 @@ func (db *BoltDatabase) RemovePackage(p Package) error {
 }
 
 func (db *BoltDatabase) World() Packages {
+	var packs []DefaultPackage
 
-	var all []Package
-	// FIXME: This should all be locked in the db - for now forbid the solver to be run in threads.
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err == nil {
-			all = append(all, pack)
-		}
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return Packages([]Package{})
 	}
-	return Packages(all)
+	defer bolt.Close()
+	err = bolt.All(&packs)
+	if err != nil {
+		return Packages([]Package{})
+	}
+	models := make([]Package, len(packs))
+	for i, _ := range packs {
+		models[i] = &packs[i]
+	}
+
+	return Packages(models)
 }
 
 func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
@@ -376,6 +374,11 @@ func (db *BoltDatabase) FindPackages(p Package) (Packages, error) {
 
 // FindPackageVersions return the list of the packages beloging to cat/name
 func (db *BoltDatabase) FindPackageVersions(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+
 	var versionsInWorld []Package
 	for _, w := range db.World() {
 		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
@@ -390,11 +393,7 @@ func (db *BoltDatabase) FindPackageVersions(p Package) (Packages, error) {
 func (db *BoltDatabase) FindPackageLabel(labelKey string) (Packages, error) {
 	var ans []Package
 
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err != nil {
-			return ans, err
-		}
+	for _, pack := range db.World() {
 		if pack.HasLabel(labelKey) {
 			ans = append(ans, pack)
 		}
@@ -410,11 +409,7 @@ func (db *BoltDatabase) FindPackageLabelMatch(pattern string) (Packages, error)
 		return nil, errors.New("Invalid regex " + pattern + "!")
 	}
 
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err != nil {
-			return ans, err
-		}
+	for _, pack := range db.World() {
 		if pack.MatchLabel(re) {
 			ans = append(ans, pack)
 		}
@@ -431,12 +426,7 @@ func (db *BoltDatabase) FindPackageMatch(pattern string) (Packages, error) {
 		return nil, errors.New("Invalid regex " + pattern + "!")
 	}
 
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err != nil {
-			return ans, err
-		}
-
+	for _, pack := range db.World() {
 		if re.MatchString(pack.HumanReadableString()) {
 			ans = append(ans, pack)
 		}

pkg/package/database_boltdb_test.go

@@ -0,0 +1,152 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or ItNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	"io/ioutil"
+	"os"
+	"regexp"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("BoltDB Database", func() {
+
+	tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+	defer os.Remove(tmpfile.Name()) // clean up
+	var db PackageDatabase
+
+	BeforeEach(func() {
+		tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		db = NewBoltDatabase(tmpfile.Name())
+	})
+	Context("Simple package", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		It("Find packages", func() {
+			ID, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.GetPackage(ID)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack).To(Equal(a))
+			ids := db.GetPackages()
+
+			Expect(ids).To(Equal([]string{"1"}))
+
+			pack, err = db.FindPackage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Expands correctly", func() {
+
+			a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+			re := regexp.MustCompile("project[0-9][=].*")
+			for _, p := range []Package{a1, a11, a01} {
+				_, err := db.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			lst, err := a.Expand(db)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(lst).To(ContainElement(a11))
+			Expect(lst).To(ContainElement(a1))
+			Expect(lst).ToNot(ContainElement(a01))
+			Expect(len(lst)).To(Equal(2))
+			p := lst.Best(nil)
+			Expect(p).To(Equal(a11))
+			// Test annotation with null map
+			Expect(a.MatchAnnotation(re)).To(Equal(false))
+		})
+
+		It("Find best package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+
+		})
+
+		It("Find specific package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Provides replaces definitions", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+			a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+			Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+
+			s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackage(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+		})
+
+	})
+
+})

pkg/package/database_mem.go

@@ -225,6 +225,11 @@ func (db *InMemoryDatabase) FindPackage(p Package) (Package, error) {
 
 // FindPackages return the list of the packages beloging to cat/name
 func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
 		return nil, errors.New("No versions found for package")

pkg/package/package.go

@@ -26,11 +26,13 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/mudler/luet/pkg/helpers"
+	version "github.com/mudler/luet/pkg/versioner"
+
 	gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/crillab/gophersat/bf"
 	"github.com/ghodss/yaml"
 	"github.com/jinzhu/copier"
-	version "github.com/mudler/luet/pkg/versioner"
 	"github.com/pkg/errors"
 )
 
@@ -38,10 +40,10 @@ import (
 // FIXME: Currently some of the methods are returning DefaultPackages due to JSON serialization of the package
 type Package interface {
 	Encode(PackageDatabase) (string, error)
+	Related(definitiondb PackageDatabase) Packages
 
 	BuildFormula(PackageDatabase, PackageDatabase) ([]bf.Formula, error)
-	IsFlagged(bool) Package
-	Flagged() bool
+
 	GetFingerPrint() string
 	GetPackageName() string
 	Requires([]*DefaultPackage) Package
@@ -59,9 +61,11 @@ type Package interface {
 	SetCategory(string)
 
 	GetName() string
+	SetName(string)
 	GetCategory() string
 
 	GetVersion() string
+	SetVersion(string)
 	RequiresContains(PackageDatabase, Package) (bool, error)
 	Matches(m Package) bool
 	BumpBuildVersion() error
@@ -91,13 +95,24 @@ type Package interface {
 	HasLabel(string) bool
 	MatchLabel(*regexp.Regexp) bool
 
+	AddAnnotation(string, string)
+	GetAnnotations() map[string]string
+	HasAnnotation(string) bool
+	MatchAnnotation(*regexp.Regexp) bool
+
+	IsHidden() bool
 	IsSelector() bool
 	VersionMatchSelector(string, version.Versioner) (bool, error)
 	SelectorMatchVersion(string, version.Versioner) (bool, error)
 
 	String() string
 	HumanReadableString() string
-	HashFingerprint() string
+	HashFingerprint(string) string
+
+	SetBuildTimestamp(s string)
+	GetBuildTimestamp() string
+
+	Clone() Package
 }
 
 type Tree interface {
@@ -132,6 +147,60 @@ func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) {
 	return unescaped, nil
 }
 
+type rawPackages []map[string]interface{}
+
+func (r rawPackages) Find(name, category, version string) map[string]interface{} {
+	for _, v := range r {
+		if v["name"] == name && v["category"] == category && v["version"] == version {
+			return v
+		}
+	}
+	return map[string]interface{}{}
+}
+
+func GetRawPackages(yml []byte) (rawPackages, error) {
+	var rawPackages struct {
+		Packages []map[string]interface{} `yaml:"packages"`
+	}
+	source, err := yaml.YAMLToJSON(yml)
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+	err = json.Unmarshal(bytes, &rawPackages)
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+	return rawPackages.Packages, nil
+
+}
+func DefaultPackagesFromYaml(yml []byte) ([]DefaultPackage, error) {
+
+	var unescaped struct {
+		Packages []DefaultPackage `json:"packages"`
+	}
+	source, err := yaml.YAMLToJSON(yml)
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+	err = json.Unmarshal(bytes, &unescaped)
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+	return unescaped.Packages, nil
+}
+
 // Major and minor gets escaped when marshalling in JSON, making compiler fails recognizing selectors for expansion
 func (t *DefaultPackage) JSON() ([]byte, error) {
 	buffer := &bytes.Buffer{}
@@ -151,19 +220,21 @@ type DefaultPackage struct {
 	State            State             `json:"state,omitempty"`
 	PackageRequires  []*DefaultPackage `json:"requires"`           // Affects YAML field names too.
 	PackageConflicts []*DefaultPackage `json:"conflicts"`          // Affects YAML field names too.
-	IsSet            bool              `json:"set,omitempty"`      // Affects YAML field names too.
 	Provides         []*DefaultPackage `json:"provides,omitempty"` // Affects YAML field names too.
+	Hidden           bool              `json:"hidden,omitempty"`   // Affects YAML field names too.
 
-	// TODO: Annotations?
+	// Annotations are used for core features/options
+	Annotations map[string]string `json:"annotations,omitempty"` // Affects YAML field names too
 
 	// Path is set only internally when tree is loaded from disk
 	Path string `json:"path,omitempty"`
 
-	Description string   `json:"description,omitempty"`
-	Uri         []string `json:"uri,omitempty"`
-	License     string   `json:"license,omitempty"`
+	Description    string   `json:"description,omitempty"`
+	Uri            []string `json:"uri,omitempty"`
+	License        string   `json:"license,omitempty"`
+	BuildTimestamp string   `json:"buildtimestamp,omitempty"`
 
-	Labels map[string]string `json:labels,omitempty`
+	Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
 }
 
 // State represent the package state
@@ -176,7 +247,7 @@ func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*D
 		Version:          version,
 		PackageRequires:  requires,
 		PackageConflicts: conflicts,
-		Labels:           make(map[string]string, 0),
+		Labels:           nil,
 	}
 }
 
@@ -194,9 +265,9 @@ func (p *DefaultPackage) GetFingerPrint() string {
 	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version)
 }
 
-func (p *DefaultPackage) HashFingerprint() string {
+func (p *DefaultPackage) HashFingerprint(salt string) string {
 	h := md5.New()
-	io.WriteString(h, p.GetFingerPrint())
+	io.WriteString(h, fmt.Sprintf("%s-%s", p.GetFingerPrint(), salt))
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
@@ -218,6 +289,16 @@ func (p *DefaultPackage) GetPackageName() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.Category)
 }
 
+// GetBuildTimestamp returns the package build timestamp
+func (p *DefaultPackage) GetBuildTimestamp() string {
+	return p.BuildTimestamp
+}
+
+// SetBuildTimestamp sets the package Build timestamp
+func (p *DefaultPackage) SetBuildTimestamp(s string) {
+	p.BuildTimestamp = s
+}
+
 // GetPath returns the path where the definition file was found
 func (p *DefaultPackage) GetPath() string {
 	return p.Path
@@ -235,26 +316,24 @@ func (p *DefaultPackage) IsSelector() bool {
 	return strings.ContainsAny(p.GetVersion(), "<>=")
 }
 
+func (p *DefaultPackage) IsHidden() bool {
+	return p.Hidden
+}
+
 func (p *DefaultPackage) HasLabel(label string) bool {
-	ans := false
-	for k, _ := range p.Labels {
-		if k == label {
-			ans = true
-			break
-		}
-	}
-	return ans
+	return helpers.MapHasKey(&p.Labels, label)
 }
 
 func (p *DefaultPackage) MatchLabel(r *regexp.Regexp) bool {
-	ans := false
-	for k, v := range p.Labels {
-		if r.MatchString(k + "=" + v) {
-			ans = true
-			break
-		}
-	}
-	return ans
+	return helpers.MapMatchRegex(&p.Labels, r)
+}
+
+func (p *DefaultPackage) HasAnnotation(label string) bool {
+	return helpers.MapHasKey(&p.Annotations, label)
+}
+
+func (p *DefaultPackage) MatchAnnotation(r *regexp.Regexp) bool {
+	return helpers.MapMatchRegex(&p.Annotations, r)
 }
 
 // AddUse adds a use to a package
@@ -287,7 +366,6 @@ func (p *DefaultPackage) Encode(db PackageDatabase) (string, error) {
 func (p *DefaultPackage) Yaml() ([]byte, error) {
 	j, err := p.JSON()
 	if err != nil {
-
 		return []byte{}, err
 	}
 	y, err := yaml.JSONToYAML(j)
@@ -298,15 +376,6 @@ func (p *DefaultPackage) Yaml() ([]byte, error) {
 	return y, nil
 }
 
-func (p *DefaultPackage) IsFlagged(b bool) Package {
-	p.IsSet = b
-	return p
-}
-
-func (p *DefaultPackage) Flagged() bool {
-	return p.IsSet
-}
-
 func (p *DefaultPackage) GetName() string {
 	return p.Name
 }
@@ -314,6 +383,9 @@ func (p *DefaultPackage) GetName() string {
 func (p *DefaultPackage) GetVersion() string {
 	return p.Version
 }
+func (p *DefaultPackage) SetVersion(v string) {
+	p.Version = v
+}
 func (p *DefaultPackage) GetDescription() string {
 	return p.Description
 }
@@ -338,15 +410,32 @@ func (p *DefaultPackage) GetCategory() string {
 func (p *DefaultPackage) SetCategory(s string) {
 	p.Category = s
 }
+
+func (p *DefaultPackage) SetName(s string) {
+	p.Name = s
+}
+
 func (p *DefaultPackage) GetUses() []string {
 	return p.UseFlags
 }
 func (p *DefaultPackage) AddLabel(k, v string) {
+	if p.Labels == nil {
+		p.Labels = make(map[string]string, 0)
+	}
 	p.Labels[k] = v
 }
+func (p *DefaultPackage) AddAnnotation(k, v string) {
+	if p.Annotations == nil {
+		p.Annotations = make(map[string]string, 0)
+	}
+	p.Annotations[k] = v
+}
 func (p *DefaultPackage) GetLabels() map[string]string {
 	return p.Labels
 }
+func (p *DefaultPackage) GetAnnotations() map[string]string {
+	return p.Annotations
+}
 func (p *DefaultPackage) GetProvides() []*DefaultPackage {
 	return p.Provides
 }
@@ -417,6 +506,48 @@ func (p *DefaultPackage) Revdeps(definitiondb PackageDatabase) Packages {
 	return versionsInWorld
 }
 
+func walkPackage(p Package, definitiondb PackageDatabase, visited map[string]interface{}) Packages {
+	var versionsInWorld Packages
+	if _, ok := visited[p.HumanReadableString()]; ok {
+		return versionsInWorld
+	}
+	visited[p.HumanReadableString()] = true
+
+	revdepvisited := make(map[string]interface{})
+	revdeps := p.ExpandedRevdeps(definitiondb, revdepvisited)
+	for _, r := range revdeps {
+		versionsInWorld = append(versionsInWorld, r)
+	}
+
+	if !p.IsSelector() {
+		versionsInWorld = append(versionsInWorld, p)
+	}
+
+	for _, re := range p.GetRequires() {
+		versions, _ := re.Expand(definitiondb)
+		for _, r := range versions {
+
+			versionsInWorld = append(versionsInWorld, r)
+			versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...)
+		}
+
+	}
+	for _, re := range p.GetConflicts() {
+		versions, _ := re.Expand(definitiondb)
+		for _, r := range versions {
+
+			versionsInWorld = append(versionsInWorld, r)
+			versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...)
+
+		}
+	}
+	return versionsInWorld.Unique()
+}
+
+func (p *DefaultPackage) Related(definitiondb PackageDatabase) Packages {
+	return walkPackage(p, definitiondb, map[string]interface{}{})
+}
+
 // ExpandedRevdeps returns the package reverse dependencies,
 // matching also selectors in versions (>, <, >=, <=)
 func (p *DefaultPackage) ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages {
@@ -622,7 +753,7 @@ func (pack *DefaultPackage) buildFormula(definitiondb PackageDatabase, db Packag
 					C = bf.Var(encodedC)
 					// Or the Candidate is true, or all the others might be not true
 					// This forces the CDCL sat implementation to look first at a solution with C=true
-					formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Or(C, bf.Or(priorityConstraints...)), bf.Or(bf.Not(C), bf.Or(priorityALO...)))))
+					formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.And(C, bf.Or(priorityConstraints...)), bf.And(bf.Not(C), bf.Or(priorityALO...)))))
 				}
 
 				// AMO - At most one
@@ -728,7 +859,6 @@ func (p *DefaultPackage) Explain() {
 	fmt.Println("Name: ", p.GetName())
 	fmt.Println("Category: ", p.GetCategory())
 	fmt.Println("Version: ", p.GetVersion())
-	fmt.Println("Installed: ", p.IsSet)
 
 	for _, req := range p.GetRequires() {
 		fmt.Println("\t-> ", req)

pkg/package/package_test.go

@@ -36,8 +36,8 @@ var _ = Describe("Package", func() {
 		})
 
 		It("Generates packages fingerprint's hashes", func() {
-			Expect(a.HashFingerprint()).ToNot(Equal(a1.HashFingerprint()))
-			Expect(a.HashFingerprint()).To(Equal("c64caa391b79adb598ad98e261aa79a0"))
+			Expect(a.HashFingerprint("")).ToNot(Equal(a1.HashFingerprint("")))
+			Expect(a.HashFingerprint("")).To(Equal("76972ef6991ec6102f33b401105c1351"))
 		})
 	})
 
@@ -46,6 +46,7 @@ var _ = Describe("Package", func() {
 		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
 		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+		re := regexp.MustCompile("project[0-9][=].*")
 		It("Expands correctly", func() {
 			definitions := NewInMemoryDatabase(false)
 			for _, p := range []Package{a1, a11, a01} {
@@ -60,6 +61,8 @@ var _ = Describe("Package", func() {
 			Expect(len(lst)).To(Equal(2))
 			p := lst.Best(nil)
 			Expect(p).To(Equal(a11))
+			// Test annotation with null map
+			Expect(a.MatchAnnotation(re)).To(Equal(false))
 		})
 	})
 
@@ -91,6 +94,34 @@ var _ = Describe("Package", func() {
 		})
 	})
 
+	Context("Find annotations on packages", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.AddAnnotation("project1", "test1")
+		a.AddAnnotation("label2", "value1")
+		b := NewPackage("B", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b.AddAnnotation("project2", "test2")
+		b.AddAnnotation("label2", "value1")
+		It("Expands correctly", func() {
+			var err error
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b} {
+				_, err = definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			re := regexp.MustCompile("project[0-9][=].*")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(re).ToNot(BeNil())
+			Expect(a.HasAnnotation("label2")).To(Equal(true))
+			Expect(a.HasAnnotation("label3")).To(Equal(false))
+			Expect(a.HasAnnotation("project1")).To(Equal(true))
+			Expect(b.HasAnnotation("project2")).To(Equal(true))
+			Expect(b.HasAnnotation("label2")).To(Equal(true))
+			Expect(b.MatchAnnotation(re)).To(Equal(true))
+			Expect(a.MatchAnnotation(re)).To(Equal(true))
+
+		})
+	})
+
 	Context("Check description", func() {
 		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a.SetDescription("Description A")
@@ -283,7 +314,6 @@ var _ = Describe("Package", func() {
 
 			Expect(p.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(p.GetName()).To(Equal(a.GetName()))
-			Expect(p.Flagged()).To(Equal(a.Flagged()))
 			Expect(p.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(p.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(p.GetRequires())).To(Equal(len(a.GetRequires())))
@@ -343,7 +373,6 @@ var _ = Describe("Package", func() {
 			a2 := a.Clone()
 			Expect(a2.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(a2.GetName()).To(Equal(a.GetName()))
-			Expect(a2.Flagged()).To(Equal(a.Flagged()))
 			Expect(a2.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(a2.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(a2.GetRequires())).To(Equal(len(a.GetRequires())))

pkg/repository/loader.go

@@ -19,6 +19,7 @@ package repository
 import (
 	"io/ioutil"
 	"path"
+	"path/filepath"
 	"regexp"
 
 	"github.com/ghodss/yaml"
@@ -28,9 +29,22 @@ import (
 )
 
 func LoadRepositories(c *LuetConfig) error {
-	var regexRepo = regexp.MustCompile(`.yml$`)
+	var regexRepo = regexp.MustCompile(`.yml$|.yaml$`)
+	var err error
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
 
 	for _, rdir := range c.RepositoriesConfDir {
+
+		rdir = filepath.Join(rootfs, rdir)
+
 		Debug("Parsing Repository Directory", rdir, "...")
 
 		files, err := ioutil.ReadDir(rdir)

pkg/repository/repository_test.go

@@ -35,12 +35,22 @@ var _ = Describe("Repository", func() {
 
 		It("Chec Load Repository 1", func() {
 			Expect(err).Should(BeNil())
-			Expect(len(cfg.SystemRepositories)).Should(Equal(1))
+			Expect(len(cfg.SystemRepositories)).Should(Equal(2))
 			Expect(cfg.SystemRepositories[0].Name).Should(Equal("test1"))
 			Expect(cfg.SystemRepositories[0].Priority).Should(Equal(999))
 			Expect(cfg.SystemRepositories[0].Type).Should(Equal("disk"))
 			Expect(len(cfg.SystemRepositories[0].Urls)).Should(Equal(1))
 			Expect(cfg.SystemRepositories[0].Urls[0]).Should(Equal("tests/repos/test1"))
 		})
+
+		It("Chec Load Repository 2", func() {
+			Expect(err).Should(BeNil())
+			Expect(len(cfg.SystemRepositories)).Should(Equal(2))
+			Expect(cfg.SystemRepositories[1].Name).Should(Equal("test2"))
+			Expect(cfg.SystemRepositories[1].Priority).Should(Equal(1000))
+			Expect(cfg.SystemRepositories[1].Type).Should(Equal("disk"))
+			Expect(len(cfg.SystemRepositories[1].Urls)).Should(Equal(1))
+			Expect(cfg.SystemRepositories[1].Urls[0]).Should(Equal("tests/repos/test2"))
+		})
 	})
 })

pkg/solver/benchmark_test.go

@@ -0,0 +1,298 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/tests/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "github.com/mudler/luet/pkg/solver"
+)
+
+var _ = Describe("Solver Benchmarks", func() {
+	db := pkg.NewInMemoryDatabase(false)
+	dbInstalled := pkg.NewInMemoryDatabase(false)
+	dbDefinitions := pkg.NewInMemoryDatabase(false)
+	var s PackageSolver
+
+	Context("Complex data sets", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+		Measure("it should be fast in resolution from a 50000 dataset", func(b Benchmarker) {
+
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 50000; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					_, err := dbInstalled.CreatePackage(C)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for i := 0; i < 1; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+					solution, err := s.Install([]pkg.Package{A})
+					Expect(err).ToNot(HaveOccurred())
+
+					Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: H, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: G, Value: true}))
+				}
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 120), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+	Context("Complex data sets - Parallel", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: ParallelSimple, Concurrency: 10}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+		Measure("it should be fast in resolution from a 50000 dataset", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 50000; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					_, err := dbInstalled.CreatePackage(C)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				for i := 0; i < 1; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+					solution, err := s.Install([]pkg.Package{A})
+					Expect(err).ToNot(HaveOccurred())
+
+					Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: H, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: G, Value: true}))
+
+					//	Expect(len(solution)).To(Equal(6))
+				}
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+	Context("Complex data sets - Parallel Upgrades", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			//	dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: ParallelSimple, Concurrency: 100}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+			tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			dbInstalled = pkg.NewBoltDatabase(tmpfile.Name())
+		})
+
+		Measure("it should be fast in resolution from a 10000*8 dataset", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 2; i < 10000; i++ {
+					C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G, H} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+				}
+
+				//C := pkg.NewPackage("C", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				G := pkg.NewPackage("G", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H := pkg.NewPackage("H", "1", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "1", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "1", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				_, err := dbInstalled.CreatePackage(A)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(B)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(D)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(H)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(G)
+				Expect(err).ToNot(HaveOccurred())
+				fmt.Println("Upgrade starts")
+
+				packages, ass, err := s.Upgrade(false, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(packages).To(ContainElement(A))
+
+				G = pkg.NewPackage("G", "9999", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H = pkg.NewPackage("H", "9999", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D = pkg.NewPackage("D", "9999", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B = pkg.NewPackage("B", "9999", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A = pkg.NewPackage("A", "9999", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				Expect(ass).To(ContainElement(PackageAssert{Package: A, Value: true}))
+
+				Expect(len(packages)).To(Equal(5))
+				//	Expect(len(solution)).To(Equal(6))
+
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+
+		Measure("it should be fast in installation with 12000 packages installed and 2000*8 available", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 2000; i++ {
+					C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					fmt.Println("Creating package, run", i)
+				}
+
+				for i := 0; i < 12000; i++ {
+					x := helpers.RandomPackage()
+					_, err := dbInstalled.CreatePackage(x)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				G := pkg.NewPackage("G", strconv.Itoa(50000), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H := pkg.NewPackage("H", strconv.Itoa(50000), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", strconv.Itoa(50000), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", strconv.Itoa(50000), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", strconv.Itoa(50000), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				ass, err := s.Install([]pkg.Package{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(ass).To(ContainElement(PackageAssert{Package: pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}), Value: true}))
+				//Expect(ass).To(Equal(5))
+				//	Expect(len(solution)).To(Equal(6))
+
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+
+		PMeasure("it should be fast in resolution from a 50000 dataset with upgrade universe", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 2; i++ {
+					C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					fmt.Println("Creating package, run", i)
+				}
+
+				G := pkg.NewPackage("G", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H := pkg.NewPackage("H", "1", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "1", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "1", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				_, err := dbInstalled.CreatePackage(A)
+				Expect(err).ToNot(HaveOccurred())
+				fmt.Println("Upgrade starts")
+
+				packages, ass, err := s.UpgradeUniverse(true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(ass).To(ContainElement(PackageAssert{Package: pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}), Value: true}))
+				Expect(packages).To(ContainElement(pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})))
+				Expect(packages).To(Equal(5))
+				//	Expect(len(solution)).To(Equal(6))
+
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+})

pkg/solver/decoder.go

@@ -22,9 +22,9 @@ import (
 	"unicode"
 
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/topsort"
 	toposort "github.com/philopon/go-toposort"
 	"github.com/pkg/errors"
-	"github.com/stevenle/topsort"
 )
 
 type PackagesAssertions []PackageAssert
@@ -150,22 +150,13 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 
 	orderedAssertions := PackagesAssertions{}
 	unorderedAssertions := PackagesAssertions{}
-	fingerprints := []string{}
 
 	tmpMap := map[string]PackageAssert{}
 	graph := topsort.NewGraph()
-
 	for _, a := range assertions {
 		graph.AddNode(a.Package.GetFingerPrint())
 		tmpMap[a.Package.GetFingerPrint()] = a
-		fingerprints = append(fingerprints, a.Package.GetFingerPrint())
 		unorderedAssertions = append(unorderedAssertions, a) // Build a list of the ones that must be ordered
-
-		if a.Value {
-			unorderedAssertions = append(unorderedAssertions, a) // Build a list of the ones that must be ordered
-		} else {
-			orderedAssertions = append(orderedAssertions, a) // Keep last the ones which are not meant to be installed
-		}
 	}
 
 	sort.Sort(unorderedAssertions)
@@ -190,7 +181,7 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 			}
 			// Expand also here, as we need to order them (or instead the solver should give back the dep correctly?)
 			graph.AddEdge(currentPkg.GetFingerPrint(), requiredDef.GetFingerPrint())
-			added[requiredDef.GetFingerPrint()] = nil
+			added[requiredDef.GetFingerPrint()] = true
 		}
 	}
 	result, err := graph.TopSort(fingerprint)
@@ -200,8 +191,11 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 	for _, res := range result {
 		a, ok := tmpMap[res]
 		if !ok {
-			return nil, errors.New("fail looking for " + res)
-			//	continue
+			//return nil, errors.New("fail looking for " + res)
+			// Since now we don't return the entire world as part of assertions
+			// if we don't find any reference must be because fingerprint we are analyzing (which is the one we are ordering against)
+			// is not part of the assertions, thus we can omit it from the result
+			continue
 		}
 		orderedAssertions = append(orderedAssertions, a)
 		//	orderedAssertions = append(PackagesAssertions{a}, orderedAssertions...) // push upfront
@@ -251,6 +245,34 @@ func (a PackagesAssertions) Less(i, j int) bool {
 
 }
 
+func (a PackagesAssertions) TrueLen() int {
+	count := 0
+	for _, ass := range a {
+		if ass.Value {
+			count++
+		}
+	}
+
+	return count
+}
+
+// HashFrom computes the assertion hash From a given package. It drops it from the assertions
+// and checks it's not the only one. if it's unique it marks it specially - so the hash
+// which is generated is unique for the selected package
+func (assertions PackagesAssertions) HashFrom(p pkg.Package) string {
+
+	var assertionhash string
+
+	// When we don't have any solution to hash for, we need to generate an UUID by ourselves
+	latestsolution := assertions.Drop(p)
+	if latestsolution.TrueLen() == 0 {
+		assertionhash = assertions.Mark(p).AssertionHash()
+	} else {
+		assertionhash = latestsolution.AssertionHash()
+	}
+	return assertionhash
+}
+
 func (assertions PackagesAssertions) AssertionHash() string {
 	var fingerprint string
 	for _, assertion := range assertions { // Note: Always order them first!
@@ -287,3 +309,18 @@ func (assertions PackagesAssertions) Cut(p pkg.Package) PackagesAssertions {
 	}
 	return ass
 }
+
+// Mark returns a new assertion with the package marked
+func (assertions PackagesAssertions) Mark(p pkg.Package) PackagesAssertions {
+	ass := PackagesAssertions{}
+
+	for _, a := range assertions {
+		if a.Package.Matches(p) {
+			marked := a.Package.Clone()
+			marked.SetName("@@" + marked.GetName())
+			a = PackageAssert{Package: marked.(*pkg.DefaultPackage), Value: a.Value, Hash: a.Hash}
+		}
+		ass = append(ass, a)
+	}
+	return ass
+}

pkg/solver/decoder_test.go

@@ -19,6 +19,7 @@ import (
 	"strconv"
 
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -29,13 +30,13 @@ var _ = Describe("Decoder", func() {
 	db := pkg.NewInMemoryDatabase(false)
 	dbInstalled := pkg.NewInMemoryDatabase(false)
 	dbDefinitions := pkg.NewInMemoryDatabase(false)
-	s := NewSolver(dbInstalled, dbDefinitions, db)
+	s := NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 	BeforeEach(func() {
 		db = pkg.NewInMemoryDatabase(false)
 		dbInstalled = pkg.NewInMemoryDatabase(false)
 		dbDefinitions = pkg.NewInMemoryDatabase(false)
-		s = NewSolver(dbInstalled, dbDefinitions, db)
+		s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 	})
 
 	Context("Assertion ordering", func() {
@@ -213,12 +214,10 @@ var _ = Describe("Decoder", func() {
 			hash2 := solution.AssertionHash()
 
 			//	Expect(len(solution)).To(Equal(6))
-			Expect(solution[0].Package.GetName()).To(Equal("A"))
-			Expect(solution[1].Package.GetName()).To(Equal("G"))
-			Expect(solution[2].Package.GetName()).To(Equal("H"))
-			Expect(solution[3].Package.GetName()).To(Equal("D"))
-			Expect(solution[4].Package.GetName()).To(Equal("B"))
-			Expect(solution[0].Value).ToNot(BeTrue())
+			Expect(solution[0].Package.GetName()).To(Equal("G"))
+			Expect(solution[1].Package.GetName()).To(Equal("H"))
+			Expect(solution[2].Package.GetName()).To(Equal("D"))
+			Expect(solution[3].Package.GetName()).To(Equal("B"))
 
 			Expect(hash).ToNot(Equal(""))
 			Expect(hash2).ToNot(Equal(""))
@@ -283,5 +282,143 @@ var _ = Describe("Decoder", func() {
 			Expect(orderY.Cut(Y).Drop(Y).AssertionHash()).To(Equal(orderZ.Cut(Z).Drop(Z).AssertionHash()))
 		})
 
+		It("HashFrom can be used equally", func() {
+
+			X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+			Z := pkg.NewPackage("Z", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{X, Y, Z} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			solution, err := s.Install([]pkg.Package{Y})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution2, err := s.Install([]pkg.Package{Z})
+			Expect(err).ToNot(HaveOccurred())
+			orderY, err := solution.Order(dbDefinitions, Y.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
+			orderZ, err := solution2.Order(dbDefinitions, Z.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(orderY.Cut(Y).Drop(Y)).To(Equal(orderZ.Cut(Z).Drop(Z)))
+
+			Expect(orderY.Cut(Y).HashFrom(Y)).To(Equal(orderZ.Cut(Z).HashFrom(Z)))
+		})
+
+		It("Unique hashes for single packages", func() {
+
+			X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{X, F, D} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			solution, err := s.Install([]pkg.Package{X})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution2, err := s.Install([]pkg.Package{F})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution3, err := s.Install([]pkg.Package{D})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(solution.AssertionHash()).ToNot(Equal(solution2.AssertionHash()))
+			Expect(solution3.AssertionHash()).To(Equal(solution.AssertionHash()))
+
+		})
+
+		It("Unique hashes for empty assertions", func() {
+			empty := solver.PackagesAssertions{}
+			empty2 := solver.PackagesAssertions{}
+
+			Expect(empty.AssertionHash()).To(Equal(empty2.AssertionHash()))
+		})
+
+		It("Unique hashes for single packages with HashFrom", func() {
+
+			X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+
+			empty := solver.PackagesAssertions{}
+
+			for _, p := range []pkg.Package{X, F, D, Y} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			solution, err := s.Install([]pkg.Package{X})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution2, err := s.Install([]pkg.Package{F})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution3, err := s.Install([]pkg.Package{D})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution4, err := s.Install([]pkg.Package{Y})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(solution.HashFrom(X)).ToNot(Equal(solution2.HashFrom(F)))
+			Expect(solution3.HashFrom(D)).To(Equal(solution.HashFrom(X)))
+
+			Expect(empty.AssertionHash()).ToNot(Equal(solution3.HashFrom(D)))
+			Expect(empty.AssertionHash()).ToNot(Equal(solution2.HashFrom(F)))
+
+			Expect(solution4.Drop(Y).AssertionHash()).To(Equal(solution4.HashFrom(Y)))
+		})
+		for index := 0; index < 300; index++ { // Just to make sure we don't have false positives
+
+			It("Always same solution", func() {
+
+				X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+				Z := pkg.NewPackage("Z", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+				W := pkg.NewPackage("W", "", []*pkg.DefaultPackage{Z, Y}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{X, Y, Z} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{W})
+				Expect(err).ToNot(HaveOccurred())
+
+				orderW, err := solution.Order(dbDefinitions, W.GetFingerPrint())
+				Expect(err).ToNot(HaveOccurred())
+				Expect(orderW[0].Package.GetName()).To(Equal("X"))
+				Expect(orderW[1].Package.GetName()).To(Equal("Y"))
+				Expect(orderW[2].Package.GetName()).To(Equal("Z"))
+				Expect(orderW[3].Package.GetName()).To(Equal("W"))
+			})
+
+		}
+
 	})
 })

pkg/solver/parallel.go

@@ -0,0 +1,845 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver
+
+import (
+
+	//. "github.com/mudler/luet/pkg/logger"
+	"fmt"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"github.com/crillab/gophersat/bf"
+	pkg "github.com/mudler/luet/pkg/package"
+)
+
+// Parallel is the default Parallel for luet
+type Parallel struct {
+	Concurrency        int
+	DefinitionDatabase pkg.PackageDatabase
+	ParallelDatabase   pkg.PackageDatabase
+	Wanted             pkg.Packages
+	InstalledDatabase  pkg.PackageDatabase
+
+	Resolver PackageResolver
+}
+
+func (s *Parallel) SetDefinitionDatabase(db pkg.PackageDatabase) {
+	s.DefinitionDatabase = db
+}
+
+// SetReSolver is a setter for the unsat ReSolver backend
+func (s *Parallel) SetResolver(r PackageResolver) {
+	s.Resolver = r
+}
+
+func (s *Parallel) World() pkg.Packages {
+	return s.DefinitionDatabase.World()
+}
+
+func (s *Parallel) Installed() pkg.Packages {
+
+	return s.InstalledDatabase.World()
+}
+
+func (s *Parallel) noRulesWorld() bool {
+	for _, p := range s.World() {
+		if len(p.GetConflicts()) != 0 || len(p.GetRequires()) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (s *Parallel) noRulesInstalled() bool {
+	for _, p := range s.Installed() {
+		if len(p.GetConflicts()) != 0 || len(p.GetRequires()) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (s *Parallel) buildParallelFormula(formulas []bf.Formula, packages pkg.Packages) (bf.Formula, error) {
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				solvable, err := p.BuildFormula(s.DefinitionDatabase, s.ParallelDatabase)
+				if err != nil {
+					panic(err)
+				}
+				for _, s := range solvable {
+					results <- s
+				}
+			}
+		}(wg, all)
+	}
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	for _, p := range packages {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	if len(formulas) != 0 {
+		return bf.And(formulas...), nil
+	}
+	return bf.True, nil
+}
+
+func (s *Parallel) BuildInstalled() (bf.Formula, error) {
+	var formulas []bf.Formula
+
+	var packages pkg.Packages
+	for _, p := range s.Installed() {
+		packages = append(packages, p)
+		for _, dep := range p.Related(s.DefinitionDatabase) {
+			packages = append(packages, dep)
+		}
+
+	}
+
+	return s.buildParallelFormula(formulas, packages)
+}
+
+// BuildWorld builds the formula which olds the requirements from the package definitions
+// which are available (global state)
+func (s *Parallel) BuildWorld(includeInstalled bool) (bf.Formula, error) {
+	var formulas []bf.Formula
+	// NOTE: This block should be enabled in case of very old systems with outdated world sets
+	if includeInstalled {
+		solvable, err := s.BuildInstalled()
+		if err != nil {
+			return nil, err
+		}
+		//f = bf.And(f, solvable)
+		formulas = append(formulas, solvable)
+	}
+	return s.buildParallelFormula(formulas, s.World())
+}
+
+// BuildWorld builds the formula which olds the requirements from the package definitions
+// which are available (global state)
+func (s *Parallel) BuildPartialWorld(includeInstalled bool) (bf.Formula, error) {
+	var formulas []bf.Formula
+	// NOTE: This block should be enabled in case of very old systems with outdated world sets
+	if includeInstalled {
+		solvable, err := s.BuildInstalled()
+		if err != nil {
+			return nil, err
+		}
+		//f = bf.And(f, solvable)
+		formulas = append(formulas, solvable)
+	}
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+	var packages pkg.Packages
+
+	all := make(chan pkg.Package)
+	results := make(chan pkg.Package, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				for _, dep := range p.Related(s.DefinitionDatabase) {
+					results <- dep
+				}
+
+			}
+		}(wg, all)
+	}
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			packages = append(packages, t)
+		}
+	}()
+
+	for _, p := range s.Wanted {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	return s.buildParallelFormula(formulas, packages)
+
+	//return s.buildParallelFormula(formulas, s.World())
+}
+
+func (s *Parallel) getList(db pkg.PackageDatabase, lsp pkg.Packages) (pkg.Packages, error) {
+	var ls pkg.Packages
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan pkg.Package, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				cp, err := db.FindPackage(p)
+				if err != nil {
+					packages, err := p.Expand(db)
+					// Expand, and relax search - if not found pick the same one
+					if err != nil || len(packages) == 0 {
+						cp = p
+					} else {
+						cp = packages.Best(nil)
+					}
+				}
+				results <- cp
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func(wg *sync.WaitGroup) {
+		defer wg2.Done()
+		for t := range results {
+			ls = append(ls, t)
+		}
+	}(wg)
+
+	for _, pp := range lsp {
+		all <- pp
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	return ls, nil
+}
+
+// Conflicts acts like ConflictsWith, but uses package's reverse dependencies to
+// determine if it conflicts with the given set
+func (s *Parallel) Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error) {
+	p, err := s.DefinitionDatabase.FindPackage(pack)
+	if err != nil {
+		p = pack
+	}
+
+	ls, err := s.getList(s.DefinitionDatabase, lsp)
+	if err != nil {
+		return false, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	if s.noRulesWorld() {
+		return false, nil
+	}
+
+	temporarySet := pkg.NewInMemoryDatabase(false)
+	for _, p := range ls {
+		temporarySet.CreatePackage(p)
+	}
+	visited := make(map[string]interface{})
+	revdeps := p.ExpandedRevdeps(temporarySet, visited)
+
+	var revdepsErr error
+	for _, r := range revdeps {
+		if revdepsErr == nil {
+			revdepsErr = errors.New("")
+		}
+		revdepsErr = errors.New(fmt.Sprintf("%s\n%s", revdepsErr.Error(), r.HumanReadableString()))
+	}
+
+	return len(revdeps) != 0, revdepsErr
+}
+
+// ConflictsWith return true if a package is part of the requirement set of a list of package
+// return false otherwise (and thus it is NOT relevant to the given list)
+func (s *Parallel) ConflictsWith(pack pkg.Package, lsp pkg.Packages) (bool, error) {
+	p, err := s.DefinitionDatabase.FindPackage(pack)
+	if err != nil {
+		p = pack //Relax search, otherwise we cannot compute solutions for packages not in definitions
+	}
+
+	ls, err := s.getList(s.DefinitionDatabase, lsp)
+	if err != nil {
+		return false, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	var formulas []bf.Formula
+
+	if s.noRulesWorld() {
+		return false, nil
+	}
+
+	encodedP, err := p.Encode(s.ParallelDatabase)
+	if err != nil {
+		return false, err
+	}
+	P := bf.Var(encodedP)
+
+	r, err := s.BuildWorld(false)
+	if err != nil {
+		return false, err
+	}
+	formulas = append(formulas, bf.And(bf.Not(P), r))
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for i := range c {
+				if i.Matches(p) {
+					continue
+				}
+
+				// XXX: Skip check on any of its requires ?  ( Drop to avoid removing system packages when selecting an uninstall)
+				// if i.RequiresContains(p) {
+				// 	fmt.Println("Requires found")
+				// 	continue
+				// }
+
+				encodedI, err := i.Encode(s.ParallelDatabase)
+				if err != nil {
+					panic(err)
+				}
+				I := bf.Var(encodedI)
+
+				results <- bf.And(I, r)
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	for _, p := range ls {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return true, nil
+	}
+
+	return false, nil
+
+}
+
+func (s *Parallel) ConflictsWithInstalled(p pkg.Package) (bool, error) {
+	return s.ConflictsWith(p, s.Installed())
+}
+
+// UninstallUniverse takes a list of candidate package and return a list of packages that would be removed
+// in order to purge the candidate. Uses the Parallel to check constraints and nothing else
+//
+// It can be compared to the counterpart Uninstall as this method acts like a uninstall --full
+// it removes all the packages and its deps. taking also in consideration other packages that might have
+// revdeps
+func (s *Parallel) UninstallUniverse(toremove pkg.Packages) (pkg.Packages, error) {
+
+	if s.noRulesInstalled() {
+		return s.getList(s.InstalledDatabase, toremove)
+	}
+
+	// resolve to packages from the db
+	toRemove, err := s.getList(s.InstalledDatabase, toremove)
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	var formulas []bf.Formula
+	r, err := s.BuildInstalled()
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	// SAT encode the clauses against the world
+	for _, p := range toRemove.Unique() {
+		encodedP, err := p.Encode(s.InstalledDatabase)
+		if err != nil {
+			return nil, errors.Wrap(err, "Package not found in definition db")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(bf.Not(P), r))
+	}
+
+	markedForRemoval := pkg.Packages{}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, errors.New("Failed finding a solution")
+	}
+	assertion, err := DecodeModel(model, s.InstalledDatabase)
+	if err != nil {
+		return nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+	}
+	return markedForRemoval, nil
+}
+
+// UpgradeUniverse mark packages for removal and returns a solution. It considers
+// the Universe db as authoritative
+// See also on the subject: https://arxiv.org/pdf/1007.1021.pdf
+func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssertions, error) {
+	var formulas []bf.Formula
+	// we first figure out which aren't up-to-date
+	// which has to be removed
+	// and which needs to be upgraded
+	removed := pkg.Packages{}
+
+	// TODO: this is memory expensive, we need to optimize this
+	universe := pkg.NewInMemoryDatabase(false)
+
+	for _, p := range s.DefinitionDatabase.World() {
+		universe.CreatePackage(p)
+	}
+	for _, p := range s.Installed() {
+		universe.CreatePackage(p)
+	}
+
+	// Build constraints for the whole defdb
+	r, err := s.BuildWorld(true)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't build world constraints")
+	}
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				available, err := universe.FindPackageVersions(p)
+				if err != nil {
+					removed = append(removed, p) /// FIXME: Racy
+				}
+				if len(available) == 0 {
+					continue
+				}
+
+				bestmatch := available.Best(nil)
+				// Found a better version available
+				if !bestmatch.Matches(p) {
+					encodedP, _ := p.Encode(universe)
+					P := bf.Var(encodedP)
+					results <- bf.And(bf.Not(P), r)
+					encodedP, _ = bestmatch.Encode(universe)
+					P = bf.Var(encodedP)
+					results <- bf.And(P, r)
+				}
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	// Grab all the installed ones, see if they are eligible for update
+	for _, p := range s.Installed() {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	// Treat removed packages from universe as marked for deletion
+	if dropremoved {
+
+		// SAT encode the clauses against the world
+		for _, p := range removed {
+			encodedP, err := p.Encode(universe)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "couldn't encode package")
+			}
+			P := bf.Var(encodedP)
+			formulas = append(formulas, bf.And(bf.Not(P), r))
+		}
+	}
+
+	markedForRemoval := pkg.Packages{}
+	if len(formulas) == 0 {
+		return pkg.Packages{}, PackagesAssertions{}, nil
+	}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, nil, errors.New("Failed finding a solution")
+	}
+
+	assertion, err := DecodeModel(model, universe)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+
+	}
+	return markedForRemoval, assertion, nil
+}
+
+// Upgrade compute upgrades of the package against the world definition.
+// It accepts two boolean indicating if it has to check for conflicts or try to attempt a full upgrade
+func (s *Parallel) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAssertions, error) {
+
+	// First get candidates that needs to be upgraded..
+
+	toUninstall := pkg.Packages{}
+	toInstall := pkg.Packages{}
+
+	// we do this in memory so we take into account of provides
+	universe := pkg.NewInMemoryDatabase(false)
+	for _, p := range s.DefinitionDatabase.World() {
+		universe.CreatePackage(p)
+	}
+
+	installedcopy := pkg.NewInMemoryDatabase(false)
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan []pkg.Package, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				installedcopy.CreatePackage(p)
+				packages, err := universe.FindPackageVersions(p)
+				if err == nil && len(packages) != 0 {
+					best := packages.Best(nil)
+					if !best.Matches(p) {
+						results <- []pkg.Package{p, best}
+					}
+				}
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			toUninstall = append(toUninstall, t[0])
+			toInstall = append(toInstall, t[1])
+		}
+	}()
+
+	for _, p := range s.InstalledDatabase.World() {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	s2 := &Parallel{Concurrency: s.Concurrency, InstalledDatabase: installedcopy, DefinitionDatabase: s.DefinitionDatabase, ParallelDatabase: pkg.NewInMemoryDatabase(false)}
+	s2.SetResolver(s.Resolver)
+	if !full {
+		ass := PackagesAssertions{}
+		for _, i := range toInstall {
+			ass = append(ass, PackageAssert{Package: i.(*pkg.DefaultPackage), Value: true})
+		}
+	}
+
+	// Then try to uninstall the versions in the system, and store that tree
+	for _, p := range toUninstall {
+		r, err := s.Uninstall(p, checkconflicts, false)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't uninstall selected candidate "+p.GetFingerPrint())
+		}
+		for _, z := range r {
+			err = installedcopy.RemovePackage(z)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't remove copy of package targetted for removal")
+			}
+		}
+	}
+	if len(toInstall) == 0 {
+		return toUninstall, PackagesAssertions{}, nil
+	}
+	r, e := s2.Install(toInstall)
+	return toUninstall, r, e
+	// To that tree, ask to install the versions that should be upgraded, and try to solve
+	// Return the solution
+
+}
+
+// Uninstall takes a candidate package and return a list of packages that would be removed
+// in order to purge the candidate. Returns error if unsat.
+func (s *Parallel) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packages, error) {
+	var res pkg.Packages
+	candidate, err := s.InstalledDatabase.FindPackage(c)
+	if err != nil {
+
+		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
+		packages, err := c.Expand(s.InstalledDatabase)
+		//	Info("Expanded", packages, err)
+		if err != nil || len(packages) == 0 {
+			candidate = c
+		} else {
+			candidate = packages.Best(nil)
+		}
+		//Relax search, otherwise we cannot compute solutions for packages not in definitions
+		//	return nil, errors.Wrap(err, "Package not found between installed")
+	}
+	// Build a fake "Installed" - Candidate and its requires tree
+	var InstalledMinusCandidate pkg.Packages
+
+	// We are asked to not perform a full uninstall (checking all the possible requires that could
+	// be removed). Let's only check if we can remove the selected package
+	if !full && checkconflicts {
+		if conflicts, err := s.Conflicts(candidate, s.Installed()); conflicts {
+			return nil, err
+		} else {
+			return pkg.Packages{candidate}, nil
+		}
+	}
+
+	// TODO: Can be optimized
+	for _, i := range s.Installed() {
+		if !i.Matches(candidate) {
+			contains, err := candidate.RequiresContains(s.ParallelDatabase, i)
+			if err != nil {
+				return nil, errors.Wrap(err, "Failed getting installed list")
+			}
+			if !contains {
+				InstalledMinusCandidate = append(InstalledMinusCandidate, i)
+			}
+		}
+	}
+
+	s2 := &Parallel{Concurrency: s.Concurrency, InstalledDatabase: pkg.NewInMemoryDatabase(false), DefinitionDatabase: s.DefinitionDatabase, ParallelDatabase: pkg.NewInMemoryDatabase(false)}
+	s2.SetResolver(s.Resolver)
+	// Get the requirements to install the candidate
+	asserts, err := s2.Install(pkg.Packages{candidate})
+	if err != nil {
+		return nil, err
+	}
+	for _, a := range asserts {
+		if a.Value {
+			if !checkconflicts {
+				res = append(res, a.Package)
+				continue
+			}
+
+			c, err := s.ConflictsWithInstalled(a.Package)
+			if err != nil {
+				return nil, err
+			}
+
+			// If doesn't conflict with installed we just consider it for removal and look for the next one
+			if !c {
+				res = append(res, a.Package)
+				continue
+			}
+
+			// If does conflicts, give it another chance by checking conflicts if in case we didn't installed our candidate and all the required packages in the system
+			c, err = s.ConflictsWith(a.Package, InstalledMinusCandidate)
+			if err != nil {
+				return nil, err
+			}
+			if !c {
+				res = append(res, a.Package)
+			}
+
+		}
+
+	}
+
+	return res, nil
+}
+
+// BuildFormula builds the main solving formula that is evaluated by the sat Parallel.
+func (s *Parallel) BuildFormula() (bf.Formula, error) {
+	var formulas []bf.Formula
+
+	r, err := s.BuildPartialWorld(false)
+	if err != nil {
+		return nil, err
+	}
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for wanted := range c {
+				encodedW, err := wanted.Encode(s.ParallelDatabase)
+				if err != nil {
+					panic(err)
+				}
+				W := bf.Var(encodedW)
+				installedWorld := s.Installed()
+				//TODO:Optimize
+				if len(installedWorld) == 0 {
+					results <- W
+					continue
+				}
+
+				for _, installed := range installedWorld {
+					encodedI, err := installed.Encode(s.ParallelDatabase)
+					if err != nil {
+						panic(err)
+					}
+					I := bf.Var(encodedI)
+					results <- bf.And(W, I)
+				}
+			}
+		}(wg, all)
+	}
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	for _, wanted := range s.Wanted {
+		all <- wanted
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	formulas = append(formulas, r)
+
+	return bf.And(formulas...), nil
+}
+
+func (s *Parallel) solve(f bf.Formula) (map[string]bool, bf.Formula, error) {
+	model := bf.Solve(f)
+	if model == nil {
+		return model, f, errors.New("Unsolvable")
+	}
+
+	return model, f, nil
+}
+
+// Solve builds the formula given the current state and returns package assertions
+func (s *Parallel) Solve() (PackagesAssertions, error) {
+	var model map[string]bool
+	var err error
+
+	f, err := s.BuildFormula()
+
+	if err != nil {
+		return nil, err
+	}
+
+	model, _, err = s.solve(f)
+	if err != nil && s.Resolver != nil {
+		return s.Resolver.Solve(f, s)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return DecodeModel(model, s.ParallelDatabase)
+}
+
+// Install given a list of packages, returns package assertions to indicate the packages that must be installed in the system in order
+// to statisfy all the constraints
+func (s *Parallel) Install(c pkg.Packages) (PackagesAssertions, error) {
+
+	coll, err := s.getList(s.DefinitionDatabase, c)
+	if err != nil {
+		return nil, errors.Wrap(err, "Packages not found in definition db")
+	}
+
+	s.Wanted = coll
+
+	if s.noRulesWorld() {
+		var ass PackagesAssertions
+		for _, p := range s.Installed() {
+			ass = append(ass, PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
+
+		}
+		for _, p := range s.Wanted {
+			ass = append(ass, PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
+		}
+		return ass, nil
+	}
+
+	return s.Solve()
+}

pkg/solver/resolver_test.go

@@ -28,13 +28,13 @@ var _ = Describe("Resolver", func() {
 	db := pkg.NewInMemoryDatabase(false)
 	dbInstalled := pkg.NewInMemoryDatabase(false)
 	dbDefinitions := pkg.NewInMemoryDatabase(false)
-	s := NewSolver(dbInstalled, dbDefinitions, db)
+	s := NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 	BeforeEach(func() {
 		db = pkg.NewInMemoryDatabase(false)
 		dbInstalled = pkg.NewInMemoryDatabase(false)
 		dbDefinitions = pkg.NewInMemoryDatabase(false)
-		s = NewSolver(dbInstalled, dbDefinitions, db)
+		s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 	})
 
 	Context("Conflict set", func() {
@@ -79,13 +79,13 @@ var _ = Describe("Resolver", func() {
 				solution, err := s.Install([]pkg.Package{D, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(len(solution)).To(Equal(6))
+				Expect(len(solution)).To(Equal(3))
 
-				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
-				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
-				Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
 
 			})
@@ -112,12 +112,12 @@ var _ = Describe("Resolver", func() {
 				solution, err := s.Install([]pkg.Package{A, D})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
-				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
 
-				Expect(len(solution)).To(Equal(4))
+				Expect(len(solution)).To(Equal(2))
 			})
 
 			It("will find out that we can install D and F by ignoring E and A", func() {
@@ -142,13 +142,13 @@ var _ = Describe("Resolver", func() {
 				solution, err := s.Install([]pkg.Package{A, D, E, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
-				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) // Was already installed
 				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
-				Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
-				Expect(len(solution)).To(Equal(6))
+				Expect(len(solution)).To(Equal(3))
 
 			})
 		})

pkg/solver/solver.go

@@ -26,6 +26,13 @@ import (
 	pkg "github.com/mudler/luet/pkg/package"
 )
 
+type SolverType int
+
+const (
+	SingleCoreSimple = 0
+	ParallelSimple   = iota
+)
+
 // PackageSolver is an interface to a generic package solving algorithm
 type PackageSolver interface {
 	SetDefinitionDatabase(pkg.PackageDatabase)
@@ -36,7 +43,10 @@ type PackageSolver interface {
 	Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error)
 
 	World() pkg.Packages
-	Upgrade(checkconflicts bool) (pkg.Packages, PackagesAssertions, error)
+	Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAssertions, error)
+
+	UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssertions, error)
+	UninstallUniverse(toremove pkg.Packages) (pkg.Packages, error)
 
 	SetResolver(PackageResolver)
 
@@ -53,16 +63,30 @@ type Solver struct {
 	Resolver PackageResolver
 }
 
-// NewSolver accepts as argument two lists of packages, the first is the initial set,
-// the second represent all the known packages.
-func NewSolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase) PackageSolver {
-	return NewResolver(installed, definitiondb, solverdb, &DummyPackageResolver{})
+type Options struct {
+	Type        SolverType
+	Concurrency int
 }
 
-// NewReSolver accepts as argument two lists of packages, the first is the initial set,
+// NewSolver accepts as argument two lists of packages, the first is the initial set,
 // the second represent all the known packages.
-func NewResolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase, re PackageResolver) PackageSolver {
-	return &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb, Resolver: re}
+func NewSolver(t Options, installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase) PackageSolver {
+	return NewResolver(t, installed, definitiondb, solverdb, &DummyPackageResolver{})
+}
+
+// NewResolver accepts as argument two lists of packages, the first is the initial set,
+// the second represent all the known packages.
+// Using constructors as in the future we foresee warmups for hot-restore solver cache
+func NewResolver(t Options, installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase, re PackageResolver) PackageSolver {
+	var s PackageSolver
+	switch t.Type {
+	case SingleCoreSimple:
+		s = &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb, Resolver: re}
+	case ParallelSimple:
+		s = &Parallel{InstalledDatabase: installed, DefinitionDatabase: definitiondb, ParallelDatabase: solverdb, Resolver: re, Concurrency: t.Concurrency}
+	}
+
+	return s
 }
 
 // SetDefinitionDatabase is a setter for the definition Database
@@ -95,9 +119,27 @@ func (s *Solver) noRulesWorld() bool {
 	return true
 }
 
+func (s *Solver) noRulesInstalled() bool {
+	for _, p := range s.Installed() {
+		if len(p.GetConflicts()) != 0 || len(p.GetRequires()) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
 func (s *Solver) BuildInstalled() (bf.Formula, error) {
 	var formulas []bf.Formula
+	var packages pkg.Packages
 	for _, p := range s.Installed() {
+		packages = append(packages, p)
+		for _, dep := range p.Related(s.DefinitionDatabase) {
+			packages = append(packages, dep)
+		}
+	}
+
+	for _, p := range packages {
 		solvable, err := p.BuildFormula(s.DefinitionDatabase, s.SolverDatabase)
 		if err != nil {
 			return nil, err
@@ -125,6 +167,7 @@ func (s *Solver) BuildWorld(includeInstalled bool) (bf.Formula, error) {
 	}
 
 	for _, p := range s.World() {
+
 		solvable, err := p.BuildFormula(s.DefinitionDatabase, s.SolverDatabase)
 		if err != nil {
 			return nil, err
@@ -134,6 +177,44 @@ func (s *Solver) BuildWorld(includeInstalled bool) (bf.Formula, error) {
 	return bf.And(formulas...), nil
 }
 
+// BuildWorld builds the formula which olds the requirements from the package definitions
+// which are available (global state)
+func (s *Solver) BuildPartialWorld(includeInstalled bool) (bf.Formula, error) {
+	var formulas []bf.Formula
+	// NOTE: This block shouldf be enabled in case of very old systems with outdated world sets
+	if includeInstalled {
+		solvable, err := s.BuildInstalled()
+		if err != nil {
+			return nil, err
+		}
+		//f = bf.And(f, solvable)
+		formulas = append(formulas, solvable)
+	}
+
+	var packages pkg.Packages
+	for _, p := range s.Wanted {
+		//	packages = append(packages, p)
+		for _, dep := range p.Related(s.DefinitionDatabase) {
+			packages = append(packages, dep)
+		}
+
+	}
+
+	for _, p := range packages {
+		solvable, err := p.BuildFormula(s.DefinitionDatabase, s.SolverDatabase)
+		if err != nil {
+			return nil, err
+		}
+		formulas = append(formulas, solvable...)
+	}
+
+	if len(formulas) != 0 {
+		return bf.And(formulas...), nil
+	}
+	return bf.True, nil
+
+}
+
 func (s *Solver) getList(db pkg.PackageDatabase, lsp pkg.Packages) (pkg.Packages, error) {
 	var ls pkg.Packages
 
@@ -251,34 +332,191 @@ func (s *Solver) ConflictsWithInstalled(p pkg.Package) (bool, error) {
 	return s.ConflictsWith(p, s.Installed())
 }
 
-func (s *Solver) Upgrade(checkconflicts bool) (pkg.Packages, PackagesAssertions, error) {
+// UninstallUniverse takes a list of candidate package and return a list of packages that would be removed
+// in order to purge the candidate. Uses the solver to check constraints and nothing else
+//
+// It can be compared to the counterpart Uninstall as this method acts like a uninstall --full
+// it removes all the packages and its deps. taking also in consideration other packages that might have
+// revdeps
+func (s *Solver) UninstallUniverse(toremove pkg.Packages) (pkg.Packages, error) {
+
+	if s.noRulesInstalled() {
+		return s.getList(s.InstalledDatabase, toremove)
+	}
+
+	// resolve to packages from the db
+	toRemove, err := s.getList(s.InstalledDatabase, toremove)
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	var formulas []bf.Formula
+	r, err := s.BuildInstalled()
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	// SAT encode the clauses against the world
+	for _, p := range toRemove.Unique() {
+		encodedP, err := p.Encode(s.InstalledDatabase)
+		if err != nil {
+			return nil, errors.Wrap(err, "Package not found in definition db")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(bf.Not(P), r))
+	}
+
+	markedForRemoval := pkg.Packages{}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, errors.New("Failed finding a solution")
+	}
+	assertion, err := DecodeModel(model, s.InstalledDatabase)
+	if err != nil {
+		return nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+	}
+	return markedForRemoval, nil
+}
+
+// UpgradeUniverse mark packages for removal and returns a solution. It considers
+// the Universe db as authoritative
+// See also on the subject: https://arxiv.org/pdf/1007.1021.pdf
+func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssertions, error) {
+	// we first figure out which aren't up-to-date
+	// which has to be removed
+	// and which needs to be upgraded
+	notUptodate := pkg.Packages{}
+	removed := pkg.Packages{}
+	toUpgrade := pkg.Packages{}
+
+	// TODO: this is memory expensive, we need to optimize this
+	universe := pkg.NewInMemoryDatabase(false)
+	for _, p := range s.DefinitionDatabase.World() {
+		universe.CreatePackage(p)
+	}
+	for _, p := range s.Installed() {
+		universe.CreatePackage(p)
+	}
+
+	// Grab all the installed ones, see if they are eligible for update
+	for _, p := range s.Installed() {
+		available, err := universe.FindPackageVersions(p)
+		if err != nil {
+			removed = append(removed, p)
+		}
+		if len(available) == 0 {
+			continue
+		}
+
+		bestmatch := available.Best(nil)
+		// Found a better version available
+		if !bestmatch.Matches(p) {
+			notUptodate = append(notUptodate, p)
+			toUpgrade = append(toUpgrade, bestmatch)
+		}
+	}
+
+	var formulas []bf.Formula
+
+	// Build constraints for the whole defdb
+	r, err := s.BuildWorld(true)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't build world constraints")
+	}
+
+	// Treat removed packages from universe as marked for deletion
+	if dropremoved {
+		notUptodate = append(notUptodate, removed...)
+	}
+
+	// SAT encode the clauses against the world
+	for _, p := range notUptodate.Unique() {
+		encodedP, err := p.Encode(universe)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "couldn't encode package")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(bf.Not(P), r))
+	}
+
+	for _, p := range toUpgrade {
+		encodedP, err := p.Encode(universe)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "couldn't encode package")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(P, r))
+	}
+
+	markedForRemoval := pkg.Packages{}
+
+	if len(formulas) == 0 {
+		return pkg.Packages{}, PackagesAssertions{}, nil
+	}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, nil, errors.New("Failed finding a solution")
+	}
+
+	assertion, err := DecodeModel(model, universe)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+
+	}
+	return markedForRemoval, assertion, nil
+}
+
+func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAssertions, error) {
 
 	// First get candidates that needs to be upgraded..
 
 	toUninstall := pkg.Packages{}
 	toInstall := pkg.Packages{}
 
-	availableCache := map[string]pkg.Packages{}
+	// we do this in memory so we take into account of provides
+	universe := pkg.NewInMemoryDatabase(false)
 	for _, p := range s.DefinitionDatabase.World() {
-		// Each one, should be expanded
-		availableCache[p.GetName()+p.GetCategory()] = append(availableCache[p.GetName()+p.GetCategory()], p)
+		universe.CreatePackage(p)
 	}
 
 	installedcopy := pkg.NewInMemoryDatabase(false)
 
 	for _, p := range s.InstalledDatabase.World() {
 		installedcopy.CreatePackage(p)
-		packages, ok := availableCache[p.GetName()+p.GetCategory()]
-		if ok && len(packages) != 0 {
+		packages, err := universe.FindPackageVersions(p)
+		if err == nil && len(packages) != 0 {
 			best := packages.Best(nil)
-			if best.GetVersion() != p.GetVersion() {
+			if !best.Matches(p) {
 				toUninstall = append(toUninstall, p)
 				toInstall = append(toInstall, best)
 			}
 		}
 	}
-	s2 := NewSolver(installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+
+	s2 := NewSolver(Options{Type: SingleCoreSimple}, installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
 	s2.SetResolver(s.Resolver)
+	if !full {
+		ass := PackagesAssertions{}
+		for _, i := range toInstall {
+			ass = append(ass, PackageAssert{Package: i.(*pkg.DefaultPackage), Value: true})
+		}
+	}
 	// Then try to uninstall the versions in the system, and store that tree
 	for _, p := range toUninstall {
 		r, err := s.Uninstall(p, checkconflicts, false)
@@ -291,7 +529,9 @@ func (s *Solver) Upgrade(checkconflicts bool) (pkg.Packages, PackagesAssertions,
 				return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't remove copy of package targetted for removal")
 			}
 		}
-
+	}
+	if len(toInstall) == 0 {
+		return toUninstall, PackagesAssertions{}, nil
 	}
 	r, e := s2.Install(toInstall)
 	return toUninstall, r, e
@@ -344,7 +584,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 		}
 	}
 
-	s2 := NewSolver(pkg.NewInMemoryDatabase(false), s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+	s2 := NewSolver(Options{Type: SingleCoreSimple}, pkg.NewInMemoryDatabase(false), s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
 	s2.SetResolver(s.Resolver)
 	// Get the requirements to install the candidate
 	asserts, err := s2.Install(pkg.Packages{candidate})
@@ -354,7 +594,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 	for _, a := range asserts {
 		if a.Value {
 			if !checkconflicts {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 				continue
 			}
 
@@ -365,7 +605,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 
 			// If doesn't conflict with installed we just consider it for removal and look for the next one
 			if !c {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 				continue
 			}
 
@@ -375,7 +615,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 				return nil, err
 			}
 			if !c {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 			}
 
 		}
@@ -388,16 +628,18 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 // BuildFormula builds the main solving formula that is evaluated by the sat solver.
 func (s *Solver) BuildFormula() (bf.Formula, error) {
 	var formulas []bf.Formula
-	r, err := s.BuildWorld(false)
+	r, err := s.BuildPartialWorld(false)
 	if err != nil {
 		return nil, err
 	}
+
 	for _, wanted := range s.Wanted {
 		encodedW, err := wanted.Encode(s.SolverDatabase)
 		if err != nil {
 			return nil, err
 		}
 		W := bf.Var(encodedW)
+		//	allW = append(allW, W)
 		installedWorld := s.Installed()
 		//TODO:Optimize
 		if len(installedWorld) == 0 {
@@ -415,8 +657,8 @@ func (s *Solver) BuildFormula() (bf.Formula, error) {
 		}
 
 	}
-	formulas = append(formulas, r)
 
+	formulas = append(formulas, r)
 	return bf.And(formulas...), nil
 }
 

pkg/solver/solver_test.go

@@ -28,13 +28,13 @@ var _ = Describe("Solver", func() {
 	db := pkg.NewInMemoryDatabase(false)
 	dbInstalled := pkg.NewInMemoryDatabase(false)
 	dbDefinitions := pkg.NewInMemoryDatabase(false)
-	s := NewSolver(dbInstalled, dbDefinitions, db)
+	s := NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 	BeforeEach(func() {
 		db = pkg.NewInMemoryDatabase(false)
 		dbInstalled = pkg.NewInMemoryDatabase(false)
 		dbDefinitions = pkg.NewInMemoryDatabase(false)
-		s = NewSolver(dbInstalled, dbDefinitions, db)
+		s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 	})
 	Context("Simple set", func() {
 		It("Solves correctly if the selected package has no requirements or conflicts and we have nothing installed yet", func() {
@@ -52,7 +52,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
@@ -75,7 +75,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{B})
 			Expect(err).ToNot(HaveOccurred())
@@ -101,17 +101,17 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: true}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false}))
+			//	Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+			//Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false}))
 
-			Expect(len(solution)).To(Equal(5))
+			Expect(len(solution)).To(Equal(3))
 		})
 
 		It("Solves correctly if the selected package to install has requirements", func() {
@@ -130,7 +130,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
@@ -156,7 +156,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
@@ -181,7 +181,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -209,7 +209,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -236,7 +236,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -263,7 +263,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{C})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -291,7 +291,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{&pkg.DefaultPackage{Name: "c", Version: ">1.0", Category: "test"}})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -317,7 +317,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -345,7 +345,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A, B})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -391,7 +391,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{C})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -431,7 +431,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A, B})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -476,7 +476,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A2, B})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
@@ -514,7 +514,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A2})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
@@ -555,7 +555,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A2})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
@@ -574,57 +574,366 @@ var _ = Describe("Solver", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 
-		It("Uninstalls simple package correctly", func() {
+		Context("Uninstall", func() {
+			It("Uninstalls simple package correctly", func() {
 
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+
+				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("Uninstalls simple package expanded correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1.2", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+
+				solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"}, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
 
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
+				Expect(solution).To(ContainElement(A))
 
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("Uninstalls simple packages not in world correctly", func() {
 
-			//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
-			Expect(len(solution)).To(Equal(1))
-		})
-		It("Uninstalls simple package expanded correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "1.2", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("Uninstalls complex packages not in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
 
-			solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"}, true, true)
-			Expect(err).ToNot(HaveOccurred())
+				Expect(solution).To(ContainElement(A))
 
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("Uninstalls complex packages correctly, even if shared deps are required by system packages", func() {
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.Uninstall(A, true, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).ToNot(ContainElement(B))
+
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("Uninstalls complex packages in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{C}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Uninstall(A, true, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(C))
+
+				Expect(len(solution)).To(Equal(2))
+			})
+
+			It("Uninstalls complex package correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				//	C // installed
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Uninstall(A, true, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(D))
+
+				Expect(len(solution)).To(Equal(3))
+
+			})
+
+			It("UninstallUniverse simple package correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("UninstallUniverse simple package expanded correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1.2", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+
+				solution, err := s.UninstallUniverse(pkg.Packages{
+					&pkg.DefaultPackage{Name: "A", Version: ">1.0"}})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("UninstallUniverse simple packages not in world correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("UninstallUniverse complex packages not in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+
+				Expect(len(solution)).To(Equal(2))
+			})
+
+			It("UninstallUniverse complex packages correctly, even if shared deps are required by system packages", func() {
+				// Here we diff a lot from standard Uninstall:
+				// all the packages that has reverse deps will be removed (aka --full)
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(C))
+
+				Expect(len(solution)).To(Equal(3))
+			})
+
+			It("UninstallUniverse complex packages in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{C}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(C))
+
+				Expect(len(solution)).To(Equal(2))
+			})
+
+			It("UninstallUniverse complex package correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				//	C // installed
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(D))
+
+				Expect(len(solution)).To(Equal(3))
+
+			})
 
-			//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
-			Expect(len(solution)).To(Equal(1))
 		})
 		It("Find conflicts", func() {
 
@@ -810,131 +1119,6 @@ var _ = Describe("Solver", func() {
 			Expect(val).ToNot(BeTrue())
 		})
 
-		It("Uninstalls simple packages not in world correctly", func() {
-
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-
-			for _, p := range []pkg.Package{B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-
-			//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
-			Expect(len(solution)).To(Equal(1))
-		})
-
-		It("Uninstalls complex packages not in world correctly", func() {
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
-
-			for _, p := range []pkg.Package{B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(1))
-		})
-
-		It("Uninstalls complex packages correctly, even if shared deps are required by system packages", func() {
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-			Expect(solution).ToNot(ContainElement(B.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(1))
-		})
-
-		It("Uninstalls complex packages in world correctly", func() {
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{C}, []*pkg.DefaultPackage{})
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-			Expect(solution).To(ContainElement(C.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(2))
-		})
-
-		It("Uninstalls complex package correctly", func() {
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
-			//	C // installed
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-			Expect(solution).To(ContainElement(B.IsFlagged(false)))
-			Expect(solution).To(ContainElement(D.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(3))
-
-		})
-
 	})
 
 	Context("Conflict set", func() {
@@ -1025,7 +1209,6 @@ var _ = Describe("Solver", func() {
 		})
 	})
 	Context("Upgrades", func() {
-
 		C := pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
 		C.SetCategory("test")
 		B := pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
@@ -1035,6 +1218,17 @@ var _ = Describe("Solver", func() {
 		A1 := pkg.NewPackage("a", "1.2", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
 		A1.SetCategory("test")
 
+		BeforeEach(func() {
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+			A1 = pkg.NewPackage("a", "1.2", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A1.SetCategory("test")
+		})
+
 		It("upgrades correctly", func() {
 			for _, p := range []pkg.Package{A1, B, C} {
 				_, err := dbDefinitions.CreatePackage(p)
@@ -1045,7 +1239,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			uninstall, solution, err := s.Upgrade(true)
+			uninstall, solution, err := s.Upgrade(true, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(uninstall)).To(Equal(1))
@@ -1056,7 +1250,56 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
 			Expect(len(solution)).To(Equal(3))
+		})
 
+		It("upgrades correctly with provides", func() {
+			B.SetProvides([]*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=0", Category: "test"}, &pkg.DefaultPackage{Name: "c", Version: ">=0", Category: "test"}})
+
+			for _, p := range []pkg.Package{A1, B} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, C} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.Upgrade(true, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(2))
+			Expect(uninstall[1].GetName()).To(Equal("c"))
+			Expect(uninstall[1].GetVersion()).To(Equal("1.5"))
+			Expect(uninstall[0].GetName()).To(Equal("a"))
+			Expect(uninstall[0].GetVersion()).To(Equal("1.1"))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(len(solution)).To(Equal(1))
+		})
+
+		It("UpgradeUniverse upgrades correctly", func() {
+			for _, p := range []pkg.Package{A1, B, C} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.UpgradeUniverse(true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(1))
+			Expect(uninstall[0].GetName()).To(Equal("a"))
+			Expect(uninstall[0].GetVersion()).To(Equal("1.1"))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+
+			Expect(len(solution)).To(Equal(4))
 		})
 	})
 })

pkg/spectooling/definition.go

@@ -0,0 +1,129 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>,
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package spectooling
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"gopkg.in/yaml.v2"
+)
+
+type DefaultPackageSanitized struct {
+	Name             string                     `json:"name" yaml:"name"`
+	Version          string                     `json:"version" yaml:"version"`
+	Category         string                     `json:"category" yaml:"category"`
+	UseFlags         []string                   `json:"use_flags,omitempty" yaml:"use_flags,omitempty"`
+	PackageRequires  []*DefaultPackageSanitized `json:"requires,omitempty" yaml:"requires,omitempty"`
+	PackageConflicts []*DefaultPackageSanitized `json:"conflicts,omitempty" yaml:"conflicts,omitempty"`
+	Provides         []*DefaultPackageSanitized `json:"provides,omitempty" yaml:"provides,omitempty"`
+
+	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
+
+	// Path is set only internally when tree is loaded from disk
+	Path string `json:"path,omitempty" yaml:"path,omitempty"`
+
+	Description string   `json:"description,omitempty" yaml:"description,omitempty"`
+	Uri         []string `json:"uri,omitempty" yaml:"uri,omitempty"`
+	License     string   `json:"license,omitempty" yaml:"license,omitempty"`
+	Hidden      bool     `json:"hidden,omitempty" yaml:"hidden,omitempty"`
+
+	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
+}
+
+func NewDefaultPackageSanitizedFromYaml(data []byte) (*DefaultPackageSanitized, error) {
+	ans := &DefaultPackageSanitized{}
+	if err := yaml.Unmarshal(data, ans); err != nil {
+		return nil, err
+	}
+	return ans, nil
+}
+
+func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
+	ans := &DefaultPackageSanitized{
+		Name:        p.GetName(),
+		Version:     p.GetVersion(),
+		Category:    p.GetCategory(),
+		UseFlags:    p.GetUses(),
+		Hidden:      p.IsHidden(),
+		Path:        p.GetPath(),
+		Description: p.GetDescription(),
+		Uri:         p.GetURI(),
+		License:     p.GetLicense(),
+		Labels:      p.GetLabels(),
+		Annotations: p.GetAnnotations(),
+	}
+
+	if p.GetRequires() != nil && len(p.GetRequires()) > 0 {
+		ans.PackageRequires = []*DefaultPackageSanitized{}
+		for _, r := range p.GetRequires() {
+			// I avoid recursive call of NewDefaultPackageSanitized
+			ans.PackageRequires = append(ans.PackageRequires,
+				&DefaultPackageSanitized{
+					Name:     r.Name,
+					Version:  r.Version,
+					Category: r.Category,
+					Hidden:   r.IsHidden(),
+				},
+			)
+		}
+	}
+
+	if p.GetConflicts() != nil && len(p.GetConflicts()) > 0 {
+		ans.PackageConflicts = []*DefaultPackageSanitized{}
+		for _, c := range p.GetConflicts() {
+			// I avoid recursive call of NewDefaultPackageSanitized
+			ans.PackageConflicts = append(ans.PackageConflicts,
+				&DefaultPackageSanitized{
+					Name:     c.Name,
+					Version:  c.Version,
+					Category: c.Category,
+					Hidden:   c.IsHidden(),
+				},
+			)
+		}
+	}
+
+	if p.GetProvides() != nil && len(p.GetProvides()) > 0 {
+		ans.Provides = []*DefaultPackageSanitized{}
+		for _, prov := range p.GetProvides() {
+			// I avoid recursive call of NewDefaultPackageSanitized
+			ans.Provides = append(ans.Provides,
+				&DefaultPackageSanitized{
+					Name:     prov.Name,
+					Version:  prov.Version,
+					Category: prov.Category,
+					Hidden:   prov.IsHidden(),
+				},
+			)
+		}
+	}
+
+	return ans
+}
+
+func (p *DefaultPackageSanitized) Yaml() ([]byte, error) {
+	return yaml.Marshal(p)
+}
+
+func (p *DefaultPackageSanitized) Clone() (*DefaultPackageSanitized, error) {
+	data, err := p.Yaml()
+	if err != nil {
+		return nil, err
+	}
+
+	return NewDefaultPackageSanitizedFromYaml(data)
+}

pkg/spectooling/package_test.go

@@ -0,0 +1,87 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>,
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package spectooling_test
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+	. "github.com/mudler/luet/pkg/spectooling"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Spec Tooling", func() {
+	Context("Conversion1", func() {
+
+		b := pkg.NewPackage("B", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		c := pkg.NewPackage("C", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		d := pkg.NewPackage("D", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		p1 := pkg.NewPackage("A", "1.0", []*pkg.DefaultPackage{b, c}, []*pkg.DefaultPackage{d})
+		virtual := pkg.NewPackage("E", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		virtual.SetCategory("virtual")
+		p1.Provides = []*pkg.DefaultPackage{virtual}
+		p1.AddLabel("label1", "value1")
+		p1.AddLabel("label2", "value2")
+		p1.SetDescription("Package1")
+		p1.SetCategory("cat1")
+		p1.SetLicense("GPL")
+		p1.AddURI("https://github.com/mudler/luet")
+		p1.AddUse("systemd")
+		It("Convert pkg1", func() {
+			res := NewDefaultPackageSanitized(p1)
+			expected_res := &DefaultPackageSanitized{
+				Name:     "A",
+				Version:  "1.0",
+				Category: "cat1",
+				PackageRequires: []*DefaultPackageSanitized{
+					&DefaultPackageSanitized{
+						Name:    "B",
+						Version: "1.0",
+					},
+					&DefaultPackageSanitized{
+						Name:    "C",
+						Version: "1.0",
+					},
+				},
+				PackageConflicts: []*DefaultPackageSanitized{
+					&DefaultPackageSanitized{
+						Name:    "D",
+						Version: "1.0",
+					},
+				},
+				Provides: []*DefaultPackageSanitized{
+					&DefaultPackageSanitized{
+						Name:     "E",
+						Category: "virtual",
+						Version:  "1.0",
+					},
+				},
+				Labels: map[string]string{
+					"label1": "value1",
+					"label2": "value2",
+				},
+				Description: "Package1",
+				License:     "GPL",
+				Uri:         []string{"https://github.com/mudler/luet"},
+				UseFlags:    []string{"systemd"},
+			}
+
+			Expect(res).To(Equal(expected_res))
+		})
+
+	})
+})

pkg/spectooling/spectooling_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>,
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package spectooling_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Spec Tooling Suite")
+}

pkg/tree/builder/gentoo/simpleparser.go

@@ -353,6 +353,12 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) (pkg.Packages, error) {
 		return pkg.Packages{}, err
 	}
 
+	// Retrieve slot
+	slot, ok := vars["SLOT"]
+	if ok && slot.String() != "0" {
+		pack.SetCategory(fmt.Sprintf("%s-%s", gp.Category, slot.String()))
+	}
+
 	// TODO: Handle this a bit better
 	iuse, ok := vars["IUSE"]
 	if ok {
@@ -417,6 +423,7 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) (pkg.Packages, error) {
 		for _, d := range gRDEPEND.GetDependencies() {
 
 			//TODO: Resolve to db or create a new one.
+			//TODO: handle SLOT too.
 			dep := &pkg.DefaultPackage{
 				Name:     d.Dep.Name,
 				Version:  d.Dep.Version + d.Dep.VersionSuffix,

pkg/tree/compiler_recipe.go

@@ -74,39 +74,92 @@ func (r *CompilerRecipe) Load(path string) error {
 			return errors.Wrap(err, "Error on walk path "+currentpath)
 		}
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
-		pack, err := ReadDefinitionFile(currentpath)
-		if err != nil {
-			return err
-		}
-		// Path is set only internally when tree is loaded from disk
-		pack.SetPath(filepath.Dir(currentpath))
+		switch info.Name() {
+		case DefinitionFile:
 
-		// Instead of rdeps, have a different tree for build deps.
-		compileDefPath := pack.Rel(CompilerDefinitionFile)
-		if helpers.Exists(compileDefPath) {
-			dat, err := ioutil.ReadFile(compileDefPath)
+			pack, err := ReadDefinitionFile(currentpath)
 			if err != nil {
-				return errors.Wrap(err,
-					"Error reading file "+CompilerDefinitionFile+" from "+
-						filepath.Dir(currentpath))
+				return err
+			}
+			// Path is set only internally when tree is loaded from disk
+			pack.SetPath(filepath.Dir(currentpath))
+
+			// Instead of rdeps, have a different tree for build deps.
+			compileDefPath := pack.Rel(CompilerDefinitionFile)
+			if helpers.Exists(compileDefPath) {
+
+				dat, err := helpers.RenderFiles(compileDefPath, currentpath)
+				if err != nil {
+					return errors.Wrap(err,
+						"Error templating file "+CompilerDefinitionFile+" from "+
+							filepath.Dir(currentpath))
+				}
+
+				packbuild, err := pkg.DefaultPackageFromYaml([]byte(dat))
+				if err != nil {
+					return errors.Wrap(err,
+						"Error reading yaml "+CompilerDefinitionFile+" from "+
+							filepath.Dir(currentpath))
+				}
+				pack.Requires(packbuild.GetRequires())
+				pack.Conflicts(packbuild.GetConflicts())
+			}
+
+			_, err = r.Database.CreatePackage(&pack)
+			if err != nil {
+				return errors.Wrap(err, "Error creating package "+pack.GetName())
+			}
+
+		case CollectionFile:
+			dat, err := ioutil.ReadFile(currentpath)
+			if err != nil {
+				return errors.Wrap(err, "Error reading file "+currentpath)
+			}
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			packsRaw, err := pkg.GetRawPackages(dat)
+
+			for _, pack := range packs {
+				pack.SetPath(filepath.Dir(currentpath))
+
+				// Instead of rdeps, have a different tree for build deps.
+				compileDefPath := pack.Rel(CompilerDefinitionFile)
+				if helpers.Exists(compileDefPath) {
+
+					raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
+					buildyaml, err := ioutil.ReadFile(compileDefPath)
+					if err != nil {
+						return errors.Wrap(err, "Error reading file "+currentpath)
+					}
+					dat, err := helpers.RenderHelm(string(buildyaml), raw)
+					if err != nil {
+						return errors.Wrap(err,
+							"Error templating file "+CompilerDefinitionFile+" from "+
+								filepath.Dir(currentpath))
+					}
+
+					packbuild, err := pkg.DefaultPackageFromYaml([]byte(dat))
+					if err != nil {
+						return errors.Wrap(err,
+							"Error reading yaml "+CompilerDefinitionFile+" from "+
+								filepath.Dir(currentpath))
+					}
+					pack.Requires(packbuild.GetRequires())
+					pack.Conflicts(packbuild.GetConflicts())
+				}
+
+				_, err = r.Database.CreatePackage(&pack)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+pack.GetName())
+				}
 			}
-			packbuild, err := pkg.DefaultPackageFromYaml(dat)
-			if err != nil {
-				return errors.Wrap(err,
-					"Error reading yaml "+CompilerDefinitionFile+" from "+
-						filepath.Dir(currentpath))
-			}
-			pack.Requires(packbuild.GetRequires())
-			pack.Conflicts(packbuild.GetConflicts())
-		}
 
-		_, err = r.Database.CreatePackage(&pack)
-		if err != nil {
-			return errors.Wrap(err, "Error creating package "+pack.GetName())
 		}
 
 		return nil

pkg/tree/installer_recipe.go

@@ -21,6 +21,7 @@
 package tree
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -70,10 +71,12 @@ func (r *InstallerRecipe) Save(path string) error {
 
 func (r *InstallerRecipe) Load(path string) error {
 
-	// tmpfile, err := ioutil.TempFile("", "luet")
-	// if err != nil {
-	// 	return err
-	// }
+	if !helpers.Exists(path) {
+		return errors.New(fmt.Sprintf(
+			"Path %s doesn't exit.", path,
+		))
+	}
+
 	r.SourcePath = append(r.SourcePath, path)
 
 	//r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))
@@ -82,7 +85,7 @@ func (r *InstallerRecipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
@@ -90,16 +93,35 @@ func (r *InstallerRecipe) Load(path string) error {
 		if err != nil {
 			return errors.Wrap(err, "Error reading file "+currentpath)
 		}
-		pack, err := pkg.DefaultPackageFromYaml(dat)
-		if err != nil {
-			return errors.Wrap(err, "Error reading yaml "+currentpath)
-		}
 
-		// Path is set only internally when tree is loaded from disk
-		pack.SetPath(filepath.Dir(currentpath))
-		_, err = r.Database.CreatePackage(&pack)
-		if err != nil {
-			return errors.Wrap(err, "Error creating package "+pack.GetName())
+		switch info.Name() {
+		case DefinitionFile:
+			pack, err := pkg.DefaultPackageFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+
+			// Path is set only internally when tree is loaded from disk
+			pack.SetPath(filepath.Dir(currentpath))
+			_, err = r.Database.CreatePackage(&pack)
+			if err != nil {
+				return errors.Wrap(err, "Error creating package "+pack.GetName())
+			}
+
+		case CollectionFile:
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			for _, p := range packs {
+				// Path is set only internally when tree is loaded from disk
+				p.SetPath(filepath.Dir(currentpath))
+				_, err = r.Database.CreatePackage(&p)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+p.GetName())
+				}
+			}
+
 		}
 
 		return nil

pkg/tree/recipes.go

@@ -21,16 +21,20 @@
 package tree
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
+	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	spectooling "github.com/mudler/luet/pkg/spectooling"
 	"github.com/pkg/errors"
 )
 
 const (
 	DefinitionFile = "definition.yaml"
+	CollectionFile = "collection.yaml"
 )
 
 func NewGeneralRecipe(db pkg.PackageDatabase) Builder { return &Recipe{Database: db} }
@@ -42,7 +46,7 @@ type Recipe struct {
 }
 
 func WriteDefinitionFile(p pkg.Package, definitionFilePath string) error {
-	data, err := p.Yaml()
+	data, err := spectooling.NewDefaultPackageSanitized(p).Yaml()
 	if err != nil {
 		return err
 	}
@@ -73,6 +77,12 @@ func (r *Recipe) Load(path string) error {
 	// if err != nil {
 	// 	return err
 	// }
+	if !helpers.Exists(path) {
+		return errors.New(fmt.Sprintf(
+			"Path %s doesn't exit.", path,
+		))
+	}
+
 	r.SourcePath = append(r.SourcePath, path)
 
 	if r.Database == nil {
@@ -85,7 +95,7 @@ func (r *Recipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
@@ -93,16 +103,34 @@ func (r *Recipe) Load(path string) error {
 		if err != nil {
 			return errors.Wrap(err, "Error reading file "+currentpath)
 		}
-		pack, err := pkg.DefaultPackageFromYaml(dat)
-		if err != nil {
-			return errors.Wrap(err, "Error reading yaml "+currentpath)
-		}
 
-		// Path is set only internally when tree is loaded from disk
-		pack.SetPath(filepath.Dir(currentpath))
-		_, err = r.Database.CreatePackage(&pack)
-		if err != nil {
-			return errors.Wrap(err, "Error creating package "+pack.GetName())
+		switch info.Name() {
+		case DefinitionFile:
+			pack, err := pkg.DefaultPackageFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+
+			// Path is set only internally when tree is loaded from disk
+			pack.SetPath(filepath.Dir(currentpath))
+			_, err = r.Database.CreatePackage(&pack)
+			if err != nil {
+				return errors.Wrap(err, "Error creating package "+pack.GetName())
+			}
+		case CollectionFile:
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			for _, p := range packs {
+				// Path is set only internally when tree is loaded from disk
+				p.SetPath(filepath.Dir(currentpath))
+				_, err = r.Database.CreatePackage(&p)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+p.GetName())
+				}
+			}
+
 		}
 
 		return nil

pkg/tree/recipes_test.go

@@ -117,10 +117,10 @@ var _ = Describe("Recipe", func() {
 				}) // Note: the definition depends on pinentry-base without an explicit version
 				Expect(err).ToNot(HaveOccurred())
 
-				s := solver.NewSolver(pkg.NewInMemoryDatabase(false), tree, tree)
+				s := solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false), tree, tree)
 				solution, err := s.Install([]pkg.Package{pack})
 				Expect(err).ToNot(HaveOccurred())
-				Expect(len(solution)).To(Equal(33))
+				Expect(len(solution)).To(Equal(14))
 
 				var allSol string
 				for _, sol := range solution {

pkg/tree/tree_test.go

@@ -23,6 +23,7 @@ package tree_test
 import (
 	"io/ioutil"
 	"os"
+	"regexp"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -58,7 +59,7 @@ var _ = Describe("Tree", func() {
 				Expect(len(CfromD.GetRequires()) != 0).To(BeTrue())
 				Expect(CfromD.GetRequires()[0].GetName()).To(Equal("b"))
 
-				s := solver.NewSolver(pkg.NewInMemoryDatabase(false), generalRecipe.GetDatabase(), db)
+				s := solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false), generalRecipe.GetDatabase(), db)
 				pack, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 				Expect(err).ToNot(HaveOccurred())
 
@@ -68,31 +69,25 @@ var _ = Describe("Tree", func() {
 				solution, err = solution.Order(generalRecipe.GetDatabase(), pack.GetFingerPrint())
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution[0].Package.GetName()).To(Equal("a"))
-				Expect(solution[0].Value).To(BeFalse())
+				Expect(solution[0].Package.GetName()).To(Equal("b"))
+				Expect(solution[0].Value).To(BeTrue())
 
-				Expect(solution[1].Package.GetName()).To(Equal("b"))
+				Expect(solution[1].Package.GetName()).To(Equal("c"))
 				Expect(solution[1].Value).To(BeTrue())
 
-				Expect(solution[2].Package.GetName()).To(Equal("c"))
+				Expect(solution[2].Package.GetName()).To(Equal("d"))
 				Expect(solution[2].Value).To(BeTrue())
-
-				Expect(solution[3].Package.GetName()).To(Equal("d"))
-				Expect(solution[3].Value).To(BeTrue())
-				Expect(len(solution)).To(Equal(4))
+				Expect(len(solution)).To(Equal(3))
 
 				newsolution := solution.Drop(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
-				Expect(len(newsolution)).To(Equal(3))
+				Expect(len(newsolution)).To(Equal(2))
 
-				Expect(newsolution[0].Package.GetName()).To(Equal("a"))
-				Expect(newsolution[0].Value).To(BeFalse())
+				Expect(newsolution[0].Package.GetName()).To(Equal("b"))
+				Expect(newsolution[0].Value).To(BeTrue())
 
-				Expect(newsolution[1].Package.GetName()).To(Equal("b"))
+				Expect(newsolution[1].Package.GetName()).To(Equal("c"))
 				Expect(newsolution[1].Value).To(BeTrue())
 
-				Expect(newsolution[2].Package.GetName()).To(Equal("c"))
-				Expect(newsolution[2].Value).To(BeTrue())
-
 			}
 		})
 	})
@@ -130,7 +125,7 @@ var _ = Describe("Tree", func() {
 				Expect(len(CfromD.GetRequires()) != 0).To(BeTrue())
 				Expect(CfromD.GetRequires()[0].GetName()).To(Equal("b"))
 
-				s := solver.NewSolver(pkg.NewInMemoryDatabase(false), generalRecipe.GetDatabase(), db)
+				s := solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false), generalRecipe.GetDatabase(), db)
 
 				Dd, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 				Expect(err).ToNot(HaveOccurred())
@@ -145,11 +140,11 @@ var _ = Describe("Tree", func() {
 
 				base, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "base", Category: "layer", Version: "0.2"})
 				Expect(err).ToNot(HaveOccurred())
-				Expect(solution).To(ContainElement(solver.PackageAssert{Package: pack.(*pkg.DefaultPackage), Value: false}))
+				Expect(solution).ToNot(ContainElement(solver.PackageAssert{Package: pack.(*pkg.DefaultPackage), Value: true}))
 				Expect(solution).To(ContainElement(solver.PackageAssert{Package: D.(*pkg.DefaultPackage), Value: true}))
-				Expect(solution).To(ContainElement(solver.PackageAssert{Package: extra.(*pkg.DefaultPackage), Value: false}))
-				Expect(solution).To(ContainElement(solver.PackageAssert{Package: base.(*pkg.DefaultPackage), Value: false}))
-				Expect(len(solution)).To(Equal(6))
+				Expect(solution).ToNot(ContainElement(solver.PackageAssert{Package: extra.(*pkg.DefaultPackage), Value: true}))
+				Expect(solution).ToNot(ContainElement(solver.PackageAssert{Package: base.(*pkg.DefaultPackage), Value: true}))
+				Expect(len(solution)).To(Equal(3))
 			}
 		})
 	})
@@ -171,4 +166,23 @@ var _ = Describe("Tree", func() {
 		})
 	})
 
+	Context("Simple tree with annotations", func() {
+		It("Read tree with annotations", func() {
+			db := pkg.NewInMemoryDatabase(false)
+			generalRecipe := NewCompilerRecipe(db)
+			r := regexp.MustCompile("^label")
+
+			err := generalRecipe.Load("../../tests/fixtures/annotations")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().World())).To(Equal(1))
+			pack, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "pkgA", Category: "test", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack.HasAnnotation("label1")).To(Equal(true))
+			Expect(pack.HasAnnotation("label3")).To(Equal(false))
+			Expect(pack.MatchAnnotation(r)).To(Equal(true))
+		})
+	})
+
 })

tests/fixtures/annotations/pkgA/0.1/build.yaml

@@ -0,0 +1,3 @@
+image: "alpine"
+steps:
+  - echo "test" > /file1

tests/fixtures/annotations/pkgA/0.1/definition.yaml

@@ -0,0 +1,6 @@
+category: "test"
+name: "pkgA"
+version: "0.1"
+annotations:
+  label1: "value1"
+  label2: "value2"

tests/fixtures/caps/pkgA/0.1/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+- apk add libcap
+unpack: true
+includes:
+  - /file1
+steps:
+- echo "test" > /file1
+- setcap cap_net_raw+ep /file1

tests/fixtures/caps/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "caps"
+version: "0.1"

tests/fixtures/caps/pkgB/0.1/build.yaml

@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+- apk add libcap
+steps:
+- echo "test" > /file2
+- setcap cap_net_raw+ep /file2

tests/fixtures/caps/pkgB/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "caps2"
+version: "0.1"

tests/fixtures/collections/build.yaml

@@ -0,0 +1,6 @@
+image: quay.io/mocaccino/extra
+steps:
+- touch /{{.Values.name}}
+- touch /build-extra-{{.Values.foo}}
+
+unpack: true

tests/fixtures/collections/collection.yaml

@@ -0,0 +1,13 @@
+packages:
+- name: "a"
+  category: "distro"
+  version: "0.1"
+  foo: "baz"
+- name: "b"
+  category: "distro"
+  version: "0.3"
+  foo: "f"
+- name: "c"
+  category: "distro"
+  version: "0.3"
+  foo: "bar"

tests/fixtures/collections/finalize.yaml

@@ -0,0 +1,2 @@
+install:
+- touch /finalize-{{.Values.name}}

tests/fixtures/config_protect/a/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo c > /c
+  - echo c > /cd
+  - mkdir /etc/a
+  - echo config > /etc/a/conf

tests/fixtures/config_protect/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.0"