Tag 0.10.1

Grab latest version, if not get a default since luet auto updates.
Check is ran as normal user or sudo/root.
Autoinstall repos -y.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: mudler
+
+---
+
+<!-- Thanks for helping us to improve Luet! We welcome all bug reports. Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
+
+**Luet version:**
+<!-- Provide the output from "luet --version" -->
+
+**CPU architecture, OS, and Version:**
+<!-- Provide the output from "uname -a"  -->
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+<!-- Steps to reproduce the behavior, including the luet command used -->
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Logs**
+<!-- If applicable, add logs with the "--debug" flag enabled to help explain your problem. -->
+
+**Additional context**
+<!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: mudler
+
+---
+
+<!-- Thanks for helping us to improve Luet! We welcome all feature requests. Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
+
+**Is your feature request related to a problem? Please describe.**
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]  -->
+
+**Describe the solution you'd like**
+<!-- A clear and concise description of what you want to happen.  -->
+
+**Describe alternatives you've considered**
+<!-- A clear and concise description of any alternative solutions or features you've considered.  -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/workflows/release.yml

@@ -11,10 +11,22 @@ jobs:
           go-version: 1.14.x
       - name: Checkout code
         uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: sudo apt-get install -y upx
       - name: Tests
-        run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage
+        run: |
+              sudo -E \
+                    env "PATH=$PATH" \
+                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                    make deps multiarch-build-small test-integration test-coverage
       - name: Build
-        run: sudo -E env "PATH=$PATH" make multiarch-build && sudo chmod -R 777 release/
+        run: sudo -E env "PATH=$PATH" make multiarch-build-small && sudo chmod -R 777 release/
       - name: Release
         uses: fnkr/github-action-ghr@v1
         if: startsWith(github.ref, 'refs/tags/')

.github/workflows/test.yml

@@ -17,5 +17,7 @@ jobs:
       uses: actions/checkout@v2
     - name: setup-docker
       uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: sudo apt-get install -y upx
     - name: Tests
-      run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage
+      run: sudo -E env "PATH=$PATH" make deps multiarch-build-small test-integration test-coverage

.travis.yml

@@ -1,15 +1,31 @@
+dist: bionic
 language: go
-services:
-  - docker
 go:
   - "1.14"
+
 env:
-  - "GO15VENDOREXPERIMENT=1"
+  global:
+    - "GO15VENDOREXPERIMENT=1"
+  jobs:
+    - "DOCKER_BUILDKIT=0"
+    - "DOCKER_BUILDKIT=1"
 before_install:
+  - sudo rm -rf /var/lib/apt/lists/*
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) edge"
+  - sudo apt-get update
+  - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
+  - export DOCKER_CLI_EXPERIMENTAL=enabled
+  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  - mkdir -vp ~/.docker/cli-plugins/
+  - curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+  - chmod a+x ~/.docker/cli-plugins/docker-buildx
+  - docker buildx version
   - sudo -E env "PATH=$PATH" apt-get install -y libcap2-bin
   - sudo -E env "PATH=$PATH" make deps
 script:
   - sudo -E env "PATH=$PATH" make multiarch-build test-integration test-coverage
+
 #after_success:
 #  - |
 #      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then

Makefile

@@ -88,6 +88,11 @@ test-docker:
 				--workdir /go/src/github.com/mudler/luet -ti golang:latest \
 				bash -c "make test"
 
-.PHONY: multiarch-build
 multiarch-build:
 	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+
+multiarch-build-small:
+	@$(MAKE) LDFLAGS+="-s -w" multiarch-build
+	for file in $(ROOT_DIR)/release/* ; do \
+		upx --brute -1 $${file} ; \
+	done

README.md

@@ -1,13 +1,14 @@
 # luet - Container-based Package manager
+
 [![Docker Repository on Quay](https://quay.io/repository/luet/base/status "Docker Repository on Quay")](https://quay.io/repository/luet/base)
 [![Go Report Card](https://goreportcard.com/badge/github.com/mudler/luet)](https://goreportcard.com/report/github.com/mudler/luet)
 [![Build Status](https://travis-ci.org/mudler/luet.svg?branch=master)](https://travis-ci.org/mudler/luet)
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
 [![codecov](https://codecov.io/gh/mudler/luet/branch/master/graph/badge.svg)](https://codecov.io/gh/mudler/luet)
 
-Luet is a multi-platform Package Manager based off from containers - it uses Docker (and other tech) to sandbox your builds and generate packages from them. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
+Luet is a multi-platform Package Manager based off from containers - it uses Docker (and others) to build packages. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
 
-It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/specfile/) in YAML notation to define both packages and rootfs. As it is based on containers, it can be used to build seed stages for Linux From Scratch installations and it can build and track updates for those systems.
+It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/) in YAML notation to define both [packages](https://luet-lab.github.io/docs/docs/concepts/packages/) and [rootfs](https://luet-lab.github.io/docs/docs/concepts/packages/#package-layers). As it is based on containers, it can be also used to build stages for Linux From Scratch installations and it can build and track updates for those systems.
 
 It is written entirely in Golang and where used as package manager, it can run in from scratch environment, with zero dependencies.
 
@@ -16,25 +17,41 @@ It is written entirely in Golang and where used as package manager, it can run i
 - Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.
 - It builds, installs, uninstalls and perform upgrades on machines
 - Installer doesn't depend on anything ( 0 dep installer !), statically built
-- Support for packages as "layers"
-- It uses SAT solving techniques to solve the deptree ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
+- You can install it aside also with your current distro package manager, and start building and distributing your packages
+- [Support for packages as "layers"](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/#building-strategies)
+- [It uses SAT solving techniques to solve the deptree](https://luet-lab.github.io/docs/docs/concepts/overview/constraints/) ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
+- Support for [collections](https://luet-lab.github.io/docs/docs/concepts/packages/collections/) and [templated package definitions](https://luet-lab.github.io/docs/docs/concepts/packages/templates/)
+- [Can be extended with Plugins and Extensions](https://luet-lab.github.io/docs/docs/concepts/plugins-and-extensions/)
+- [Can build packages in Kubernetes (experimental)](https://github.com/mudler/luet-k8s)
 
 ## Install
 
-To install luet, you can grab a release on the [Release page](https://github.com/mudler/luet/releases) or compile it in your machine (requires Golang installed):
+To install luet, you can grab a release on the [Release page](https://github.com/mudler/luet/releases) or to install it in your system:
 
-    $ git clone https://github.com/mudler/luet.git
-    $ cd luet
-    $ make build
+```bash
+$ curl https://get.mocaccino.org/luet/get_luet_root.sh | sudo sh
+$ luet search ...
+$ luet install ..
+$ luet --help
+```
 
-## Status
+## Build from source
 
-Luet is not feature-complete yet, it can build, install/uninstall/upgrade packages - but it doesn't support yet all the features you would normally expect from a Package Manager nowadays.
+```bash
+$ git clone https://github.com/mudler/luet.git
+$ cd luet
+$ make build
+```
+
+## Documentation
+
+[Documentation](https://luet-lab.github.io/docs) is available, or
+run `luet --help`,  any subcommand is documented as well, try e.g.: `luet build --help`.
 
 # Dependency solving
 
 Luet uses SAT and Reinforcement learning engine for dependency solving.
-It encodes the package requirements into a SAT problem, using gophersat to solve the dependency tree and give a concrete model as result.
+It encodes the package requirements into a SAT problem, using [gophersat](https://github.com/crillab/gophersat) to solve the dependency tree and give a concrete model as result.
 
 ## SAT encoding
 
@@ -48,10 +65,6 @@ when they arises while trying to validate your queries against the system model.
 
 To leverage it, simply pass ```--solver-type qlearning``` to the subcommands that supports it ( you can check out by invoking ```--help``` ).
 
-## Documentation
-
-[Documentation](https://luet-lab.github.io/docs) is available, or
-run `luet --help`,  any subcommand is documented as well, try e.g.: `luet build --help`.
 
 ## Authors
 

cmd/build.go

@@ -15,15 +15,19 @@
 package cmd
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
+	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/spf13/cobra"
@@ -33,9 +37,30 @@ import (
 var buildCmd = &cobra.Command{
 	Use:   "build <package name> <package name> <package name> ...",
 	Short: "build a package or a tree",
-	Long:  `build packages or trees from luet tree definitions. Packages are in [category]/[name]-[version] form`,
-	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("clean", cmd.Flags().Lookup("clean"))
+	Long: `Builds one or more packages from a tree (current directory is implied):
+
+	$ luet build utils/busybox utils/yq ...
+	
+Builds all packages
+	
+	$ luet build --all
+	
+Builds only the leaf packages:
+	
+	$ luet build --full
+
+Build package revdeps:
+	
+	$ luet build --revdeps utils/yq
+
+Build package without dependencies (needs the images already in the host, or either need to be available online):
+
+	$ luet build --nodeps utils/yq ...
+
+Build packages specifying multiple definition trees:
+
+	$ luet build --tree overlay/path --tree overlay/path2 utils/yq ...
+`, PreRun: func(cmd *cobra.Command, args []string) {
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
 		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
@@ -46,10 +71,12 @@ var buildCmd = &cobra.Command{
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
 		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("values", cmd.Flags().Lookup("values"))
 
 		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
 		viper.BindPFlag("push", cmd.Flags().Lookup("push"))
 		viper.BindPFlag("pull", cmd.Flags().Lookup("pull"))
+		viper.BindPFlag("wait", cmd.Flags().Lookup("wait"))
 		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
 
 		LuetCfg.Viper.BindPFlag("keep-exported-images", cmd.Flags().Lookup("keep-exported-images"))
@@ -61,7 +88,6 @@ var buildCmd = &cobra.Command{
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
-		clean := viper.GetBool("clean")
 		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("destination")
 		concurrency := LuetCfg.GetGeneral().Concurrency
@@ -72,24 +98,28 @@ var buildCmd = &cobra.Command{
 		databaseType := viper.GetString("database")
 		compressionType := viper.GetString("compression")
 		imageRepository := viper.GetString("image-repository")
+		values := viper.GetString("values")
+		wait := viper.GetBool("wait")
 		push := viper.GetBool("push")
 		pull := viper.GetBool("pull")
 		keepImages := viper.GetBool("keep-images")
 		nodeps := viper.GetBool("nodeps")
 		onlydeps := viper.GetBool("onlydeps")
 		keepExportedImages := viper.GetBool("keep-exported-images")
+		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
 		full, _ := cmd.Flags().GetBool("full")
-		skip, _ := cmd.Flags().GetBool("skip-if-metadata-exists")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		var results Results
 
-		compilerSpecs := compiler.NewLuetCompilationspecs()
-		var compilerBackend compiler.CompilerBackend
-		var db pkg.PackageDatabase
-		switch backendType {
-		case "img":
-			compilerBackend = backend.NewSimpleImgBackend()
-		case "docker":
-			compilerBackend = backend.NewSimpleDockerBackend()
+		out, _ := cmd.Flags().GetString("output")
+		if out != "terminal" {
+			LuetCfg.GetLogging().SetLogLevel("error")
 		}
+		pretend, _ := cmd.Flags().GetBool("pretend")
+		compilerSpecs := compiler.NewLuetCompilationspecs()
+		var db pkg.PackageDatabase
+
+		compilerBackend := backend.NewBackend(backendType)
 
 		switch databaseType {
 		case "memory":
@@ -107,10 +137,6 @@ var buildCmd = &cobra.Command{
 
 		generalRecipe := tree.NewCompilerRecipe(db)
 
-		if len(treePaths) <= 0 {
-			Fatal("No tree path supplied!")
-		}
-
 		for _, src := range treePaths {
 			Info("Loading tree", src)
 
@@ -137,16 +163,23 @@ var buildCmd = &cobra.Command{
 		opts := compiler.NewDefaultCompilerOptions()
 		opts.SolverOptions = *LuetCfg.GetSolverOptions()
 		opts.ImageRepository = imageRepository
-		opts.Clean = clean
 		opts.PullFirst = pull
 		opts.KeepImg = keepImages
 		opts.Push = push
 		opts.OnlyDeps = onlydeps
 		opts.NoDeps = nodeps
+		opts.Wait = wait
 		opts.KeepImageExport = keepExportedImages
-		opts.SkipIfMetadataExists = skip
+		opts.PackageTargetOnly = onlyTarget
+		opts.BuildValuesFile = values
+		var solverOpts solver.Options
+		if concurrent {
+			solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
+		} else {
+			solverOpts = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+		}
 
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts)
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts, solverOpts)
 		luetCompiler.SetConcurrency(concurrency)
 		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
 		if full {
@@ -194,9 +227,58 @@ var buildCmd = &cobra.Command{
 		if revdeps {
 			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
+		} else if pretend {
+			toCalculate := []compiler.CompilationSpec{}
+			if full {
+				var err error
+				toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...)
+				if err != nil {
+					errs = append(errs, err)
+				}
+			} else {
+				toCalculate = compilerSpecs.All()
+			}
+
+			for _, sp := range toCalculate {
+				packs, err := luetCompiler.ComputeDepTree(sp)
+				if err != nil {
+					errs = append(errs, err)
+				}
+				for _, p := range packs {
+					results.Packages = append(results.Packages,
+						PackageResult{
+							Name:       p.Package.GetName(),
+							Version:    p.Package.GetVersion(),
+							Category:   p.Package.GetCategory(),
+							Repository: "",
+							Hidden:     p.Package.IsHidden(),
+							Target:     sp.GetPackage().HumanReadableString(),
+						})
+				}
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			case "terminal":
+				for _, p := range results.Packages {
+					Info(p.String())
+				}
+			}
 		} else {
 			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
-
 		}
 		if len(errs) != 0 {
 			for _, e := range errs {
@@ -215,30 +297,35 @@ func init() {
 	if err != nil {
 		Fatal(err)
 	}
-	buildCmd.Flags().Bool("clean", true, "Build all packages without considering the packages present in the build directory")
-	buildCmd.Flags().StringSliceP("tree", "t", []string{}, "Path of the tree to use.")
+	buildCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the tree to use.")
 	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
 	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
 	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
+	buildCmd.Flags().String("values", "", "Build values file to interpolate with each package")
 
-	buildCmd.Flags().String("destination", path, "Destination folder")
-	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
+	buildCmd.Flags().String("destination", filepath.Join(path, "build"), "Destination folder")
+	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip, zstd")
 	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
 	buildCmd.Flags().Bool("push", false, "Push images to a hub")
 	buildCmd.Flags().Bool("pull", false, "Pull images from a hub")
+	buildCmd.Flags().Bool("wait", false, "Don't build all intermediate images, but wait for them until they are available")
 	buildCmd.Flags().Bool("keep-images", true, "Keep built docker images in the host")
 	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
 	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
 	buildCmd.Flags().Bool("keep-exported-images", false, "Keep exported images used during building")
-	buildCmd.Flags().Bool("skip-if-metadata-exists", false, "Skip package if metadata exists")
-
+	buildCmd.Flags().Bool("only-target-package", false, "Build packages of only the required target. Otherwise builds all the necessary ones not present in the destination")
 	buildCmd.Flags().String("solver-type", "", "Solver strategy")
 	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+
+	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
+
+	buildCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 
 	RootCmd.AddCommand(buildCmd)
 }

cmd/convert.go

@@ -1,101 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-package cmd
-
-import (
-	"io/ioutil"
-
-	. "github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
-	pkg "github.com/mudler/luet/pkg/package"
-	tree "github.com/mudler/luet/pkg/tree"
-
-	"github.com/mudler/luet/pkg/tree/builder/gentoo"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-)
-
-var convertCmd = &cobra.Command{
-	Use:   "convert",
-	Short: "convert other package manager tree into luet",
-	Long:  `Parses external PM and produces a luet parsable tree`,
-	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
-		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
-	},
-	Run: func(cmd *cobra.Command, args []string) {
-
-		t := viper.GetString("type")
-		databaseType := viper.GetString("database")
-		var db pkg.PackageDatabase
-
-		if len(args) != 2 {
-			Fatal("Incorrect number of arguments")
-		}
-
-		input := args[0]
-		output := args[1]
-		Info("Converting trees from " + input + " [" + t + "]")
-
-		var builder tree.Parser
-		switch t {
-		case "gentoo":
-			builder = gentoo.NewGentooBuilder(
-				&gentoo.SimpleEbuildParser{},
-				LuetCfg.GetGeneral().Concurrency,
-				gentoo.InMemory)
-		default: // dup
-			builder = gentoo.NewGentooBuilder(
-				&gentoo.SimpleEbuildParser{},
-				LuetCfg.GetGeneral().Concurrency,
-				gentoo.InMemory)
-		}
-
-		switch databaseType {
-		case "memory":
-			db = pkg.NewInMemoryDatabase(false)
-		case "boltdb":
-			tmpdir, err := ioutil.TempDir("", "package")
-			if err != nil {
-				Fatal(err)
-			}
-			db = pkg.NewBoltDatabase(tmpdir)
-		}
-		defer db.Clean()
-
-		packageTree, err := builder.Generate(input)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-
-		defer packageTree.Clean()
-		Info("Tree generated")
-
-		generalRecipe := tree.NewGeneralRecipe(packageTree)
-		Info("Saving generated tree to " + output)
-
-		err = generalRecipe.Save(output)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-	},
-}
-
-func init() {
-	convertCmd.Flags().String("type", "gentoo", "source type")
-	convertCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
-
-	RootCmd.AddCommand(convertCmd)
-}

cmd/create-repo.go

@@ -16,8 +16,10 @@ package cmd
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -30,11 +32,31 @@ import (
 var createrepoCmd = &cobra.Command{
 	Use:   "create-repo",
 	Short: "Create a luet repository from a build",
-	Long:  `Generate and renew repository metadata`,
+	Long: `Builds tree metadata from a set of packages and a tree definition:
+
+	$ luet create-repo
+
+Provide specific paths for packages, tree, and metadata output which is generated:
+
+	$ luet create-repo --packages my/packages/path --tree my/tree/path --output my/packages/path ...
+
+Provide name and description of the repository:
+
+	$ luet create-repo --name "foo" --description "bar" ...
+
+Change compression method:
+	
+	$ luet create-repo --tree-compression gzip --meta-compression gzip
+
+Create a repository from the metadata description defined in the luet.yaml config file:
+
+	$ luet create-repo --repo repository1
+`,
 	PreRun: func(cmd *cobra.Command, args []string) {
 		viper.BindPFlag("packages", cmd.Flags().Lookup("packages"))
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("output", cmd.Flags().Lookup("output"))
+		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
 		viper.BindPFlag("name", cmd.Flags().Lookup("name"))
 		viper.BindPFlag("descr", cmd.Flags().Lookup("descr"))
 		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
@@ -45,6 +67,10 @@ var createrepoCmd = &cobra.Command{
 		viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename"))
 		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
 		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
+
+		viper.BindPFlag("force-push", cmd.Flags().Lookup("force-push"))
+		viper.BindPFlag("push-images", cmd.Flags().Lookup("push-images"))
+
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var err error
@@ -63,9 +89,13 @@ var createrepoCmd = &cobra.Command{
 		metatype := viper.GetString("meta-compression")
 		metaName := viper.GetString("meta-filename")
 		source_repo := viper.GetString("repo")
+		backendType := viper.GetString("backend")
 
 		treeFile := installer.NewDefaultTreeRepositoryFile()
 		metaFile := installer.NewDefaultMetaRepositoryFile()
+		compilerBackend := backend.NewBackend(backendType)
+		force := viper.GetBool("force-push")
+		imagePush := viper.GetBool("push-images")
 
 		if source_repo != "" {
 			// Search for system repository
@@ -88,11 +118,11 @@ var createrepoCmd = &cobra.Command{
 				lrepo.Priority,
 				packages,
 				treePaths,
-				pkg.NewInMemoryDatabase(false))
+				pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force)
 
 		} else {
 			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
-				treePaths, pkg.NewInMemoryDatabase(false))
+				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force)
 		}
 
 		if err != nil {
@@ -118,7 +148,7 @@ var createrepoCmd = &cobra.Command{
 		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
-		err = repo.Write(dst, reset)
+		err = repo.Write(dst, reset, true)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
@@ -130,19 +160,23 @@ func init() {
 	if err != nil {
 		Fatal(err)
 	}
-	createrepoCmd.Flags().String("packages", path, "Packages folder (output from build)")
-	createrepoCmd.Flags().StringSliceP("tree", "t", []string{}, "Path of the source trees to use.")
-	createrepoCmd.Flags().String("output", path, "Destination folder")
+	createrepoCmd.Flags().String("packages", filepath.Join(path, "build"), "Packages folder (output from build)")
+	createrepoCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the source trees to use.")
+	createrepoCmd.Flags().String("output", filepath.Join(path, "build"), "Destination for generated archives. With 'docker' repository type, it should be an image reference (e.g 'foo/bar')")
 	createrepoCmd.Flags().String("name", "luet", "Repository name")
 	createrepoCmd.Flags().String("descr", "luet", "Repository description")
 	createrepoCmd.Flags().StringSlice("urls", []string{}, "Repository URLs")
-	createrepoCmd.Flags().String("type", "disk", "Repository type (disk)")
+	createrepoCmd.Flags().String("type", "disk", "Repository type (disk, http, docker)")
 	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
 	createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
+	createrepoCmd.Flags().String("backend", "docker", "backend used (docker,img)")
 
-	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip")
+	createrepoCmd.Flags().Bool("force-push", false, "Force overwrite of docker images if already present online")
+	createrepoCmd.Flags().Bool("push-images", false, "Enable/Disable docker image push for docker repositories")
+
+	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
-	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
 
 	RootCmd.AddCommand(createrepoCmd)

cmd/database.go

@@ -0,0 +1,41 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/database"
+
+	"github.com/spf13/cobra"
+)
+
+var databaseGroupCmd = &cobra.Command{
+	Use:   "database [command] [OPTIONS]",
+	Short: "Manage system database (dangerous commands ahead!)",
+	Long: `Allows to manipulate Luet internal database of installed packages. Use with caution!
+
+Removing packages by hand from the database can result in a broken system, and thus it's not reccomended.
+`,
+}
+
+func init() {
+	RootCmd.AddCommand(databaseGroupCmd)
+
+	databaseGroupCmd.AddCommand(
+		NewDatabaseCreateCommand(),
+		NewDatabaseRemoveCommand(),
+	)
+}

cmd/database/create.go

@@ -0,0 +1,81 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"io/ioutil"
+
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseCreateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "create <artifact_metadata1.yaml> <artifact_metadata1.yaml>",
+		Short: "Insert a package in the system DB",
+		Long: `Inserts a package in the system database:
+
+		$ luet database create foo.yaml
+
+"luet database create" injects a package in the system database without actually installing it, use it with caution.
+
+This commands takes multiple yaml input file representing package artifacts, that are usually generated while building packages.
+
+The yaml must contain the package definition, and the file list at least.
+
+For reference, inspect a "metadata.yaml" file generated while running "luet build"`,
+		Args: cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			systemDB := LuetCfg.GetSystemDB()
+
+			for _, a := range args {
+				dat, err := ioutil.ReadFile(a)
+				if err != nil {
+					Fatal("Failed reading ", a, ": ", err.Error())
+				}
+				art, err := compiler.NewPackageArtifactFromYaml(dat)
+				if err != nil {
+					Fatal("Failed reading yaml ", a, ": ", err.Error())
+				}
+
+				files := art.GetFiles()
+
+				if _, err := systemDB.CreatePackage(art.GetCompileSpec().GetPackage()); err != nil {
+					Fatal("Failed to create ", a, ": ", err.Error())
+				}
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.GetCompileSpec().GetPackage().GetFingerPrint(), Files: files}); err != nil {
+					Fatal("Failed setting package files for ", a, ": ", err.Error())
+				}
+
+				Info(art.GetCompileSpec().GetPackage().HumanReadableString(), " created")
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/database/remove.go

@@ -0,0 +1,65 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	. "github.com/mudler/luet/pkg/logger"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseRemoveCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "remove [package1] [package2] ...",
+		Short: "Remove a package from the system DB (forcefully - you normally don't want to do that)",
+		Long: `Removes a package in the system database without actually uninstalling it:
+
+		$ luet database remove foo/bar
+
+This commands takes multiple packages as arguments and prunes their entries from the system database.
+`,
+		Args: cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			systemDB := LuetCfg.GetSystemDB()
+
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					Fatal("Invalid package string ", a, ": ", err.Error())
+				}
+
+				if err := systemDB.RemovePackage(pack); err != nil {
+					Fatal("Failed removing ", a, ": ", err.Error())
+				}
+
+				if err := systemDB.RemovePackageFiles(pack); err != nil {
+					Fatal("Failed removing files for ", a, ": ", err.Error())
+				}
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/helpers/cli.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+	"strings"
 
 	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -41,7 +42,41 @@ func CreateRegexArray(rgx []string) ([]*regexp.Regexp, error) {
 	return ans, nil
 }
 
+func packageData(p string) (string, string) {
+	cat := ""
+	name := ""
+	if strings.Contains(p, "/") {
+		packagedata := strings.Split(p, "/")
+		cat = packagedata[0]
+		name = packagedata[1]
+	} else {
+		name = p
+	}
+	return cat, name
+}
 func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
+
+	if !strings.HasPrefix(p, "=") {
+		ver := ">=0"
+		cat := ""
+		name := ""
+
+		if strings.Contains(p, "@") {
+			packageinfo := strings.Split(p, "@")
+			ver = packageinfo[1]
+			cat, name = packageData(packageinfo[0])
+		} else {
+			cat, name = packageData(p)
+		}
+
+		return &pkg.DefaultPackage{
+			Name:     name,
+			Category: cat,
+			Version:  ver,
+			Uri:      make([]string, 0),
+		}, nil
+	}
+
 	gp, err := _gentoo.ParsePackageStr(p)
 	if err != nil {
 		return nil, err

cmd/helpers/cli_suite_test.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package gentoo_test
+package cmd_helpers_test
 
 import (
 	"testing"
@@ -25,8 +25,8 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func TestGentooBuilder(t *testing.T) {
+func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
 	LoadConfig(config.LuetCfg)
-	RunSpecs(t, "Gentoo Suite")
+	RunSpecs(t, "CLI helpers test Suite")
 }

cmd/helpers/cli_test.go

@@ -0,0 +1,70 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_helpers_test
+
+import (
+	. "github.com/mudler/luet/cmd/helpers"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("CLI Helpers", func() {
+	Context("Can parse package strings correctly", func() {
+		It("accept single package names", func() {
+			pack, err := ParsePackageStr("foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal(""))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept unversioned packages with category", func() {
+			pack, err := ParsePackageStr("cat/foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept versioned packages with category", func() {
+			pack, err := ParsePackageStr("cat/foo@1.1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("1.1"))
+		})
+		It("accept versioned ranges with category", func() {
+			pack, err := ParsePackageStr("cat/foo@>=1.1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=1.1"))
+		})
+		It("accept gentoo regex parsing without versions", func() {
+			pack, err := ParsePackageStr("=cat/foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept gentoo regex parsing with versions", func() {
+			pack, err := ParsePackageStr("=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("1.2"))
+		})
+	})
+})

cmd/install.go

@@ -16,9 +16,9 @@ package cmd
 
 import (
 	"os"
-	"path/filepath"
 
 	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
@@ -29,8 +29,24 @@ import (
 )
 
 var installCmd = &cobra.Command{
-	Use:     "install <pkg1> <pkg2> ...",
-	Short:   "Install a package",
+	Use:   "install <pkg1> <pkg2> ...",
+	Short: "Install a package",
+	Long: `Installs one or more packages without asking questions:
+
+	$ luet install -y utils/busybox utils/yq ...
+	
+To install only deps of a package:
+	
+	$ luet install --onlydeps utils/busybox ...
+	
+To not install deps of a package:
+	
+	$ luet install --nodeps utils/busybox ...
+
+To force install a package:
+	
+	$ luet install --force utils/busybox ...
+`,
 	Aliases: []string{"i"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
@@ -42,11 +58,10 @@ var installCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
-	Long: `Install packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
 		var toInstall pkg.Packages
-		var systemDB pkg.PackageDatabase
 
 		for _, a := range args {
 			pack, err := helpers.ParsePackageStr(a)
@@ -73,11 +88,20 @@ var installCmd = &cobra.Command{
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps := LuetCfg.Viper.GetBool("nodeps")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
+
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
 		LuetCfg.GetSolverOptions().MaxAttempts = attempts
 
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
 		// Load config protect configs
@@ -90,16 +114,11 @@ var installCmd = &cobra.Command{
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
 			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
 		})
 		inst.Repositories(repos)
 
-		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-		} else {
-			systemDB = pkg.NewInMemoryDatabase(true)
-		}
-		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
 		err := inst.Install(toInstall, system)
 		if err != nil {
 			Fatal("Error: " + err.Error())
@@ -121,6 +140,8 @@ func init() {
 	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
 	installCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
 	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	installCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	installCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/pack.go

@@ -0,0 +1,98 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var packCmd = &cobra.Command{
+	Use:   "pack <package name>",
+	Short: "pack a custom package",
+	Long: `Pack creates a package from a directory, generating the metadata required from a tree to generate a repository.
+
+Pack can be used to manually replace what "luet build" does automatically by reading the packages build.yaml files.
+
+	$ mkdir -p output/etc/foo
+	$ echo "my config" > output/etc/foo
+	$ luet pack foo/bar@1.1 --source output
+
+Afterwards, you can use the content generated and associate it with a tree and a corresponding definition.yaml file with "luet create-repo".
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
+		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+		viper.BindPFlag("source", cmd.Flags().Lookup("source"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		sourcePath := viper.GetString("source")
+
+		dst := viper.GetString("destination")
+		compressionType := viper.GetString("compression")
+		concurrency := LuetCfg.GetGeneral().Concurrency
+
+		if len(args) != 1 {
+			Fatal("You must specify a package name")
+		}
+
+		packageName := args[0]
+
+		p, err := helpers.ParsePackageStr(packageName)
+		if err != nil {
+			Fatal("Invalid package string ", packageName, ": ", err.Error())
+		}
+
+		spec := &compiler.LuetCompilationSpec{Package: p}
+		artifact := compiler.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
+		artifact.SetCompressionType(compiler.CompressionImplementation(compressionType))
+		err = artifact.Compress(sourcePath, concurrency)
+		if err != nil {
+			Fatal("failed compressing ", packageName, ": ", err.Error())
+		}
+		artifact.SetCompileSpec(spec)
+		filelist, err := artifact.FileList()
+		if err != nil {
+			Fatal("failed generating file list for ", packageName, ": ", err.Error())
+		}
+		artifact.SetFiles(filelist)
+		artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
+		err = artifact.WriteYaml(dst)
+		if err != nil {
+			Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	packCmd.Flags().String("source", path, "Source folder")
+	packCmd.Flags().String("destination", path, "Destination folder")
+	packCmd.Flags().String("compression", "gzip", "Compression alg: none, gzip")
+
+	RootCmd.AddCommand(packCmd)
+}

cmd/reclaim.go

@@ -16,13 +16,11 @@ package cmd
 
 import (
 	"os"
-	"path/filepath"
 
 	installer "github.com/mudler/luet/pkg/installer"
 
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
-	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
 )
@@ -35,10 +33,13 @@ var reclaimCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
-	Long: `Add packages to the systemdb if files belonging to packages 
-in available repositories exists in the target root.`,
+	Long: `Reclaim tries to find association between packages in the online repositories and the system one.
+
+	$ luet reclaim
+
+It scans the target file system, and if finds a match with a package available in the repositories, it marks as installed in the system database.
+`,
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
 
 		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
 		repos := installer.Repositories{}
@@ -61,13 +62,7 @@ in available repositories exists in the target root.`,
 		})
 		inst.Repositories(repos)
 
-		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-		} else {
-			systemDB = pkg.NewInMemoryDatabase(true)
-		}
-		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
 		err := inst.Reclaim(system)
 		if err != nil {
 			Fatal("Error: " + err.Error())

cmd/replace.go

@@ -0,0 +1,148 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+var replaceCmd = &cobra.Command{
+	Use:     "replace <pkg1> <pkg2> --for <pkg3> --for <pkg4> ...",
+	Short:   "replace a set of packages",
+	Aliases: []string{"r"},
+	Long: `Replaces one or a group of packages without asking questions:
+
+	$ luet replace -y system/busybox ... --for shells/bash --for system/coreutils ...
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("for", cmd.Flags().Lookup("for"))
+
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var toUninstall pkg.Packages
+		var toAdd pkg.Packages
+
+		f := LuetCfg.Viper.GetStringSlice("for")
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps := LuetCfg.Viper.GetBool("nodeps")
+		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
+
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toUninstall = append(toUninstall, pack)
+		}
+
+		for _, a := range f {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toAdd = append(toAdd, pack)
+		}
+
+		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+		})
+		inst.Repositories(repos)
+
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Swap(toUninstall, toAdd, system)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	replaceCmd.Flags().String("system-dbpath", path, "System db path")
+	replaceCmd.Flags().String("system-target", path, "System rootpath")
+	replaceCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	replaceCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	replaceCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	replaceCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	replaceCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	replaceCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	replaceCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	replaceCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	replaceCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	replaceCmd.Flags().StringSlice("for", []string{}, "Packages that has to be installed in place of others")
+
+	RootCmd.AddCommand(replaceCmd)
+}

cmd/root.go

@@ -24,6 +24,8 @@ import (
 	"strings"
 
 	"github.com/marcsauter/single"
+	bus "github.com/mudler/luet/pkg/bus"
+
 	extensions "github.com/mudler/cobra-extensions"
 	config "github.com/mudler/luet/pkg/config"
 	helpers "github.com/mudler/luet/pkg/helpers"
@@ -38,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.8.6"
+	LuetCLIVersion = "0.10.1"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -52,9 +54,31 @@ var (
 
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
-	Use:     "luet",
-	Short:   "Package manager for the XXth century!",
-	Long:    `Package manager which uses containers to build packages`,
+	Use:   "luet",
+	Short: "Container based package manager",
+	Long: `Luet is a single-binary package manager based on containers to build packages.
+	
+To install a package:
+
+	$ luet install package
+
+To search for a package in the repositories:
+
+$ luet search package
+
+To list all packages installed in the system:
+
+	$ luet search --installed .
+
+To show hidden packages:
+
+	$ luet search --hidden package
+
+To build a package, from a tree definition:
+
+	$ luet build --tree tree/path package
+	
+`,
 	Version: fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime),
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		err := LoadConfig(config.LuetCfg)
@@ -68,6 +92,18 @@ var RootCmd = &cobra.Command{
 		if err != nil {
 			Fatal("failed on init tmp basedir:", err.Error())
 		}
+
+		viper.BindPFlag("plugin", cmd.Flags().Lookup("plugin"))
+
+		plugin := viper.GetStringSlice("plugin")
+
+		bus.Manager.Load(plugin...).Register()
+		if len(bus.Manager.Plugins) != 0 {
+			Info(":lollipop:Enabled plugins:")
+			for _, p := range bus.Manager.Plugins {
+				Info("\t:arrow_right:", p.Name)
+			}
+		}
 	},
 	PersistentPostRun: func(cmd *cobra.Command, args []string) {
 		// Cleanup all tmp directories used by luet
@@ -156,6 +192,7 @@ func init() {
 		"Disable config protect analysis.")
 	pflags.StringP("logfile", "l", config.LuetCfg.GetLogging().Path,
 		"Logfile path. Empty value disable log to file.")
+	pflags.StringSlice("plugin", []string{}, "A list of runtime plugins to load")
 
 	// os/user doesn't work in from scratch environments.
 	// Check if i can retrieve user informations.
@@ -175,6 +212,8 @@ func init() {
 	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("debug"))
 	config.LuetCfg.Viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
 	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
+	config.LuetCfg.Viper.BindPFlag("plugin", pflags.Lookup("plugin"))
+
 	// Currently I maintain this only from cli.
 	config.LuetCfg.Viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))
 	config.LuetCfg.Viper.BindPFlag("config_protect_skip", pflags.Lookup("skip-config-protect"))

cmd/search.go

@@ -17,9 +17,11 @@ package cmd
 import (
 	"fmt"
 	"os"
-	"path/filepath"
+	"strings"
 
 	"github.com/ghodss/yaml"
+	"github.com/jedib0t/go-pretty/table"
+	"github.com/jedib0t/go-pretty/v6/list"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -32,6 +34,7 @@ type PackageResult struct {
 	Category   string `json:"category"`
 	Version    string `json:"version"`
 	Repository string `json:"repository"`
+	Target     string `json:"target"`
 	Hidden     bool   `json:"hidden"`
 }
 
@@ -39,10 +42,64 @@ type Results struct {
 	Packages []PackageResult `json:"packages"`
 }
 
+func (r PackageResult) String() string {
+	return fmt.Sprintf("%s/%s-%s required for %s", r.Category, r.Name, r.Version, r.Target)
+}
+
+var rows table.Row = table.Row{"Package", "Category", "Name", "Version", "Repository", "Description", "License", "URI"}
+
+func packageToRow(repo string, p pkg.Package) table.Row {
+	return table.Row{p.HumanReadableString(), p.GetCategory(), p.GetName(), p.GetVersion(), repo, p.GetDescription(), p.GetLicense(), strings.Join(p.GetURI(), "\n")}
+}
+
+func packageToList(l list.Writer, repo string, p pkg.Package) {
+	l.AppendItem(p.HumanReadableString())
+	l.Indent()
+	l.AppendItem(fmt.Sprintf("Category: %s", p.GetCategory()))
+	l.AppendItem(fmt.Sprintf("Name: %s", p.GetName()))
+	l.AppendItem(fmt.Sprintf("Version: %s", p.GetVersion()))
+	l.AppendItem(fmt.Sprintf("Description: %s", p.GetDescription()))
+	l.AppendItem(fmt.Sprintf("Repository: %s ", repo))
+	l.AppendItem(fmt.Sprintf("Uri: %s ", strings.Join(p.GetURI(), "\n")))
+	l.UnIndent()
+}
+
 var searchCmd = &cobra.Command{
-	Use:     "search <term>",
-	Short:   "Search packages",
-	Long:    `Search for installed and available packages`,
+	Use:   "search <term>",
+	Short: "Search packages",
+	Long: `Search for installed and available packages
+	
+To search a package in the repositories:
+
+	$ luet search <regex>
+
+To search a package and display results in a table (wide screens):
+
+	$ luet search --table <regex>
+
+To look into the installed packages:
+
+	$ luet search --installed <regex>
+
+Note: the regex argument is optional, if omitted implies "all"
+
+To search a package by label:
+
+	$ luet search --by-label <label>
+
+or by regex against the label:
+
+	$ luet search --by-label-regex <label>
+
+It can also show a package revdeps by:
+
+	$ luet search --revdeps <regex>
+
+Search can also return results in the terminal in different ways: as terminal output, as json or as yaml.
+
+	$ luet search --json <regex> # JSON output
+	$ luet search --yaml <regex> # YAML output
+`,
 	Aliases: []string{"s"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
@@ -54,12 +111,12 @@ var searchCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
 		var results Results
-		if len(args) != 1 {
+		if len(args) > 1 {
 			Fatal("Wrong number of arguments (expected 1)")
+		} else if len(args) == 0 {
+			args = []string{"."}
 		}
-
 		hidden, _ := cmd.Flags().GetBool("hidden")
 
 		installed := LuetCfg.Viper.GetBool("installed")
@@ -70,6 +127,7 @@ var searchCmd = &cobra.Command{
 		searchWithLabel, _ := cmd.Flags().GetBool("by-label")
 		searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex")
 		revdeps, _ := cmd.Flags().GetBool("revdeps")
+		tableMode, _ := cmd.Flags().GetBool("table")
 
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
@@ -81,6 +139,9 @@ var searchCmd = &cobra.Command{
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
 		LuetCfg.GetSolverOptions().MaxAttempts = attempts
 
+		l := list.NewWriter()
+		t := table.NewWriter()
+		t.AppendHeader(rows)
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
 		if !installed {
@@ -119,7 +180,8 @@ var searchCmd = &cobra.Command{
 			for _, m := range matches {
 				if !revdeps {
 					if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
-						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
+						t.AppendRow(packageToRow(m.Repo.GetName(), m.Package))
+						packageToList(l, m.Repo.GetName(), m.Package)
 						results.Packages = append(results.Packages,
 							PackageResult{
 								Name:       m.Package.GetName(),
@@ -130,10 +192,11 @@ var searchCmd = &cobra.Command{
 							})
 					}
 				} else {
-					visited := make(map[string]interface{})
-					for _, revdep := range m.Package.ExpandedRevdeps(m.Repo.GetTree().GetDatabase(), visited) {
+					packs, _ := m.Repo.GetTree().GetDatabase().GetRevdeps(m.Package)
+					for _, revdep := range packs {
 						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
-							Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
+							t.AppendRow(packageToRow(m.Repo.GetName(), revdep))
+							packageToList(l, m.Repo.GetName(), revdep)
 							results.Packages = append(results.Packages,
 								PackageResult{
 									Name:       revdep.GetName(),
@@ -148,13 +211,7 @@ var searchCmd = &cobra.Command{
 			}
 		} else {
 
-			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-				systemDB = pkg.NewBoltDatabase(
-					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-			} else {
-				systemDB = pkg.NewInMemoryDatabase(true)
-			}
-			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+			system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
 
 			var err error
 			iMatches := pkg.Packages{}
@@ -173,7 +230,8 @@ var searchCmd = &cobra.Command{
 			for _, pack := range iMatches {
 				if !revdeps {
 					if !pack.IsHidden() || pack.IsHidden() && hidden {
-						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+						t.AppendRow(packageToRow("system", pack))
+						packageToList(l, "system", pack)
 						results.Packages = append(results.Packages,
 							PackageResult{
 								Name:       pack.GetName(),
@@ -184,11 +242,11 @@ var searchCmd = &cobra.Command{
 							})
 					}
 				} else {
-					visited := make(map[string]interface{})
-
-					for _, revdep := range pack.ExpandedRevdeps(system.Database, visited) {
+					packs, _ := system.Database.GetRevdeps(pack)
+					for _, revdep := range packs {
 						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
-							Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+							t.AppendRow(packageToRow("system", pack))
+							packageToList(l, "system", pack)
 							results.Packages = append(results.Packages,
 								PackageResult{
 									Name:       revdep.GetName(),
@@ -203,6 +261,16 @@ var searchCmd = &cobra.Command{
 			}
 		}
 
+		t.AppendFooter(rows)
+		t.SetStyle(table.StyleColoredBright)
+
+		l.SetStyle(list.StyleConnectedRounded)
+		if tableMode {
+			Info(t.Render())
+		} else {
+			Info(l.Render())
+		}
+
 		y, err := yaml.Marshal(results)
 		if err != nil {
 			fmt.Printf("err: %v\n", err)
@@ -240,6 +308,7 @@ func init() {
 	searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex")
 	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
 	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
+	searchCmd.Flags().Bool("table", false, "show output in a table (wider screens)")
 
 	RootCmd.AddCommand(searchCmd)
 }

cmd/tree.go

@@ -34,5 +34,6 @@ func init() {
 		NewTreePkglistCommand(),
 		NewTreeValidateCommand(),
 		NewTreeBumpCommand(),
+		NewTreeImageCommand(),
 	)
 }

cmd/tree/images.go

@@ -0,0 +1,140 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"fmt"
+	"os"
+
+	//. "github.com/mudler/luet/pkg/config"
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func NewTreeImageCommand() *cobra.Command {
+
+	var ans = &cobra.Command{
+		Use:   "images [OPTIONS]",
+		Short: "List of the images of a package",
+		PreRun: func(cmd *cobra.Command, args []string) {
+			t, _ := cmd.Flags().GetStringArray("tree")
+			if len(t) == 0 {
+				Fatal("Mandatory tree param missing.")
+			}
+
+			if len(args) != 1 {
+				Fatal("Expects one package as parameter")
+			}
+			viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var results TreeResults
+
+			treePath, _ := cmd.Flags().GetStringArray("tree")
+			imageRepository := viper.GetString("image-repository")
+
+			out, _ := cmd.Flags().GetString("output")
+			if out != "terminal" {
+				LuetCfg.GetLogging().SetLogLevel("error")
+			}
+
+			reciper := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			for _, t := range treePath {
+				err := reciper.Load(t)
+				if err != nil {
+					Fatal("Error on load tree ", err)
+				}
+			}
+			compilerBackend := backend.NewSimpleDockerBackend()
+
+			opts := compiler.NewDefaultCompilerOptions()
+			opts.SolverOptions = *LuetCfg.GetSolverOptions()
+			opts.ImageRepository = imageRepository
+
+			solverOpts := solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
+			luetCompiler := compiler.NewLuetCompiler(compilerBackend, reciper.GetDatabase(), opts, solverOpts)
+
+			a := args[0]
+
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+
+			spec, err := luetCompiler.FromPackage(pack)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+			asserts, err := luetCompiler.ComputeDepTree(spec)
+
+			for _, assertion := range asserts { //highly dependent on the order
+
+				//buildImageHash := imageRepository + ":" + assertion.Hash.BuildHash
+				currentPackageImageHash := imageRepository + ":" + assertion.Hash.PackageHash
+
+				results.Packages = append(results.Packages, TreePackageResult{
+					Name:     assertion.Package.GetName(),
+					Version:  assertion.Package.GetVersion(),
+					Category: assertion.Package.GetCategory(),
+					Image:    currentPackageImageHash,
+				})
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			default:
+				for _, p := range results.Packages {
+					fmt.Println(fmt.Sprintf("%s/%s-%s: %s", p.Category, p.Name, p.Version, p.Image))
+				}
+			}
+		},
+	}
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
+	ans.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+
+	return ans
+}

cmd/tree/pkglist.go

@@ -18,6 +18,7 @@ package cmd_tree
 
 import (
 	"fmt"
+	"os"
 	"sort"
 
 	//. "github.com/mudler/luet/pkg/config"
@@ -37,6 +38,7 @@ type TreePackageResult struct {
 	Category string `json:"category"`
 	Version  string `json:"version"`
 	Path     string `json:"path"`
+	Image    string `json:"image"`
 }
 
 type TreeResults struct {
@@ -116,7 +118,7 @@ func NewTreePkglistCommand() *cobra.Command {
 			if deps {
 				emptyInstallationDb := pkg.NewInMemoryDatabase(false)
 
-				depSolver = solver.NewSolver(pkg.NewInMemoryDatabase(false),
+				depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
 					reciper.GetDatabase(),
 					emptyInstallationDb)
 
@@ -167,9 +169,8 @@ func NewTreePkglistCommand() *cobra.Command {
 
 				if addPkg {
 					if revdeps {
-
-						visited := make(map[string]interface{})
-						for _, revdep := range p.ExpandedRevdeps(reciper.GetDatabase(), visited) {
+						packs, _ := reciper.GetDatabase().GetRevdeps(p)
+						for _, revdep := range packs {
 							if full {
 								pkgstr = pkgDetail(revdep)
 							} else if verbose {
@@ -266,7 +267,10 @@ func NewTreePkglistCommand() *cobra.Command {
 
 		},
 	}
-
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
 	ans.Flags().BoolP("buildtime", "b", false, "Build time match")
 	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 	ans.Flags().Bool("revdeps", false, "Search package reverse dependencies")
@@ -274,7 +278,7 @@ func NewTreePkglistCommand() *cobra.Command {
 
 	ans.Flags().BoolP("verbose", "v", false, "Add package version")
 	ans.Flags().BoolP("full", "f", false, "Show package detail")
-	ans.Flags().StringArrayP("tree", "t", []string{}, "Path of the tree to use.")
+	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
 	ans.Flags().StringSliceVarP(&matches, "matches", "m", []string{},
 		"Include only matched packages from list. (Use string as regex).")
 	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},

cmd/tree/validate.go

@@ -85,7 +85,7 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 
 	if opts.WithSolver {
 		emptyInstallationDb := pkg.NewInMemoryDatabase(false)
-		depSolver = solver.NewSolver(pkg.NewInMemoryDatabase(false),
+		depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
 			reciper.GetDatabase(),
 			emptyInstallationDb)
 	}
@@ -458,12 +458,15 @@ func NewTreeValidateCommand() *cobra.Command {
 			}
 		},
 	}
-
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
 	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
 	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")
 	ans.Flags().BoolP("with-solver", "s", false,
 		"Enable check of requires also with solver.")
-	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{},
+	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{path},
 		"Path of the tree to use.")
 	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},
 		"Exclude matched packages from analysis. (Use string as regex).")

cmd/uninstall.go

@@ -16,13 +16,13 @@ package cmd
 
 import (
 	"os"
-	"path/filepath"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/spf13/cobra"
 )
@@ -41,55 +41,61 @@ var uninstallCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
-
+		toRemove := []pkg.Package{}
 		for _, a := range args {
 
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
 				Fatal("Invalid package string ", a, ": ", err.Error())
 			}
+			toRemove = append(toRemove, pack)
+		}
 
-			stype := LuetCfg.Viper.GetString("solver.type")
-			discount := LuetCfg.Viper.GetFloat64("solver.discount")
-			rate := LuetCfg.Viper.GetFloat64("solver.rate")
-			attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
-			force := LuetCfg.Viper.GetBool("force")
-			nodeps, _ := cmd.Flags().GetBool("nodeps")
-			full, _ := cmd.Flags().GetBool("full")
-			checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
-			fullClean, _ := cmd.Flags().GetBool("full-clean")
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps, _ := cmd.Flags().GetBool("nodeps")
+		full, _ := cmd.Flags().GetBool("full")
+		checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
+		fullClean, _ := cmd.Flags().GetBool("full-clean")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
 
-			LuetCfg.GetSolverOptions().Type = stype
-			LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-			LuetCfg.GetSolverOptions().Discount = float32(discount)
-			LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
-			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
 
-			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-				Concurrency:        LuetCfg.GetGeneral().Concurrency,
-				SolverOptions:      *LuetCfg.GetSolverOptions(),
-				NoDeps:             nodeps,
-				Force:              force,
-				FullUninstall:      full,
-				FullCleanUninstall: fullClean,
-				CheckConflicts:     checkconflicts,
-			})
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			FullUninstall:               full,
+			FullCleanUninstall:          fullClean,
+			CheckConflicts:              checkconflicts,
+			Ask:                         !yes,
+			PreserveSystemEssentialData: true,
+		})
 
-			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-				systemDB = pkg.NewBoltDatabase(
-					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-			} else {
-				systemDB = pkg.NewInMemoryDatabase(true)
-			}
-			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
-			err = inst.Uninstall(pack, system)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+
+		if err := inst.Uninstall(system, toRemove...); err != nil {
+			Fatal("Error: " + err.Error())
 		}
 	},
 }
@@ -110,6 +116,8 @@ func init() {
 	uninstallCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
 	uninstallCmd.Flags().Bool("conflictscheck", true, "Check if the package marked for deletion is required by other packages")
 	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
+	uninstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	uninstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -16,12 +16,11 @@ package cmd
 
 import (
 	"os"
-	"path/filepath"
 
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
-	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/spf13/cobra"
 )
@@ -38,10 +37,10 @@ var upgradeCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
 	Long: `Upgrades packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
 
 		repos := installer.Repositories{}
 		for _, repo := range LuetCfg.SystemRepositories {
@@ -63,39 +62,40 @@ var upgradeCmd = &cobra.Command{
 		universe, _ := cmd.Flags().GetBool("universe")
 		clean, _ := cmd.Flags().GetBool("clean")
 		sync, _ := cmd.Flags().GetBool("sync")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
 
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
 		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
 
 		Debug("Solver", LuetCfg.GetSolverOptions().String())
 
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                LuetCfg.GetGeneral().Concurrency,
-			SolverOptions:              *LuetCfg.GetSolverOptions(),
-			Force:                      force,
-			FullUninstall:              full,
-			NoDeps:                     nodeps,
-			SolverUpgrade:              universe,
-			RemoveUnavailableOnUpgrade: clean,
-			UpgradeNewRevisions:        sync,
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			Force:                       force,
+			FullUninstall:               full,
+			NoDeps:                      nodeps,
+			SolverUpgrade:               universe,
+			RemoveUnavailableOnUpgrade:  clean,
+			UpgradeNewRevisions:         sync,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
 		})
 		inst.Repositories(repos)
-		_, err := inst.SyncRepositories(false)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
 
-		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-		} else {
-			systemDB = pkg.NewInMemoryDatabase(true)
-		}
-		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
-		err = inst.Upgrade(system)
-		if err != nil {
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		if err := inst.Upgrade(system); err != nil {
 			Fatal("Error: " + err.Error())
 		}
 	},
@@ -114,10 +114,12 @@ func init() {
 	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
 	upgradeCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
-	upgradeCmd.Flags().Bool("full", true, "Attempts to remove as much packages as possible which aren't required (slow)")
+	upgradeCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
 	upgradeCmd.Flags().Bool("universe", false, "Use ONLY the SAT solver to compute upgrades (experimental)")
 	upgradeCmd.Flags().Bool("clean", false, "Try to drop removed packages (experimental, only when --universe is enabled)")
 	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
+	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	upgradeCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

contrib/config/get_luet_root.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+if [ $(id -u) -ne 0 ]
+  then echo "Please run the installer with sudo/as root"
+  exit
+fi
+
+set -ex
+export LUET_NOLOCK=true
+
+LUET_VERSION=$(curl -s https://api.github.com/repos/mudler/luet/releases/latest | ( grep -oP '"tag_name": "\K(.*)(?=")' || echo "0.9.24" ))
+LUET_ROOTFS=${LUET_ROOTFS:-/}
+LUET_DATABASE_PATH=${LUET_DATABASE_PATH:-/var/luet/db}
+LUET_DATABASE_ENGINE=${LUET_DATABASE_ENGINE:-boltdb}
+LUET_CONFIG_PROTECT=${LUET_CONFIG_PROTECT:-1}
+
+curl -L https://github.com/mudler/luet/releases/download/${LUET_VERSION}/luet-${LUET_VERSION}-linux-amd64 --output luet
+chmod +x luet
+
+mkdir -p /etc/luet/repos.conf.d || true
+mkdir -p $LUET_DATABASE_PATH || true
+mkdir -p /var/tmp/luet || true
+
+if [ "${LUET_CONFIG_PROTECT}" = "1" ] ; then
+  mkdir -p /etc/luet/config.protect.d || true
+  curl -L https://raw.githubusercontent.com/mudler/luet/master/contrib/config/config.protect.d/01_etc.yml.example --output /etc/luet/config.protect.d/01_etc.yml
+fi
+curl -L https://raw.githubusercontent.com/mocaccinoOS/repository-index/master/packages/mocaccino-repository-index.yml --output /etc/luet/repos.conf.d/mocaccino-repository-index.yml
+
+cat > /etc/luet/luet.yaml <<EOF
+general:
+  debug: false
+system:
+  rootfs: ${LUET_ROOTFS}
+  database_path: "${LUET_DATABASE_PATH}"
+  database_engine: "${LUET_DATABASE_ENGINE}"
+  tmpdir_base: "/var/tmp/luet"
+EOF
+
+./luet install -y repository/luet repository/mocaccino-repository-index
+./luet install -y system/luet system/luet-extensions
+
+rm -rf luet
+

contrib/config/luet.yaml

@@ -69,6 +69,7 @@
 #   Default $TMPDIR/tmpluet
 #   tmpdir_base: "/tmp/tmpluet"
 #
+#
 # ---------------------------------------------
 # Repositories configurations directories.
 # ---------------------------------------------
@@ -93,6 +94,11 @@
 # annotation.
 # config_protect_skip: false
 #
+# The paths used for load repositories and config
+# protects are based on host rootfs.
+# If set to false rootfs path is used as prefix.
+# config_from_host: true
+#
 # System repositories
 # ---------------------------------------------
 # In alternative to define repositories files

go.mod

@@ -1,49 +1,65 @@
 module github.com/mudler/luet
 
-go 1.12
+go 1.14
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/Sabayon/pkgs-checker v0.6.3-0.20200912135508-97c41780e9b6
+	github.com/Sabayon/pkgs-checker v0.7.2
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/briandowns/spinner v1.7.0
-	github.com/cavaliercoder/grab v2.0.0+incompatible
-	github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3
-	github.com/docker/docker v17.12.0-ce-rc1.0.20200417035958-130b0bc6032c+incompatible
-	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
+	github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec
+	github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc
+	github.com/crillab/gophersat v1.3.2-0.20201023142334-3fc2ac466765
+	github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
+	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
+	github.com/fsouza/go-dockerclient v1.6.4
+	github.com/genuinetools/img v0.5.11
 	github.com/ghodss/yaml v1.0.0
+	github.com/google/go-containerregistry v0.2.1
+	github.com/hashicorp/go-multierror v1.0.0
 	github.com/hashicorp/go-version v1.2.0
+	github.com/jedib0t/go-pretty v4.3.0+incompatible
+	github.com/jedib0t/go-pretty/v6 v6.0.5
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
+	github.com/klauspost/compress v1.8.3
 	github.com/klauspost/pgzip v1.2.1
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
-	github.com/kr/pretty v0.2.0 // indirect
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
-	github.com/mattn/go-isatty v0.0.10 // indirect
-	github.com/moby/sys/mount v0.1.1-0.20200320164225-6154f11e6840 // indirect
+	github.com/moby/buildkit v0.7.2
+	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/onsi/ginkgo v1.12.1
-	github.com/onsi/gomega v1.10.0
-	github.com/otiai10/copy v1.0.2
-	github.com/pelletier/go-toml v1.6.0 // indirect
+	github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
+	github.com/onsi/ginkgo v1.14.2
+	github.com/onsi/gomega v1.10.3
+	github.com/opencontainers/image-spec v1.0.1
+	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
 	github.com/pkg/errors v0.9.1
+	github.com/schollz/progressbar/v3 v3.7.1
+	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
-	github.com/spf13/viper v1.6.3
-	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
-	go.etcd.io/bbolt v1.3.4
+	github.com/spf13/viper v1.7.0
+	go.etcd.io/bbolt v1.3.5
 	go.uber.org/atomic v1.5.1 // indirect
 	go.uber.org/multierr v1.4.0 // indirect
 	go.uber.org/zap v1.13.0
-	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
-	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
-	gopkg.in/yaml.v2 v2.2.8
-	gotest.tools/v3 v3.0.2 // indirect
+	gopkg.in/yaml.v2 v2.3.0
 	helm.sh/helm/v3 v3.3.4
-	mvdan.cc/sh/v3 v3.0.0-beta1
+
 )
 
 replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200605210607-749178b8f80d+incompatible
+
+replace github.com/containerd/containerd => github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55
+
+replace github.com/hashicorp/go-immutable-radix => github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe
+
+replace github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305
+
+replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc9.0.20200221051241-688cf6d43cc4

pkg/bus/events.go

@@ -0,0 +1,67 @@
+package bus
+
+import (
+	"github.com/mudler/go-pluggable"
+)
+
+var (
+	// Package events
+
+	// EventPackageInstall is the event fired when a new package is being installed
+	EventPackageInstall pluggable.EventType = "package.install"
+	// EventPackageUnInstall is the event fired when a new package is being uninstalled
+	EventPackageUnInstall pluggable.EventType = "package.uninstall"
+
+	// Package build
+
+	// EventPackagePreBuild is the event fired before a package is being built
+	EventPackagePreBuild pluggable.EventType = "package.pre.build"
+	// EventPackagePreBuildArtifact is the event fired before a package tarball is being generated
+	EventPackagePreBuildArtifact pluggable.EventType = "package.pre.build_artifact"
+	// EventPackagePostBuildArtifact is the event fired after a package tarball is generated
+	EventPackagePostBuildArtifact pluggable.EventType = "package.post.build_artifact"
+	// EventPackagePostBuild is the event fired after a package was built
+	EventPackagePostBuild pluggable.EventType = "package.post.build"
+
+	// Image build
+
+	// EventImagePreBuild is the event fired before a image is being built
+	EventImagePreBuild pluggable.EventType = "image.pre.build"
+	// EventImagePrePull is the event fired before a image is being pulled
+	EventImagePrePull pluggable.EventType = "image.pre.pull"
+	// EventImagePrePush is the event fired before a image is being pushed
+	EventImagePrePush pluggable.EventType = "image.pre.push"
+	// EventImagePostBuild is the event fired after an image is being built
+	EventImagePostBuild pluggable.EventType = "image.post.build"
+	// EventImagePostPull is the event fired after an image is being pulled
+	EventImagePostPull pluggable.EventType = "image.post.pull"
+	// EventImagePostPush is the event fired after an image is being pushed
+	EventImagePostPush pluggable.EventType = "image.post.push"
+
+	// Repository events
+
+	// EventRepositoryPreBuild is the event fired before a repository is being built
+	EventRepositoryPreBuild pluggable.EventType = "repository.pre.build"
+	// EventRepositoryPostBuild is the event fired after a repository was built
+	EventRepositoryPostBuild pluggable.EventType = "repository.post.build"
+)
+
+// Manager is the bus instance manager, which subscribes plugins to events emitted by Luet
+var Manager *pluggable.Manager = pluggable.NewManager(
+	[]pluggable.EventType{
+		EventPackageInstall,
+		EventPackageUnInstall,
+		EventPackagePreBuild,
+		EventPackagePreBuildArtifact,
+		EventPackagePostBuildArtifact,
+		EventPackagePostBuild,
+		EventRepositoryPreBuild,
+		EventRepositoryPostBuild,
+		EventImagePreBuild,
+		EventImagePrePull,
+		EventImagePrePush,
+		EventImagePostBuild,
+		EventImagePostPull,
+		EventImagePostPush,
+	},
+)

pkg/compiler/artifact.go

@@ -28,12 +28,14 @@ import (
 	"regexp"
 
 	system "github.com/docker/docker/pkg/system"
+	zstd "github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
 	"strings"
 	"sync"
 
+	bus "github.com/mudler/luet/pkg/bus"
 	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -46,8 +48,9 @@ import (
 type CompressionImplementation string
 
 const (
-	None CompressionImplementation = "none" // e.g. tar for standard packages
-	GZip CompressionImplementation = "gzip"
+	None      CompressionImplementation = "none" // e.g. tar for standard packages
+	GZip      CompressionImplementation = "gzip"
+	Zstandard CompressionImplementation = "zstd"
 )
 
 type ArtifactIndex []Artifact
@@ -141,14 +144,14 @@ func (a *PackageArtifact) Hash() error {
 
 func (a *PackageArtifact) Verify() error {
 	sum := Checksums{}
-	err := sum.Generate(a)
-	if err != nil {
+	if err := sum.Generate(a); err != nil {
 		return err
 	}
-	err = sum.Compare(a.Checksums)
-	if err != nil {
+
+	if err := sum.Compare(a.Checksums); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -170,6 +173,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
 	}
 
+	bus.Manager.Publish(bus.EventPackagePreBuildArtifact, a)
+
 	mangle, err := NewPackageArtifactFromYaml(data)
 	if err != nil {
 		return errors.Wrap(err, "Generated invalid artifact")
@@ -191,6 +196,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 		return errors.Wrap(err, "While writing PackageArtifact YAML")
 	}
 	//a.CompileSpec.GetPackage().SetPath(p)
+	bus.Manager.Publish(bus.EventPackagePostBuildArtifact, a)
 
 	return nil
 }
@@ -231,13 +237,121 @@ func (a *PackageArtifact) GetPath() string {
 	return a.Path
 }
 
+func (a *PackageArtifact) GetFileName() string {
+	return path.Base(a.GetPath())
+}
+
 func (a *PackageArtifact) SetPath(p string) {
 	a.Path = p
 }
 
-// Compress Archives and compress (TODO) to the artifact path
+func (a *PackageArtifact) genDockerfile() string {
+	return `
+FROM scratch
+COPY * /`
+}
+
+// CreateArtifactForFile creates a new artifact from the given file
+func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
+
+	fileName := path.Base(s)
+	archive, err := LuetCfg.GetSystem().TempDir("archive")
+	if err != nil {
+		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
+	}
+	defer os.RemoveAll(archive) // clean up
+	helpers.CopyFile(s, filepath.Join(archive, fileName))
+	artifact, err := LuetCfg.GetSystem().TempDir("artifact")
+	if err != nil {
+		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
+	}
+	a := &PackageArtifact{Path: filepath.Join(artifact, fileName)}
+
+	for _, o := range opts {
+		o(a)
+	}
+
+	return a, a.Compress(archive, 1)
+}
+
+// GenerateFinalImage takes an artifact and builds a Docker image with its content
+func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend, keepPerms bool) (CompilerBackendOptions, error) {
+	builderOpts := CompilerBackendOptions{}
+	archive, err := LuetCfg.GetSystem().TempDir("archive")
+	if err != nil {
+		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	}
+	defer os.RemoveAll(archive) // clean up
+
+	uncompressedFiles := filepath.Join(archive, "files")
+	dockerFile := filepath.Join(archive, "Dockerfile")
+
+	if err := os.MkdirAll(uncompressedFiles, os.ModePerm); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	}
+
+	data := a.genDockerfile()
+	if err := ioutil.WriteFile(dockerFile, []byte(data), 0644); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path)
+	}
+
+	if err := a.Unpack(uncompressedFiles, keepPerms); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
+	}
+
+	builderOpts = CompilerBackendOptions{
+		ImageName:      imageName,
+		SourcePath:     archive,
+		DockerFileName: dockerFile,
+		Context:        uncompressedFiles,
+	}
+	return builderOpts, b.BuildImage(builderOpts)
+}
+
+// Compress is responsible to archive and compress to the artifact Path.
+// It accepts a source path, which is the content to be archived/compressed
+// and a concurrency parameter.
 func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	switch a.CompressionType {
+
+	case Zstandard:
+		err := helpers.Tar(src, a.Path)
+		if err != nil {
+			return err
+		}
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return err
+		}
+		defer original.Close()
+
+		zstdFile := a.getCompressedName()
+		bufferedReader := bufio.NewReader(original)
+
+		// Open a file for writing.
+		dst, err := os.Create(zstdFile)
+		if err != nil {
+			return err
+		}
+
+		enc, err := zstd.NewWriter(dst)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(enc, bufferedReader)
+		if err != nil {
+			enc.Close()
+			return err
+		}
+		if err := enc.Close(); err != nil {
+			return err
+		}
+
+		os.RemoveAll(a.Path) // Remove original
+		Debug("Removed artifact", a.Path)
+
+		a.Path = zstdFile
+		return nil
 	case GZip:
 		err := helpers.Tar(src, a.Path)
 		if err != nil {
@@ -249,7 +363,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 		}
 		defer original.Close()
 
-		gzipfile := a.Path + ".gz"
+		gzipfile := a.getCompressedName()
 		bufferedReader := bufio.NewReader(original)
 
 		// Open a file for writing.
@@ -268,6 +382,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 		}
 		w.Close()
 		os.RemoveAll(a.Path) // Remove original
+		Debug("Removed artifact", a.Path)
 		//	a.CompressedPath = gzipfile
 		a.Path = gzipfile
 		return nil
@@ -275,12 +390,31 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.Tar(src, a.Path)
-
+		return helpers.Tar(src, a.getCompressedName())
 	}
 	return errors.New("Compression type must be supplied")
 }
 
+func (a *PackageArtifact) getCompressedName() string {
+	switch a.CompressionType {
+	case Zstandard:
+		return a.Path + ".zst"
+
+	case GZip:
+		return a.Path + ".gz"
+	}
+	return a.Path
+}
+
+// GetUncompressedName returns the artifact path without the extension suffix
+func (a *PackageArtifact) GetUncompressedName() string {
+	switch a.CompressionType {
+	case Zstandard, GZip:
+		return strings.TrimSuffix(a.Path, filepath.Ext(a.Path))
+	}
+	return a.Path
+}
+
 func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
 	// If the destination path already exists I rename target file name with postfix.
 	var destPath string
@@ -330,35 +464,25 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 
 func (a *PackageArtifact) GetProtectFiles() []string {
 	ans := []string{}
+	annotationDir := ""
 
-	if !LuetCfg.ConfigProtectSkip &&
-		LuetCfg.GetConfigProtectConfFiles() != nil &&
-		len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+	if !LuetCfg.ConfigProtectSkip {
 
-		for _, file := range a.Files {
-			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
-				for _, dir := range conf.Directories {
-					// Note file is without / at begin.
-					if strings.HasPrefix("/"+file, filepath.Clean(dir)) {
-						// docker archive modifier works with path without / at begin.
-						ans = append(ans, file)
-						goto nextFile
-					}
-				}
+		// a.CompileSpec could be nil when artifact.Unpack is used for tree tarball
+		if a.CompileSpec != nil &&
+			a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
 			}
-
-			if a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
-				dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
-				if ok {
-					if strings.HasPrefix("/"+file, filepath.Clean(dir)) {
-						ans = append(ans, file)
-						goto nextFile
-					}
-				}
-			}
-
-		nextFile:
 		}
+		// TODO: check if skip this if we have a.CompileSpec nil
+
+		cp := NewConfigProtect(annotationDir)
+		cp.Map(a.Files)
+
+		// NOTE: for unpack we need files path without initial /
+		ans = cp.GetProtectFiles(false)
 	}
 
 	return ans
@@ -373,6 +497,40 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
 
 	switch a.CompressionType {
+	case Zstandard:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+
+		d, err := zstd.NewReader(bufferedReader)
+		if err != nil {
+			return err
+		}
+		defer d.Close()
+
+		_, err = io.Copy(archive, d)
+		if err != nil {
+			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
+		}
+
+		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
+		if err != nil {
+			return err
+		}
+		return nil
 	case GZip:
 		// Create the uncompressed archive
 		archive, err := os.Create(a.GetPath() + ".uncompressed")
@@ -418,6 +576,27 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 func (a *PackageArtifact) FileList() ([]string, error) {
 	var tr *tar.Reader
 	switch a.CompressionType {
+	case Zstandard:
+		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		if err != nil {
+			return []string{}, err
+		}
+		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := zstd.NewReader(bufferedReader)
+		if err != nil {
+			return []string{}, err
+		}
+		defer r.Close()
+		tr = tar.NewReader(r)
 	case GZip:
 		// Create the uncompressed archive
 		archive, err := os.Create(a.GetPath() + ".uncompressed")
@@ -526,7 +705,7 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 }
 
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, t CompressionImplementation) (Artifact, error) {
+func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t CompressionImplementation) (Artifact, error) {
 
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
@@ -556,7 +735,8 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 	}
 
 	// Handle includes in spec. If specified they filter what gets in the package
-	if len(includes) > 0 {
+
+	if len(includes) > 0 && len(excludes) == 0 {
 		var includeRegexp []*regexp.Regexp
 		for _, i := range includes {
 			r, e := regexp.Compile(i)
@@ -584,6 +764,81 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 				Debug("File ", a.Name, " deleted")
 			}
 		}
+
+	} else if len(includes) == 0 && len(excludes) != 0 {
+		var excludeRegexp []*regexp.Regexp
+		for _, i := range excludes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			excludeRegexp = append(excludeRegexp, r)
+		}
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		ADD:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range excludeRegexp {
+					if i.MatchString(a.Name) {
+						continue ADD
+					}
+				}
+				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
+		}
+
+	} else if len(includes) != 0 && len(excludes) != 0 {
+
+		var includeRegexp []*regexp.Regexp
+		for _, i := range includes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			includeRegexp = append(includeRegexp, r)
+		}
+		var excludeRegexp []*regexp.Regexp
+		for _, i := range excludes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			excludeRegexp = append(excludeRegexp, r)
+		}
+
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		EXCLUDES:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) {
+						for _, e := range excludeRegexp {
+							if e.MatchString(a.Name) {
+								continue EXCLUDES
+							}
+						}
+						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+						continue EXCLUDES
+					}
+				}
+			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
+		}
+
 	} else {
 		// Otherwise just grab all
 		for _, l := range layers {

pkg/compiler/artifact_test.go

@@ -20,7 +20,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/solver"
 
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
@@ -41,7 +43,7 @@ var _ = Describe("Artifact", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -95,37 +97,46 @@ ENV PACKAGE_CATEGORY=app-admin`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
-			opts = CompilerBackendOptions{
+			opts2 := CompilerBackendOptions{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
 				Destination:    filepath.Join(tmpdir, "output2.tar"),
 			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
+			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
-			diffs, err := b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))
+			diffs, err := b.Changes(opts, opts2)
 			Expect(err).ToNot(HaveOccurred())
 
+			artifacts := []ArtifactNode{{
+				Name: "/luetbuild/LuetDockerfile",
+				Size: 175,
+			}}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
 			Expect(diffs).To(Equal(
 				[]ArtifactLayer{{
 					FromImage: filepath.Join(tmpdir2, "output1.tar"),
 					ToImage:   filepath.Join(tmpdir, "output2.tar"),
 					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+						Additions: artifacts,
 					},
 				}}))
 			err = b.ExtractRootfs(CompilerBackendOptions{SourcePath: filepath.Join(tmpdir, "output2.tar"), Destination: rootfs}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, None)
+			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, None)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
 			err = helpers.Untar(artifact.GetPath(), unpacked, false)
@@ -149,5 +160,59 @@ RUN echo bar > /test2`))
 			Expect(err).To(HaveOccurred())
 		})
 
+		It("Generates packages images", func() {
+			b := NewSimpleDockerBackend()
+			imageprefix := "foo/"
+			testString := []byte(`funky test data`)
+
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "artifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpWork, err := ioutil.TempDir(os.TempDir(), "artifact2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpWork) // clean up
+
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "test"), testString, 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			artifact := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			artifact.SetCompileSpec(&LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}})
+
+			err = artifact.Compress(tmpdir, 1)
+			Expect(err).ToNot(HaveOccurred())
+			resultingImage := imageprefix + "foo--1.0"
+			opts, err := artifact.GenerateFinalImage(resultingImage, b, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(opts.ImageName).To(Equal(resultingImage))
+
+			Expect(b.ImageExists(resultingImage)).To(BeTrue())
+
+			result, err := ioutil.TempDir(os.TempDir(), "result")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(result) // clean up
+
+			err = b.ExtractRootfs(CompilerBackendOptions{ImageName: resultingImage, Destination: result}, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			content, err := ioutil.ReadFile(filepath.Join(result, "test"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(content).To(Equal(testString))
+		})
+
+		It("Retrieves uncompressed name", func() {
+			a := NewPackageArtifact("foo.tar.gz")
+			a.SetCompressionType(compiler.GZip)
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+
+			a = NewPackageArtifact("foo.tar.zst")
+			a.SetCompressionType(compiler.Zstandard)
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+
+			a = NewPackageArtifact("foo.tar")
+			a.SetCompressionType(compiler.None)
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+		})
 	})
 })

pkg/compiler/backend/common.go

@@ -0,0 +1,43 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/mudler/luet/pkg/compiler"
+)
+
+const (
+	ImgBackend    = "img"
+	DockerBackend = "docker"
+)
+
+func imageAvailable(image string) bool {
+	_, err := crane.Digest(image)
+	return err == nil
+}
+
+func NewBackend(s string) compiler.CompilerBackend {
+	var compilerBackend compiler.CompilerBackend
+
+	switch s {
+	case ImgBackend:
+		compilerBackend = NewSimpleImgBackend()
+	case DockerBackend:
+		compilerBackend = NewSimpleDockerBackend()
+	}
+	return compilerBackend
+}

pkg/compiler/backend/diff.go

@@ -52,7 +52,10 @@ import (
 //     }
 //   }
 // ]
-func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]compiler.ArtifactLayer, error) {
+func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
+
+	srcImage := fromImage.Destination
+	dstImage := toImage.Destination
 
 	res := compiler.ArtifactLayer{FromImage: srcImage, ToImage: dstImage}
 
@@ -95,6 +98,7 @@ func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]c
 
 	srcImageExtract := compiler.CompilerBackendOptions{
 		SourcePath:  srcImage,
+		ImageName:   fromImage.ImageName,
 		Destination: srcRootFS,
 	}
 	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
@@ -123,6 +127,7 @@ func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]c
 
 	dstImageExtract := compiler.CompilerBackendOptions{
 		SourcePath:  dstImage,
+		ImageName:   toImage.ImageName,
 		Destination: dstRootFS,
 	}
 	err = b.ExtractRootfs(dstImageExtract, false)

pkg/compiler/backend/diff_test.go

@@ -32,12 +32,14 @@ var _ = Describe("Docker image diffs", func() {
 
 	Context("Generate diffs from docker images", func() {
 		It("Detect no changes", func() {
-			err := b.DownloadImage(compiler.CompilerBackendOptions{
-				ImageName: "alpine:latest",
-			})
+			opts := compiler.CompilerBackendOptions{
+				ImageName:   "alpine:latest",
+				Destination: "alpine:latest",
+			}
+			err := b.DownloadImage(opts)
 			Expect(err).ToNot(HaveOccurred())
 
-			layers, err := GenerateChanges(b, "alpine:latest", "alpine:latest")
+			layers, err := GenerateChanges(b, opts, opts)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(layers)).To(Equal(1))
 			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
@@ -55,7 +57,13 @@ var _ = Describe("Docker image diffs", func() {
 			})
 			Expect(err).ToNot(HaveOccurred())
 
-			layers, err := GenerateChanges(b, "quay.io/mocaccino/micro", "quay.io/mocaccino/extra")
+			layers, err := GenerateChanges(b, compiler.CompilerBackendOptions{
+				ImageName:   "quay.io/mocaccino/micro",
+				Destination: "quay.io/mocaccino/micro",
+			}, compiler.CompilerBackendOptions{
+				ImageName:   "quay.io/mocaccino/extra",
+				Destination: "quay.io/mocaccino/extra",
+			})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(layers)).To(Equal(1))
 

pkg/compiler/backend/simpledocker.go

@@ -24,6 +24,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	docker "github.com/fsouza/go-dockerclient"
 	capi "github.com/mudler/docker-companion/api"
 
 	"github.com/mudler/luet/pkg/compiler"
@@ -45,7 +46,12 @@ func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	path := opts.SourcePath
 	dockerfileName := opts.DockerFileName
-	buildarg := []string{"build", "-f", dockerfileName, "-t", name, "."}
+	context := opts.Context
+
+	if context == "" {
+		context = "."
+	}
+	buildarg := []string{"build", "-f", dockerfileName, "-t", name, context}
 
 	Debug(":whale2: Building image " + name)
 	cmd := exec.Command("docker", buildarg...)
@@ -56,6 +62,21 @@ func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
 	}
 	Info(":whale: Building image " + name + " done")
 
+	if os.Getenv("DOCKER_SQUASH") == "true" {
+		Info(":whale: Squashing image " + name)
+		var client *docker.Client
+
+		client, err = docker.NewClientFromEnv()
+		if err != nil {
+			return errors.Wrap(err, "could not connect to the Docker daemon")
+		}
+		err = capi.Squash(client, name, name)
+		if err != nil {
+			return errors.Wrap(err, "Failed squashing image")
+		}
+		Info(":whale: Squashing image " + name + " done")
+	}
+
 	if config.LuetCfg.GetGeneral().ShowBuildOutput {
 		Info(string(out))
 	} else {
@@ -83,7 +104,7 @@ func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
 	cmd := exec.Command("docker", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed pulling image: "+string(out))
 	}
 	Info(":whale: Downloaded image:", name)
 	return nil
@@ -95,13 +116,17 @@ func (*SimpleDocker) ImageExists(imagename string) bool {
 	cmd := exec.Command("docker", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		Warning("Image not present")
+		Debug("Image not present")
 		Debug(string(out))
 		return false
 	}
 	return true
 }
 
+func (*SimpleDocker) ImageAvailable(imagename string) bool {
+	return imageAvailable(imagename)
+}
+
 func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
@@ -158,10 +183,23 @@ type ManifestEntry struct {
 	Layers []string `json:"Layers"`
 }
 
-func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
+func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
 	src := opts.SourcePath
 	dst := opts.Destination
 
+	if src == "" && opts.ImageName != "" {
+		tempUnpack, err := ioutil.TempDir(dst, "tempUnpack")
+		if err != nil {
+			return errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+		defer os.RemoveAll(tempUnpack) // clean up
+		imageExport := filepath.Join(tempUnpack, "image.tar")
+		if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
+			return errors.Wrap(err, "while exporting image before extraction")
+		}
+		src = imageExport
+	}
+
 	rootfs, err := ioutil.TempDir(dst, "tmprootfs")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
@@ -202,7 +240,7 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 			}
 		}
 	}
-
+	// TODO: Drop capi in favor of the img approach already used in pkg/installer/repository
 	export, err := capi.CreateExport(rootfs)
 	if err != nil {
 		return err
@@ -222,7 +260,7 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 }
 
 // Changes retrieves changes between image layers
-func (d *SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+func (d *SimpleDocker) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
 	diffs, err := GenerateChanges(d, fromImage, toImage)
 
 	if config.LuetCfg.GetGeneral().Debug {

pkg/compiler/backend/simpledocker_test.go

@@ -18,6 +18,7 @@ package backend_test
 import (
 	. "github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/solver"
 
 	"io/ioutil"
 	"os"
@@ -40,7 +41,7 @@ var _ = Describe("Docker backend", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -86,33 +87,48 @@ ENV PACKAGE_CATEGORY=app-admin`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
-			opts = CompilerBackendOptions{
+			opts2 := CompilerBackendOptions{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
 				Destination:    filepath.Join(tmpdir, "output2.tar"),
 			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
+			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
-			Expect(b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))).To(Equal(
+			artifacts := []ArtifactNode{{
+				Name: "/luetbuild/LuetDockerfile",
+				Size: 175,
+			}}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
+			Expect(b.Changes(opts, opts2)).To(Equal(
 				[]ArtifactLayer{{
 					FromImage: filepath.Join(tmpdir2, "output1.tar"),
 					ToImage:   filepath.Join(tmpdir, "output2.tar"),
 					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+						Additions: artifacts,
 					},
 				}}))
 
 		})
 
+		It("Detects available images", func() {
+			b := NewSimpleDockerBackend()
+			Expect(b.ImageAvailable("quay.io/mocaccino/extra")).To(BeTrue())
+			Expect(b.ImageAvailable("ubuntu:20.10")).To(BeTrue())
+			Expect(b.ImageAvailable("igjo5ijgo25nho52")).To(BeFalse())
+		})
 	})
 })

pkg/compiler/backend/simpleimg.go

@@ -18,6 +18,7 @@ package backend
 import (
 	"os"
 	"os/exec"
+	"strings"
 
 	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/logger"
@@ -35,9 +36,14 @@ func NewSimpleImgBackend() compiler.CompilerBackend {
 func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	path := opts.SourcePath
+	context := opts.Context
+
+	if context == "" {
+		context = "."
+	}
 	dockerfileName := opts.DockerFileName
 
-	buildarg := []string{"build", "-f", dockerfileName, "-t", name, "."}
+	buildarg := []string{"build", "-f", dockerfileName, "-t", name, context}
 	Spinner(22)
 	defer SpinnerStop()
 	Debug(":tea: Building image " + name)
@@ -59,7 +65,7 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	defer SpinnerStop()
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed removing image: "+string(out))
 	}
 
 	Info(":tea: Image " + name + " removed")
@@ -75,7 +81,7 @@ func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
 	cmd := exec.Command("img", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed downloading image: "+string(out))
 	}
 
 	Info(":tea: Image " + name + " downloaded")
@@ -97,10 +103,20 @@ func (*SimpleImg) CopyImage(src, dst string) error {
 	return nil
 }
 
+func (*SimpleImg) ImageAvailable(imagename string) bool {
+	return imageAvailable(imagename)
+}
+
+// ImageExists check if the given image is available locally
 func (*SimpleImg) ImageExists(imagename string) bool {
-	// NOOP: not implemented
-	// TODO: Since img doesn't have an inspect command,
-	// we need to parse the ls output manually
+	cmd := exec.Command("img", "ls")
+	out, err := cmd.Output()
+	if err != nil {
+		return false
+	}
+	if strings.Contains(string(out), imagename) {
+		return true
+	}
 	return false
 }
 
@@ -124,13 +140,13 @@ func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
 	Debug(":tea: Saving image " + name)
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed exporting image: "+string(out))
 	}
 	Info(":tea: Image " + name + " saved")
 	return nil
 }
 
-// TODO: Dup in docker, refactor common code in helpers for shared parts
+// ExtractRootfs extracts the docker image content inside the destination
 func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
 	name := opts.ImageName
 	path := opts.Destination
@@ -144,12 +160,11 @@ func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms
 	}
 	Info(":tea: Image " + name + " extracted")
 	return nil
-	//return NewSimpleDockerBackend().ExtractRootfs(opts, keepPerms)
 }
 
 // TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
 // Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (i *SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+func (i *SimpleImg) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
 	return GenerateChanges(i, fromImage, toImage)
 }
 

pkg/compiler/compiler.go

@@ -17,7 +17,6 @@ package compiler
 
 import (
 	"fmt"
-	"github.com/ghodss/yaml"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -27,6 +26,9 @@ import (
 	"sync"
 	"time"
 
+	bus "github.com/mudler/luet/pkg/bus"
+	yaml "gopkg.in/yaml.v2"
+
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -37,6 +39,7 @@ import (
 
 const BuildFile = "build.yaml"
 const DefinitionFile = "definition.yaml"
+const CollectionFile = "collection.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
@@ -47,9 +50,10 @@ type LuetCompiler struct {
 	Concurrency               int
 	CompressionType           CompressionImplementation
 	Options                   CompilerOptions
+	SolverOptions             solver.Options
 }
 
-func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions) Compiler {
+func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions, solvopts solver.Options) Compiler {
 	// The CompilerRecipe will gives us a tree with only build deps listed.
 	return &LuetCompiler{
 		Backend: backend,
@@ -62,8 +66,8 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *Compi
 		CompressionType: opt.CompressionType,
 		KeepImg:         opt.KeepImg,
 		Concurrency:     opt.Concurrency,
-		Clean:           opt.Clean,
 		Options:         *opt,
+		SolverOptions:   solvopts,
 	}
 }
 
@@ -178,7 +182,7 @@ func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpec
 	return artifacts, allErrors
 }
 
-func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string) error {
+func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, include bool) error {
 	var includeRegexp []*regexp.Regexp
 	for _, i := range includes {
 		r, e := regexp.Compile(i)
@@ -210,7 +214,7 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 			}
 		}
 
-		if !match {
+		if include && !match || !include && match {
 			toRemove = append(toRemove, currentpath)
 		}
 
@@ -232,9 +236,68 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 	return nil
 }
 
-func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec) (Artifact, error) {
+func (cs *LuetCompiler) unpackFs(rootfs string, concurrency int, p CompilationSpec) (Artifact, error) {
+	if p.GetPackageDir() != "" {
+		Info(":tophat: Packing from output dir", p.GetPackageDir())
+		rootfs = filepath.Join(rootfs, p.GetPackageDir())
+	}
 
-	pkgTag := ":package:  " + p.GetPackage().GetName()
+	if len(p.GetIncludes()) > 0 {
+		// strip from includes
+		cs.stripFromRootfs(p.GetIncludes(), rootfs, true)
+	}
+	if len(p.GetExcludes()) > 0 {
+		// strip from includes
+		cs.stripFromRootfs(p.GetExcludes(), rootfs, false)
+	}
+	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
+	artifact.SetCompressionType(cs.CompressionType)
+
+	if err := artifact.Compress(rootfs, concurrency); err != nil {
+		return nil, errors.Wrap(err, "Error met while creating package archive")
+	}
+
+	artifact.SetCompileSpec(p)
+	return artifact, nil
+}
+
+func (cs *LuetCompiler) unpackDelta(rootfs string, concurrency int, keepPermissions bool, p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions) (Artifact, error) {
+	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
+	if cs.Options.PullFirst && !cs.Backend.ImageExists(builderOpts.ImageName) && cs.Backend.ImageAvailable(builderOpts.ImageName) {
+		err := cs.Backend.DownloadImage(builderOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not pull image")
+		}
+	} else if !cs.Backend.ImageExists(builderOpts.ImageName) {
+		return nil, errors.New("No image found for " + builderOpts.ImageName)
+	}
+	if err := cs.Backend.ExportImage(builderOpts); err != nil {
+		return nil, errors.Wrap(err, "Could not export image"+builderOpts.ImageName)
+	}
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(builderOpts.Destination)
+	}
+	Info(pkgTag, ":hammer: Generating delta")
+	diffs, err := cs.Backend.Changes(builderOpts, runnerOpts)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not generate changes from layers")
+	}
+	artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.CompressionType)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not generate deltas")
+	}
+
+	artifact.SetCompileSpec(p)
+	return artifact, nil
+}
+
+func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
+	concurrency int, keepPermissions bool,
+	p CompilationSpec) (CompilerBackendOptions, CompilerBackendOptions, error) {
+
+	var runnerOpts, builderOpts CompilerBackendOptions
+
+	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
 
 	// Use packageImage as salt into the fp being used
 	// so the hash is unique also in cases where
@@ -248,21 +311,13 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	fp := p.GetPackage().HashFingerprint(packageImage)
 
 	if buildertaggedImage == "" {
-		buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder"
+		buildertaggedImage = cs.ImageRepository + ":builder-" + fp
 		Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image)
 	}
 
 	// TODO:  Cleanup, not actually hit
 	if packageImage == "" {
-		packageImage = cs.ImageRepository + "-" + fp
-	}
-
-	if !cs.Clean {
-		exists := cs.Backend.ImageExists(buildertaggedImage) && cs.Backend.ImageExists(packageImage)
-		if art, err := LoadArtifactFromYaml(p); err == nil && (cs.Options.SkipIfMetadataExists || exists) {
-			Debug("Artifact reloaded. Skipping build")
-			return art, err
-		}
+		packageImage = cs.ImageRepository + ":builder-invalid" + fp
 	}
 
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
@@ -270,19 +325,18 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 
 	err := os.MkdirAll(p.Rel("build"), os.ModePerm)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for building")
+		return builderOpts, runnerOpts, errors.Wrap(err, "Error met while creating tempdir for building")
 	}
 	buildDir, err := ioutil.TempDir(p.Rel("build"), "pack")
 	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for building")
+		return builderOpts, runnerOpts, errors.Wrap(err, "Error met while creating tempdir for building")
 	}
 	defer os.RemoveAll(buildDir) // clean up
 
 	// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
 	err = helpers.CopyDir(p.GetPackage().GetPath(), buildDir)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not copy package sources")
-
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not copy package sources")
 	}
 
 	// Copy file into the build context, the compilespec might have requested to do so.
@@ -293,72 +347,116 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 	}
 
-	Info(pkgTag, "Generating :whale: definition for builder image from", image)
-
 	// First we create the builder image
-	p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile"))
-	builderOpts := CompilerBackendOptions{
+	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")); err != nil {
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
+	}
+
+	if len(p.GetPreBuildSteps()) == 0 {
+		buildertaggedImage = image
+	}
+
+	// Then we write the step image, which uses the builder one
+	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")); err != nil {
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
+	}
+
+	builderOpts = CompilerBackendOptions{
 		ImageName:      buildertaggedImage,
 		SourcePath:     buildDir,
 		DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 	}
-
-	buildBuilderImage := true
-	if cs.Options.PullFirst {
-		if err := cs.Backend.DownloadImage(builderOpts); err == nil {
-			buildBuilderImage = false
-		}
-	}
-
-	if buildBuilderImage {
-		if err = cs.Backend.BuildImage(builderOpts); err != nil {
-			return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
-		}
-	}
-
-	if err = cs.Backend.ExportImage(builderOpts); err != nil {
-		return nil, errors.Wrap(err, "Could not export image")
-	}
-
-	if !cs.Options.KeepImageExport {
-		defer os.Remove(builderOpts.Destination)
-	}
-
-	if cs.Options.Push && buildBuilderImage {
-		if err = cs.Backend.Push(builderOpts); err != nil {
-			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
-		}
-	}
-	// Then we write the step image, which uses the builder one
-	p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
-	runnerOpts := CompilerBackendOptions{
+	runnerOpts = CompilerBackendOptions{
 		ImageName:      packageImage,
 		SourcePath:     buildDir,
 		DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 	}
 
-	// if !keepPackageImg {
-	// 	err = cs.Backend.ImageDefinitionToTar(runnerOpts)
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "Could not export image to tar")
-	// 	}
-	// } else {
-	buildPackageImage := true
-	if cs.Options.PullFirst {
-		//Best effort pull
-		if err := cs.Backend.DownloadImage(runnerOpts); err == nil {
-			buildPackageImage = false
+	buildAndPush := func(opts CompilerBackendOptions) error {
+		buildImage := true
+		if cs.Options.PullFirst {
+			bus.Manager.Publish(bus.EventImagePrePull, opts)
+			err := cs.Backend.DownloadImage(opts)
+			if err == nil {
+				buildImage = false
+			} else {
+				Warning("Failed to download '" + opts.ImageName + "'. Will keep going and build the image unless you use --fatal")
+				Warning(err.Error())
+			}
+			bus.Manager.Publish(bus.EventImagePostPull, opts)
+		}
+		if buildImage {
+			bus.Manager.Publish(bus.EventImagePreBuild, opts)
+			if err := cs.Backend.BuildImage(opts); err != nil {
+				return errors.Wrap(err, "Could not build image: "+image+" "+opts.DockerFileName)
+			}
+			bus.Manager.Publish(bus.EventImagePostBuild, opts)
+			if cs.Options.Push {
+				bus.Manager.Publish(bus.EventImagePrePush, opts)
+				if err = cs.Backend.Push(opts); err != nil {
+					return errors.Wrap(err, "Could not push image: "+image+" "+opts.DockerFileName)
+				}
+				bus.Manager.Publish(bus.EventImagePostPush, opts)
+			}
+		}
+		return nil
+	}
+	if len(p.GetPreBuildSteps()) != 0 {
+		Info(pkgTag, ":whale: Generating 'builder' image from", image, "as", buildertaggedImage, "with prelude steps")
+		if err := buildAndPush(builderOpts); err != nil {
+			return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
 		}
 	}
 
-	if buildPackageImage {
-		if err := cs.Backend.BuildImage(runnerOpts); err != nil {
-			return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
-		}
+	// Even if we might not have any steps to build, we do that so we can tag the image used in this moment and use that to cache it in a registry, or in the system.
+	// acting as a docker tag.
+	Info(pkgTag, ":whale: Generating 'package' image from", buildertaggedImage, "as", packageImage, "with build steps")
+	if err := buildAndPush(runnerOpts); err != nil {
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
 	}
 
+	return builderOpts, runnerOpts, nil
+}
+
+func (cs *LuetCompiler) genArtifact(p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions, concurrency int, keepPermissions bool) (Artifact, error) {
+
+	// generate Artifact
+	var artifact Artifact
+	var rootfs string
+	var err error
+	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
+
+	// We can't generate delta in this case. It implies the package is a virtual, and nothing has to be done really
+	if p.EmptyPackage() {
+		fakePackage := p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")
+
+		rootfs, err = ioutil.TempDir(p.GetOutputPath(), "rootfs")
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not create tempdir")
+		}
+		defer os.RemoveAll(rootfs) // clean up
+
+		artifact := NewPackageArtifact(fakePackage)
+		artifact.SetCompressionType(cs.CompressionType)
+
+		if err := artifact.Compress(rootfs, concurrency); err != nil {
+			return nil, errors.Wrap(err, "Error met while creating package archive")
+		}
+
+		artifact.SetCompileSpec(p)
+		artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
+
+		err = artifact.WriteYaml(p.GetOutputPath())
+		if err != nil {
+			return artifact, errors.Wrap(err, "Failed while writing metadata file")
+		}
+		Info(pkgTag, "   :white_check_mark: done (empty virtual package)")
+		return artifact, nil
+	}
+
+	// prepare folder content of the image with the package compiled inside
 	if err := cs.Backend.ExportImage(runnerOpts); err != nil {
 		return nil, errors.Wrap(err, "Failed exporting image")
 	}
@@ -367,24 +465,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		defer os.Remove(runnerOpts.Destination)
 	}
 
-	if cs.Options.Push && buildPackageImage {
-		err = cs.Backend.Push(runnerOpts)
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
-		}
-	}
-	//	}
-
-	var artifact Artifact
-	unpack := p.ImageUnpack()
-
-	// If package_dir was specified in the spec, we want to treat the content of the directory
-	// as the root of our archive.  ImageUnpack is implied to be true. override it
-	if p.GetPackageDir() != "" {
-		unpack = true
-	}
-
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	rootfs, err = ioutil.TempDir(p.GetOutputPath(), "rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
 	}
@@ -392,59 +473,24 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 
 	// TODO: Compression and such
 	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
-		ImageName:  packageImage,
+		ImageName:  runnerOpts.ImageName,
 		SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions)
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not extract rootfs")
 	}
 
-	if !keepImg {
-		// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
-		// TODO: Handle caching and optionally do not remove things
-		err = cs.Backend.RemoveImage(builderOpts)
+	if p.UnpackedPackage() {
+		// Take content of container as a base for our package files
+		artifact, err = cs.unpackFs(rootfs, concurrency, p)
 		if err != nil {
-			Warning("Could not remove image ", builderOpts.ImageName)
-			//	return nil, errors.Wrap(err, "Could not remove image")
+			return nil, errors.Wrap(err, "Error met while extracting image")
 		}
-		err = cs.Backend.RemoveImage(runnerOpts)
-		if err != nil {
-			Warning("Could not remove image ", builderOpts.ImageName)
-			//	return nil, errors.Wrap(err, "Could not remove image")
-		}
-	}
-
-	if unpack {
-
-		if p.GetPackageDir() != "" {
-			Info(":tophat: Packing from output dir", p.GetPackageDir())
-			rootfs = filepath.Join(rootfs, p.GetPackageDir())
-		}
-
-		if len(p.GetIncludes()) > 0 {
-			// strip from includes
-			cs.stripIncludesFromRootfs(p.GetIncludes(), rootfs)
-		}
-		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
-		artifact.SetCompressionType(cs.CompressionType)
-
-		err = artifact.Compress(rootfs, concurrency)
-		if err != nil {
-			return nil, errors.Wrap(err, "Error met while creating package archive")
-		}
-
-		artifact.SetCompileSpec(p)
 	} else {
-		Info(pkgTag, "Generating delta")
-		diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
+		// Generate delta between the two images
+		artifact, err = cs.unpackDelta(rootfs, concurrency, keepPermissions, p, builderOpts, runnerOpts)
 		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate changes from layers")
+			return nil, errors.Wrap(err, "Error met while generating delta")
 		}
-		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate deltas")
-		}
-
-		artifact.SetCompileSpec(p)
 	}
 
 	filelist, err := artifact.FileList()
@@ -453,7 +499,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	}
 
 	artifact.SetFiles(filelist)
-
 	artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
 
 	err = artifact.WriteYaml(p.GetOutputPath())
@@ -465,6 +510,66 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	return artifact, nil
 }
 
+func (cs *LuetCompiler) waitForImage(image string) {
+	if cs.Options.PullFirst && cs.Options.Wait && !cs.Backend.ImageAvailable(image) {
+		Info(fmt.Sprintf("Waiting for image %s to be available... :zzz:", image))
+		Spinner(22)
+		defer SpinnerStop()
+		for !cs.Backend.ImageAvailable(image) {
+			Info(fmt.Sprintf("Image %s not available yet, sleeping", image))
+			time.Sleep(5 * time.Second)
+		}
+	}
+}
+
+func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string,
+	concurrency int,
+	keepPermissions, keepImg bool,
+	p CompilationSpec, generateArtifact bool) (Artifact, error) {
+
+	// If it is a virtual, check if we have to generate an empty artifact or not.
+	if generateArtifact && p.EmptyPackage() && !p.HasImageSource() {
+		return cs.genArtifact(p, CompilerBackendOptions{}, CompilerBackendOptions{}, concurrency, keepPermissions)
+	} else if p.EmptyPackage() && !p.HasImageSource() {
+		return &PackageArtifact{}, nil
+	}
+
+	if !generateArtifact {
+		exists := cs.Backend.ImageExists(packageImage)
+		if art, err := LoadArtifactFromYaml(p); err == nil && exists { // If YAML is correctly loaded, and both images exists, no reason to rebuild.
+			Debug("Artifact reloaded from YAML. Skipping build")
+			return art, err
+		}
+		cs.waitForImage(packageImage)
+		if cs.Options.PullFirst && cs.Backend.ImageAvailable(packageImage) {
+			return &PackageArtifact{}, nil
+		}
+	}
+
+	builderOpts, runnerOpts, err := cs.buildPackageImage(image, buildertaggedImage, packageImage, concurrency, keepPermissions, p)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed building package image")
+	}
+
+	if !keepImg {
+		defer func() {
+			// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
+			if err := cs.Backend.RemoveImage(builderOpts); err != nil {
+				Warning("Could not remove image ", builderOpts.ImageName)
+			}
+			if err := cs.Backend.RemoveImage(runnerOpts); err != nil {
+				Warning("Could not remove image ", runnerOpts.ImageName)
+			}
+		}()
+	}
+
+	if !generateArtifact {
+		return &PackageArtifact{}, nil
+	}
+
+	return cs.genArtifact(p, builderOpts, runnerOpts, concurrency, keepPermissions)
+}
+
 func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) {
 	compilerSpecs := NewLuetCompilationspecs()
 
@@ -514,7 +619,7 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]Com
 
 func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
 
-	s := solver.NewResolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
+	s := solver.NewResolver(cs.SolverOptions, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
 
 	solution, err := s.Install(pkg.Packages{p.GetPackage()})
 	if err != nil {
@@ -554,19 +659,30 @@ func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifa
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
 	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
-	if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" {
-		Error("Package with no deps and no seed image supplied, bailing out")
-		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() +
-			" with no deps and no seed image supplied, bailing out")
+	Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
+	if !p.HasImageSource() && !p.EmptyPackage() {
+		return nil,
+			fmt.Errorf(
+				"%s is invalid: package has no dependencies and no seed image supplied while it has steps defined",
+				p.GetPackage().GetFingerPrint(),
+			)
 	}
 
 	targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
 	targetPackageHash := cs.ImageRepository + ":" + targetAssertion.Hash.PackageHash
 
+	bus.Manager.Publish(bus.EventPackagePreBuild, struct {
+		CompileSpec CompilationSpec
+		Assert      solver.PackageAssert
+	}{
+		CompileSpec: p,
+		Assert:      *targetAssertion,
+	})
+
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p)
+		return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@@ -580,6 +696,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	depsN := 0
 	currentN := 0
 
+	packageDeps := !cs.Options.PackageTargetOnly
 	if !cs.Options.NoDeps {
 		Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
 		for _, assertion := range dependencies { //highly dependent on the order
@@ -589,8 +706,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 
 		for _, assertion := range dependencies { //highly dependent on the order
 			currentN++
-			pkgTag := fmt.Sprintf(":package:  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
-			Info(pkgTag, "   :zap:  Building dependency")
+			pkgTag := fmt.Sprintf(":package: %d/%d %s ⤑ :hammer: build %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
+			Info(pkgTag, " starts")
 			compileSpec, err := cs.FromPackage(assertion.Package)
 			if err != nil {
 				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
@@ -602,10 +719,18 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from", buildImageHash)
 			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
 
+			bus.Manager.Publish(bus.EventPackagePreBuild, struct {
+				CompileSpec CompilationSpec
+				Assert      solver.PackageAssert
+			}{
+				CompileSpec: compileSpec,
+				Assert:      assertion,
+			})
+
 			lastHash = currentPackageImageHash
 			if compileSpec.GetImage() != "" {
 				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
-				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 				if err != nil {
 					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				}
@@ -615,14 +740,23 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 			}
 
 			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
-			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 			if err != nil {
 				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				//	deperrs = append(deperrs, err)
 				//		break // stop at first error
 			}
+
+			bus.Manager.Publish(bus.EventPackagePostBuild, struct {
+				CompileSpec CompilationSpec
+				Artifact    Artifact
+			}{
+				CompileSpec: compileSpec,
+				Artifact:    artifact,
+			})
+
 			departifacts = append(departifacts, artifact)
-			Info(pkgTag, ":collision: Done")
+			Info(pkgTag, ":white_check_mark: Done")
 		}
 
 	} else if len(dependencies) > 0 {
@@ -630,14 +764,23 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	}
 
 	if !cs.Options.OnlyDeps {
-		Info(":package:", p.GetPackage().HumanReadableString(), ":cyclone:  Building package target from:", lastHash)
-		artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p)
+		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
+		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", lastHash)
+		artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 		if err != nil {
 			return artifact, err
 		}
 		artifact.SetDependencies(departifacts)
 		artifact.SetSourceAssertion(p.GetSourceAssertion())
 
+		bus.Manager.Publish(bus.EventPackagePostBuild, struct {
+			CompileSpec CompilationSpec
+			Artifact    Artifact
+		}{
+			CompileSpec: p,
+			Artifact:    artifact,
+		})
+
 		return artifact, err
 	} else {
 		return departifacts[len(departifacts)-1], nil
@@ -653,32 +796,51 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 		return nil, err
 	}
 
-	buildFile := pack.Rel(BuildFile)
-	if !helpers.Exists(buildFile) {
-		return nil, errors.New("No build file present for " + p.GetFingerPrint())
-	}
-	defFile := pack.Rel(DefinitionFile)
-	if !helpers.Exists(defFile) {
-		return nil, errors.New("No build file present for " + p.GetFingerPrint())
-	}
-	def, err := ioutil.ReadFile(defFile)
-	if err != nil {
-		return nil, err
+	var dataresult []byte
+
+	val := pack.Rel(DefinitionFile)
+	if _, err := os.Stat(pack.Rel(CollectionFile)); err == nil {
+		val = pack.Rel(CollectionFile)
+
+		data, err := ioutil.ReadFile(val)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+val)
+		}
+
+		dataBuild, err := ioutil.ReadFile(pack.Rel(BuildFile))
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+val)
+		}
+		packsRaw, err := pkg.GetRawPackages(data)
+
+		raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
+
+		d := map[string]interface{}{}
+		if len(cs.Options.BuildValuesFile) > 0 {
+			defBuild, err := ioutil.ReadFile(cs.Options.BuildValuesFile)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+val)
+			}
+			err = yaml.Unmarshal(defBuild, &d)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+val)
+			}
+		}
+
+		dat, err := helpers.RenderHelm(string(dataBuild), raw, d)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
+		}
+		dataresult = []byte(dat)
+	} else {
+		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, cs.Options.BuildValuesFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
+		}
+		dataresult = []byte(out)
 	}
 
-	build, err := ioutil.ReadFile(buildFile)
-	if err != nil {
-		return nil, err
-	}
-	var values templatedata
-	if err = yaml.Unmarshal(def, &values); err != nil {
-		return nil, err
-	}
-	out, err := helpers.RenderHelm(string(build), values)
-	if err != nil {
-		return nil, err
-	}
-	return NewLuetCompilationSpec([]byte(out), pack)
+	return NewLuetCompilationSpec(dataresult, pack)
 }
 
 func (cs *LuetCompiler) GetBackend() CompilerBackend {

pkg/compiler/compiler_test.go

@@ -23,6 +23,7 @@ import (
 	sd "github.com/mudler/luet/pkg/compiler/backend"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -38,7 +39,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -49,8 +50,8 @@ var _ = Describe("Compiler", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			compiler.SetConcurrency(2)
@@ -82,7 +83,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -108,7 +109,7 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
-	Context("Templated packages",func(){
+	Context("Templated packages", func() {
 		It("Renders", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
@@ -117,10 +118,10 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/templates")
 			Expect(err).ToNot(HaveOccurred())
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
-			pkg ,err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			spec, err := compiler.FromPackage(pkg)
@@ -141,7 +142,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -198,7 +199,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -240,7 +241,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -264,6 +265,146 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
+		It("Compiles and excludes files", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludes")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles includes and excludes files", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludesincludes")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+		})
+
+		It("Compiles and excludes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles includes and excludes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludeincludeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
 		It("Compiles and includes ony wanted files also from unpacked packages", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
@@ -275,7 +416,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -313,7 +454,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -354,7 +495,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -398,7 +539,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -440,7 +581,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -478,7 +619,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
 			Expect(err).ToNot(HaveOccurred())
@@ -517,7 +658,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
@@ -569,7 +710,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -612,7 +753,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -645,7 +786,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{
 				Name:     "dironly",
@@ -700,7 +841,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -736,7 +877,7 @@ var _ = Describe("Compiler", func() {
 			err := generalRecipe.Load("../../tests/fixtures/includeimage")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
 			Expect(err).ToNot(HaveOccurred())
@@ -755,7 +896,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())

pkg/compiler/interface.go

@@ -42,6 +42,7 @@ type CompilerBackendOptions struct {
 	SourcePath     string
 	DockerFileName string
 	Destination    string
+	Context        string
 }
 
 type CompilerOptions struct {
@@ -49,13 +50,15 @@ type CompilerOptions struct {
 	PullFirst, KeepImg, Push bool
 	Concurrency              int
 	CompressionType          CompressionImplementation
-	Clean                    bool
 	KeepImageExport          bool
 
-	OnlyDeps             bool
-	NoDeps               bool
-	SolverOptions        config.LuetSolverOptions
-	SkipIfMetadataExists bool
+	Wait            bool
+	OnlyDeps        bool
+	NoDeps          bool
+	SolverOptions   config.LuetSolverOptions
+	BuildValuesFile string
+
+	PackageTargetOnly bool
 }
 
 func NewDefaultCompilerOptions() *CompilerOptions {
@@ -66,7 +69,6 @@ func NewDefaultCompilerOptions() *CompilerOptions {
 		CompressionType: None,
 		KeepImg:         true,
 		Concurrency:     runtime.NumCPU(),
-		Clean:           true,
 		OnlyDeps:        false,
 		NoDeps:          false,
 	}
@@ -76,7 +78,7 @@ type CompilerBackend interface {
 	BuildImage(CompilerBackendOptions) error
 	ExportImage(CompilerBackendOptions) error
 	RemoveImage(CompilerBackendOptions) error
-	Changes(fromImage, toImage string) ([]ArtifactLayer, error)
+	Changes(fromImage, toImage CompilerBackendOptions) ([]ArtifactLayer, error)
 	ImageDefinitionToTar(CompilerBackendOptions) error
 	ExtractRootfs(opts CompilerBackendOptions, keepPerms bool) error
 
@@ -84,6 +86,7 @@ type CompilerBackend interface {
 	DownloadImage(opts CompilerBackendOptions) error
 
 	Push(opts CompilerBackendOptions) error
+	ImageAvailable(string) bool
 
 	ImageExists(string) bool
 }
@@ -108,9 +111,13 @@ type Artifact interface {
 
 	SetFiles(f []string)
 	GetFiles() []string
+	GetFileName() string
 
 	GetChecksums() Checksums
 	SetChecksums(c Checksums)
+
+	GenerateFinalImage(string, CompilerBackend, bool) (CompilerBackendOptions, error)
+	GetUncompressedName() string
 }
 
 type ArtifactNode struct {
@@ -145,6 +152,7 @@ type ArtifactLayersSummary struct {
 type CompilationSpec interface {
 	ImageUnpack() bool // tells if the definition is just an image
 	GetIncludes() []string
+	GetExcludes() []string
 
 	RenderBuildImage() (string, error)
 	WriteBuildImageDefinition(string) error
@@ -175,6 +183,10 @@ type CompilationSpec interface {
 
 	SetPackageDir(string)
 	GetPackageDir() string
+
+	EmptyPackage() bool
+	UnpackedPackage() bool
+	HasImageSource() bool
 }
 
 type CompilationSpecs interface {

pkg/compiler/spec.go

@@ -102,6 +102,7 @@ type LuetCompilationSpec struct {
 	OutputPath string   `json:"-"` // Where the build processfiles go
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
+	Excludes   []string `json:"excludes"`
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
@@ -148,6 +149,10 @@ func (cs *LuetCompilationSpec) GetIncludes() []string {
 	return cs.Includes
 }
 
+func (cs *LuetCompilationSpec) GetExcludes() []string {
+	return cs.Excludes
+}
+
 func (cs *LuetCompilationSpec) GetRetrieve() []string {
 	return cs.Retrieve
 }
@@ -180,6 +185,24 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 	cs.Seed = s
 }
 
+func (cs *LuetCompilationSpec) EmptyPackage() bool {
+	return len(cs.BuildSteps()) == 0 && len(cs.GetPreBuildSteps()) == 0 && !cs.UnpackedPackage()
+}
+
+func (cs *LuetCompilationSpec) UnpackedPackage() bool {
+	// If package_dir was specified in the spec, we want to treat the content of the directory
+	// as the root of our archive.  ImageUnpack is implied to be true. override it
+	unpack := cs.ImageUnpack()
+	if cs.GetPackageDir() != "" {
+		unpack = true
+	}
+	return unpack
+}
+
+func (cs *LuetCompilationSpec) HasImageSource() bool {
+	return len(cs.GetPackage().GetRequires()) != 0 || cs.GetImage() != ""
+}
+
 func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
 	var err error
 	if len(cs.Retrieve) > 0 {
@@ -198,10 +221,9 @@ func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
 	return err
 }
 
-// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
-func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
+func (cs *LuetCompilationSpec) genDockerfile(image string, steps []string) string {
 	spec := `
-FROM ` + cs.GetSeedImage() + `
+FROM ` + image + `
 COPY . /luetbuild
 WORKDIR /luetbuild
 ENV PACKAGE_NAME=` + cs.Package.GetName() + `
@@ -226,31 +248,22 @@ ADD ` + s + ` /luetbuild/`
 ENV ` + s
 	}
 
-	for _, s := range cs.GetPreBuildSteps() {
+	for _, s := range steps {
 		spec = spec + `
 RUN ` + s
 	}
-	return spec, nil
+	return spec
 }
 
-// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
+// RenderBuildImage renders the dockerfile of the image used as a pre-build step
+func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
+	return cs.genDockerfile(cs.GetSeedImage(), cs.GetPreBuildSteps()), nil
+
+}
+
+// RenderStepImage renders the dockerfile used for the image used for building the package
 func (cs *LuetCompilationSpec) RenderStepImage(image string) (string, error) {
-	spec := `
-FROM ` + image + `
-ENV PACKAGE_NAME=` + cs.Package.GetName() + `
-ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
-ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
-
-	for _, s := range cs.Env {
-		spec = spec + `
-ENV ` + s
-	}
-	for _, s := range cs.BuildSteps() {
-		spec = spec + `
-RUN ` + s
-	}
-
-	return spec, nil
+	return cs.genDockerfile(image, cs.BuildSteps()), nil
 }
 
 func (cs *LuetCompilationSpec) WriteBuildImageDefinition(path string) error {

pkg/compiler/spec_test.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -61,7 +62,7 @@ var _ = Describe("Spec", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -95,6 +96,8 @@ ENV test=1`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
@@ -114,7 +117,7 @@ RUN echo bar > /test2`))
 
 		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 		spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
 		Expect(err).ToNot(HaveOccurred())
 
@@ -167,9 +170,13 @@ ENV test=1`))
 
 		Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=a
 ENV PACKAGE_VERSION=1.0
 ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
 ENV test=1
 RUN echo foo > /test
 RUN echo bar > /test2`))

pkg/config/config.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/mudler/luet/pkg/helpers"
+	pkg "github.com/mudler/luet/pkg/package"
 	solver "github.com/mudler/luet/pkg/solver"
 
 	v "github.com/spf13/viper"
@@ -64,10 +65,20 @@ type LuetGeneralConfig struct {
 }
 
 type LuetSolverOptions struct {
-	Type        string  `mapstructure:"type"`
-	LearnRate   float32 `mapstructure:"rate"`
-	Discount    float32 `mapstructure:"discount"`
-	MaxAttempts int     `mapstructure:"max_attempts"`
+	Type           string            `mapstructure:"type"`
+	LearnRate      float32           `mapstructure:"rate"`
+	Discount       float32           `mapstructure:"discount"`
+	MaxAttempts    int               `mapstructure:"max_attempts"`
+	Implementation solver.SolverType `mapstructure:"implementation"`
+}
+
+func (opts LuetSolverOptions) ResolverIsSet() bool {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		return true
+	default:
+		return false
+	}
 }
 
 func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
@@ -96,7 +107,7 @@ type LuetSystemConfig struct {
 	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
 }
 
-func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
 	dbpath = filepath.Join(dbpath, "repos/"+name)
 	err := os.MkdirAll(dbpath, os.ModePerm)
@@ -106,7 +117,7 @@ func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	return dbpath
 }
 
-func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+func (sc *LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
 	dbpath := filepath.Join(sc.Rootfs,
 		sc.DatabasePath)
 	err := os.MkdirAll(dbpath, os.ModePerm)
@@ -116,7 +127,7 @@ func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
 	return dbpath
 }
 
-func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
 	var cachepath string
 	if sc.PkgsCachePath != "" {
 		cachepath = sc.PkgsCachePath
@@ -134,6 +145,10 @@ func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
 	return
 }
 
+func (sc *LuetSystemConfig) GetRootFsAbs() (string, error) {
+	return filepath.Abs(sc.Rootfs)
+}
+
 type LuetRepository struct {
 	Name           string            `json:"name" yaml:"name" mapstructure:"name"`
 	Description    string            `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
@@ -203,6 +218,7 @@ type LuetConfig struct {
 	RepositoriesConfDir  []string         `mapstructure:"repos_confdir"`
 	ConfigProtectConfDir []string         `mapstructure:"config_protect_confdir"`
 	ConfigProtectSkip    bool             `mapstructure:"config_protect_skip"`
+	ConfigFromHost       bool             `mapstructure:"config_from_host"`
 	CacheRepositories    []LuetRepository `mapstructure:"repetitors"`
 	SystemRepositories   []LuetRepository `mapstructure:"repositories"`
 
@@ -250,6 +266,8 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
 	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
 	viper.SetDefault("config_protect_skip", false)
+	// TODO: Set default to false when we are ready for migration.
+	viper.SetDefault("config_from_host", true)
 	viper.SetDefault("cache_repositories", []string{})
 	viper.SetDefault("system_repositories", []string{})
 
@@ -259,6 +277,16 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("solver.max_attempts", 9000)
 }
 
+func (c *LuetConfig) GetSystemDB() pkg.PackageDatabase {
+	switch LuetCfg.GetSystem().DatabaseEngine {
+	case "boltdb":
+		return pkg.NewBoltDatabase(
+			filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+	default:
+		return pkg.NewInMemoryDatabase(true)
+	}
+}
+
 func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
 	c.SystemRepositories = append(c.SystemRepositories, r)
 }

pkg/config/config_protect.go

@@ -18,6 +18,8 @@ package config
 
 import (
 	"fmt"
+	"path/filepath"
+	"strings"
 )
 
 type ConfigProtectConfFile struct {
@@ -39,3 +41,79 @@ func (c *ConfigProtectConfFile) String() string {
 	return fmt.Sprintf("[%s] filename: %s, dirs: %s", c.Name, c.Filename,
 		c.Directories)
 }
+
+type ConfigProtect struct {
+	AnnotationDir string
+	MapProtected  map[string]bool
+}
+
+func NewConfigProtect(annotationDir string) *ConfigProtect {
+	if len(annotationDir) > 0 && annotationDir[0:1] != "/" {
+		annotationDir = "/" + annotationDir
+	}
+	return &ConfigProtect{
+		AnnotationDir: annotationDir,
+		MapProtected:  make(map[string]bool, 0),
+	}
+}
+
+func (c *ConfigProtect) AddAnnotationDir(d string) {
+	c.AnnotationDir = d
+}
+
+func (c *ConfigProtect) GetAnnotationDir() string {
+	return c.AnnotationDir
+}
+
+func (c *ConfigProtect) Map(files []string) {
+	if LuetCfg.ConfigProtectSkip {
+		return
+	}
+
+	for _, file := range files {
+
+		if file[0:1] != "/" {
+			file = "/" + file
+		}
+
+		if len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
+				for _, dir := range conf.Directories {
+					// Note file is without / at begin (on unpack)
+					if strings.HasPrefix(file, filepath.Clean(dir)) {
+						// docker archive modifier works with path without / at begin.
+						c.MapProtected[file] = true
+						goto nextFile
+					}
+				}
+			}
+		}
+
+		if c.AnnotationDir != "" && strings.HasPrefix(file, filepath.Clean(c.AnnotationDir)) {
+			c.MapProtected[file] = true
+		}
+	nextFile:
+	}
+
+}
+
+func (c *ConfigProtect) Protected(file string) bool {
+	if file[0:1] != "/" {
+		file = "/" + file
+	}
+	_, ans := c.MapProtected[file]
+	return ans
+}
+
+func (c *ConfigProtect) GetProtectFiles(withSlash bool) []string {
+	ans := []string{}
+
+	for key, _ := range c.MapProtected {
+		if withSlash {
+			ans = append(ans, key)
+		} else {
+			ans = append(ans, key[1:])
+		}
+	}
+	return ans
+}

pkg/config/config_protect_test.go

@@ -0,0 +1,118 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+
+	Context("Test config protect", func() {
+
+		It("Protect1", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("/etc")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+		It("Protect2", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{},
+			))
+		})
+
+		It("Protect3: Annotation dir without initial slash", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("etc")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+	})
+
+})

pkg/helpers/file.go

@@ -19,11 +19,51 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
+	"strings"
 	"time"
 
 	copy "github.com/otiai10/copy"
 )
 
+func OrderFiles(target string, files []string) ([]string, []string) {
+
+	var newFiles []string
+	var notPresent []string
+
+	for _, f := range files {
+		target := filepath.Join(target, f)
+		fi, err := os.Lstat(target)
+		if err != nil {
+			notPresent = append(notPresent, f)
+			continue
+		}
+		if m := fi.Mode(); !m.IsDir() {
+			newFiles = append(newFiles, f)
+		}
+	}
+
+	dirs := []string{}
+
+	for _, f := range files {
+		target := filepath.Join(target, f)
+		fi, err := os.Lstat(target)
+		if err != nil {
+			continue
+		}
+		if m := fi.Mode(); m.IsDir() {
+			dirs = append(dirs, f)
+		}
+	}
+
+	// Compare how many sub paths there are, and push at the end the ones that have less subpaths
+	sort.Slice(dirs, func(i, j int) bool {
+		return len(strings.Split(dirs[i], string(os.PathSeparator))) > len(strings.Split(dirs[j], string(os.PathSeparator)))
+	})
+
+	return append(newFiles, dirs...), notPresent
+}
+
 func ListDir(dir string) ([]string, error) {
 	content := []string{}
 
@@ -77,14 +117,15 @@ func Read(file string) (string, error) {
 	return string(dat), nil
 }
 
-func ensureDir(fileName string) {
+func EnsureDir(fileName string) error {
 	dirName := filepath.Dir(fileName)
 	if _, serr := os.Stat(dirName); serr != nil {
 		merr := os.MkdirAll(dirName, os.ModePerm) // FIXME: It should preserve permissions from src to dst instead
 		if merr != nil {
-			panic(merr)
+			return merr
 		}
 	}
+	return nil
 }
 
 // CopyFile copies the contents of the file named src to the file named
@@ -93,7 +134,7 @@ func ensureDir(fileName string) {
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
 func CopyFile(src, dst string) (err error) {
-	return copy.Copy(src, dst)
+	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }
 
 func IsDirectory(path string) (bool, error) {
@@ -110,5 +151,5 @@ func IsDirectory(path string) (bool, error) {
 func CopyDir(src string, dst string) (err error) {
 	src = filepath.Clean(src)
 	dst = filepath.Clean(dst)
-	return copy.Copy(src, dst)
+	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }

pkg/helpers/file_test.go

@@ -16,6 +16,10 @@
 package helpers_test
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -28,4 +32,55 @@ var _ = Describe("Helpers", func() {
 			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
 		})
 	})
+
+	Context("Orders dir and files correctly", func() {
+		It("puts files first and folders at end", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			err = ioutil.WriteFile(filepath.Join(testDir, "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(testDir, "baz"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "bar"), 0755)
+			Expect(err).ToNot(HaveOccurred())
+			err = ioutil.WriteFile(filepath.Join(testDir, "bar", "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "baz2"), 0755)
+			Expect(err).ToNot(HaveOccurred())
+			err = ioutil.WriteFile(filepath.Join(testDir, "baz2", "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			ordered, notExisting := OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
+
+			Expect(ordered).To(Equal([]string{"baz", "bar/foo", "foo", "baz2/foo", "bar", "baz2"}))
+			Expect(notExisting).To(Equal([]string{"notexisting"}))
+		})
+
+		It("orders correctly when there are folders with folders", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			err = os.MkdirAll(filepath.Join(testDir, "bar"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+			err = os.MkdirAll(filepath.Join(testDir, "foo"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "bar"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz", "fa"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			ordered, _ := OrderFiles(testDir, []string{"foo", "foo/bar", "bar", "foo/baz/fa", "foo/baz"})
+			Expect(ordered).To(Equal([]string{"foo/baz/fa", "foo/bar", "foo/baz", "foo", "bar"}))
+		})
+	})
 })

pkg/helpers/helm.go

@@ -1,15 +1,17 @@
 package helpers
 
 import (
+	"io/ioutil"
 
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chartutil"
 	"helm.sh/helm/v3/pkg/engine"
-	"github.com/pkg/errors"
 )
 
 // RenderHelm renders the template string with helm
-func RenderHelm(template string, values map[string]interface{}) (string,error) {
+func RenderHelm(template string, values, d map[string]interface{}) (string, error) {
 	c := &chart.Chart{
 		Metadata: &chart.Metadata{
 			Name:    "",
@@ -18,17 +20,52 @@ func RenderHelm(template string, values map[string]interface{}) (string,error) {
 		Templates: []*chart.File{
 			{Name: "templates", Data: []byte(template)},
 		},
-		Values:    map[string]interface{}{"Values":values},
+		Values: map[string]interface{}{"Values": values},
 	}
 
-	v, err := chartutil.CoalesceValues(c, map[string]interface{}{})
+	v, err := chartutil.CoalesceValues(c, map[string]interface{}{"Values": d})
 	if err != nil {
-		return "",errors.Wrap(err,"while rendering template")
+		return "", errors.Wrap(err, "while rendering template")
 	}
 	out, err := engine.Render(c, v)
 	if err != nil {
-		return "",errors.Wrap(err,"while rendering template")
+		return "", errors.Wrap(err, "while rendering template")
 	}
 
-	return out["templates"],nil
+	return out["templates"], nil
+}
+
+type templatedata map[string]interface{}
+
+func RenderFiles(toTemplate, valuesFile string, defaultFile string) (string, error) {
+	raw, err := ioutil.ReadFile(toTemplate)
+	if err != nil {
+		return "", errors.Wrap(err, "reading file "+toTemplate)
+	}
+
+	if !Exists(valuesFile) {
+		return "", errors.Wrap(err, "file not existing "+valuesFile)
+	}
+	val, err := ioutil.ReadFile(valuesFile)
+	if err != nil {
+		return "", errors.Wrap(err, "reading file "+valuesFile)
+	}
+
+	var values templatedata
+	d := templatedata{}
+	if len(defaultFile) > 0 {
+		def, err := ioutil.ReadFile(defaultFile)
+		if err != nil {
+			return "", errors.Wrap(err, "reading file "+valuesFile)
+		}
+		if err = yaml.Unmarshal(def, &d); err != nil {
+			return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
+		}
+	}
+
+	if err = yaml.Unmarshal(val, &values); err != nil {
+		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
+	}
+
+	return RenderHelm(string(raw), values, d)
 }

pkg/helpers/helm_test.go

@@ -16,17 +16,132 @@
 package helpers_test
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
+func writeFile(path string, content string) {
+	err := ioutil.WriteFile(path, []byte(content), 0644)
+	Expect(err).ToNot(HaveOccurred())
+}
+
 var _ = Describe("Helpers", func() {
 	Context("RenderHelm", func() {
 		It("Renders templates", func() {
-			out, err := RenderHelm("{{.Values.Test}}",map[string]interface{}{"Test":"foo"})
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(out).To(Equal("foo"))
+			Expect(out).To(Equal("foobar"))
+		})
+		It("Renders templates with overrides", func() {
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foobar"))
+		})
+
+		It("Renders templates", func() {
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foobar"))
+		})
+
+		It("Render files default overrides", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+			writeFile(d, `
+foo: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("baz"))
+
+		})
+
+		It("Render files from values", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+			writeFile(d, `
+faa: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bar"))
+
+		})
+
+		It("Render files from values if no default", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, "")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bar"))
+		})
+
+		It("doesn't interpolate if no one provides the values", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foao: "bar"
+`)
+			writeFile(d, `
+faa: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal(""))
+
 		})
 	})
 })

pkg/installer/client/docker.go

@@ -0,0 +1,186 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/docker/go-units"
+	"github.com/pkg/errors"
+
+	imgworker "github.com/mudler/luet/pkg/installer/client/imgworker"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+type DockerClient struct {
+	RepoData RepoData
+}
+
+func NewDockerClient(r RepoData) *DockerClient {
+	return &DockerClient{RepoData: r}
+}
+
+func downloadAndExtractDockerImage(image, dest string) error {
+	temp, err := config.LuetCfg.GetSystem().TempDir("contentstore")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(temp)
+	Debug("Temporary directory", temp)
+	c, err := imgworker.New(temp)
+	if err != nil {
+		return errors.Wrapf(err, "failed creating client")
+	}
+	defer c.Close()
+
+	// FROM Slightly adapted from genuinetools/img https://github.com/genuinetools/img/blob/54d0ca981c1260546d43961a538550eef55c87cf/pull.go
+	Debug("Pulling image", image)
+	listedImage, err := c.Pull(image)
+	if err != nil {
+		return errors.Wrapf(err, "failed listing images")
+
+	}
+	Debug("Pulled:", listedImage.Target.Digest)
+	Debug("Size:", units.BytesSize(float64(listedImage.ContentSize)))
+	Debug("Unpacking", image, "to", dest)
+	os.RemoveAll(dest)
+	return c.Unpack(image, dest)
+}
+
+func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+	//var u *url.URL = nil
+	var err error
+	var temp string
+	var resultingArtifact compiler.Artifact
+	artifactName := path.Base(artifact.GetPath())
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
+	Debug("Cache file", cacheFile)
+	if err := helpers.EnsureDir(cacheFile); err != nil {
+		return nil, errors.Wrapf(err, "could not create cache folder %s for %s", config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), cacheFile)
+	}
+	ok := false
+
+	// TODO:
+	// Files are in URI/packagename:version (GetPackageImageName() method)
+	// use downloadAndExtract .. and egenrate an archive to consume. Checksum should be already checked while downloading the image
+	// with the above functions, because Docker images already contain such metadata
+	// - Check how verification is done when calling DownloadArtifact outside, similarly we need to check DownloadFile, and how verification
+	// is done in such cases (see repository.go)
+
+	// Check if file is already in cache
+	if helpers.Exists(cacheFile) {
+		Debug("Cache hit for artifact", artifactName)
+		resultingArtifact = artifact
+		resultingArtifact.SetPath(cacheFile)
+		resultingArtifact.SetChecksums(compiler.Checksums{})
+	} else {
+
+		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
+		if err != nil {
+			return nil, err
+		}
+		defer os.RemoveAll(temp)
+
+		for _, uri := range c.RepoData.Urls {
+			Debug("Downloading artifact", artifactName, "from", uri)
+
+			imageName := fmt.Sprintf("%s:%s", uri, artifact.GetCompileSpec().GetPackage().GetFingerPrint())
+
+			// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
+			err = downloadAndExtractDockerImage(imageName, temp)
+			if err != nil {
+				Debug("Failed download of image", imageName)
+				continue
+			}
+			Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile)
+
+			newart := artifact
+			// We discard checksum, that are checked while during pull and unpack
+			newart.SetChecksums(compiler.Checksums{})
+			newart.SetPath(cacheFile)                    // First set to cache file
+			newart.SetPath(newart.GetUncompressedName()) // Calculate the real path from cacheFile
+			err = newart.Compress(temp, 1)
+			if err != nil {
+				Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))
+				continue
+			}
+			resultingArtifact = newart
+
+			ok = true
+			break
+		}
+
+		if !ok {
+			return nil, err
+		}
+	}
+
+	return resultingArtifact, nil
+}
+
+func (c *DockerClient) DownloadFile(name string) (string, error) {
+	var file *os.File = nil
+	var err error
+	var temp string
+
+	// Files should be in URI/repository:<file>
+	ok := false
+
+	temp, err = config.LuetCfg.GetSystem().TempDir("tree")
+	if err != nil {
+		return "", err
+	}
+
+	for _, uri := range c.RepoData.Urls {
+
+		file, err = config.LuetCfg.GetSystem().TempFile("DockerClient")
+		if err != nil {
+			continue
+		}
+
+		Debug("Downloading file", name, "from", uri)
+
+		imageName := fmt.Sprintf("%s:%s", uri, name)
+		//imageName := fmt.Sprintf("%s/%s:%s", uri, "repository", name)
+		err = downloadAndExtractDockerImage(imageName, temp)
+		if err != nil {
+			Debug("Failed download of image", imageName)
+			continue
+		}
+
+		Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name())
+		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
+
+	if !ok {
+		return "", err
+	}
+
+	return file.Name(), err
+}

pkg/installer/client/docker_test.go

@@ -0,0 +1,77 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	compiler "github.com/mudler/luet/pkg/compiler"
+	helpers "github.com/mudler/luet/pkg/helpers"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/mudler/luet/pkg/installer/client"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// This test expect that the repository defined in UNIT_TEST_DOCKER_IMAGE is in zstd format.
+// the repository is built by the 01_simple_docker.sh integration test file.
+// This test also require root. At the moment, unpacking docker images with 'img' requires root permission to
+// mount/unmount layers.
+var _ = Describe("Docker client", func() {
+	Context("With repository", func() {
+		repoImage := os.Getenv("UNIT_TEST_DOCKER_IMAGE")
+		var repoURL []string
+		var c *DockerClient
+		BeforeEach(func() {
+			if repoImage == "" {
+				Skip("UNIT_TEST_DOCKER_IMAGE not specified")
+			}
+			repoURL = []string{repoImage}
+			c = NewDockerClient(RepoData{Urls: repoURL})
+		})
+
+		It("Downloads single files", func() {
+			f, err := c.DownloadFile("repository.yaml")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(f)).To(ContainSubstring("Test Repo"))
+			os.RemoveAll(f)
+		})
+
+		It("Downloads artifacts", func() {
+			f, err := c.DownloadArtifact(&compiler.PackageArtifact{
+				Path: "test.tar",
+				CompileSpec: &compiler.LuetCompilationSpec{
+					Package: &pkg.DefaultPackage{
+						Name:     "c",
+						Category: "test",
+						Version:  "1.0",
+					},
+				},
+			})
+			Expect(err).ToNot(HaveOccurred())
+			tmpdir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			Expect(f.Unpack(tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Read(filepath.Join(tmpdir, "c"))).To(Equal("c\n"))
+			Expect(helpers.Read(filepath.Join(tmpdir, "cd"))).To(Equal("c\n"))
+			os.RemoveAll(f.GetPath())
+		})
+	})
+})

pkg/installer/client/http.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"time"
 
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -30,6 +31,8 @@ import (
 	"github.com/mudler/luet/pkg/helpers"
 
 	"github.com/cavaliercoder/grab"
+
+	"github.com/schollz/progressbar/v3"
 )
 
 type HttpClient struct {
@@ -75,7 +78,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 
 	// Check if file is already in cache
 	if helpers.Exists(cacheFile) {
-		Info("Use artifact", artifactName, "from cache.")
+		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 
 		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
@@ -87,7 +90,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 		client := grab.NewClient()
 
 		for _, uri := range c.RepoData.Urls {
-			Info("Downloading artifact", artifactName, "from", uri)
+			Debug("Downloading artifact", artifactName, "from", uri)
 
 			u, err = url.Parse(uri)
 			if err != nil {
@@ -101,20 +104,61 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 			}
 
 			resp := client.Do(req)
+
+			bar := progressbar.NewOptions64(
+				resp.Size(),
+				progressbar.OptionSetDescription(
+					fmt.Sprintf("[cyan] %s - [reset]",
+						filepath.Base(resp.Request.HTTPRequest.URL.RequestURI()))),
+				progressbar.OptionSetRenderBlankState(true),
+				progressbar.OptionEnableColorCodes(config.LuetCfg.GetLogging().Color),
+				progressbar.OptionClearOnFinish(),
+				progressbar.OptionShowBytes(true),
+				progressbar.OptionShowCount(),
+				progressbar.OptionSetPredictTime(true),
+				progressbar.OptionFullWidth(),
+				progressbar.OptionSetTheme(progressbar.Theme{
+					Saucer:        "[white]=[reset]",
+					SaucerHead:    "[white]>[reset]",
+					SaucerPadding: " ",
+					BarStart:      "[",
+					BarEnd:        "]",
+				}))
+
+			bar.Reset()
+			// start download loop
+			t := time.NewTicker(500 * time.Millisecond)
+			defer t.Stop()
+
+		download_loop:
+
+			for {
+				select {
+				case <-t.C:
+					bar.Set64(resp.BytesComplete())
+
+				case <-resp.Done:
+					// download is complete
+					break download_loop
+				}
+			}
+
 			if err = resp.Err(); err != nil {
 				continue
 			}
 
-			Info("Downloaded", artifactName, "of",
-				fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
-				fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
-
-			Debug("Copying file ", filepath.Join(temp, artifactName), "to", cacheFile)
-			err = helpers.CopyFile(filepath.Join(temp, artifactName), cacheFile)
 			if err != nil {
 				continue
 			}
 
+			Info("\nDownloaded", artifactName, "of",
+				fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
+				fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
+
+			Debug("\nCopying file ", filepath.Join(temp, artifactName), "to", cacheFile)
+			err = helpers.CopyFile(filepath.Join(temp, artifactName), cacheFile)
+
+			bar.Finish()
 			ok = true
 			break
 		}
@@ -158,7 +202,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 		}
 		u.Path = path.Join(u.Path, name)
 
-		Info("Downloading", u.String())
+		Debug("Downloading", u.String())
 
 		req, err = c.PrepareReq(temp, u.String())
 		if err != nil {

pkg/installer/client/imgworker/client.go

@@ -0,0 +1,78 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/containerd/namespaces"
+	"github.com/genuinetools/img/types"
+	"github.com/moby/buildkit/control"
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/util/appcontext"
+	"github.com/moby/buildkit/worker/base"
+	"github.com/pkg/errors"
+)
+
+// Client holds the information for the client we will use for communicating
+// with the buildkit controller.
+type Client struct {
+	backend   string
+	localDirs map[string]string
+	root      string
+
+	sessionManager *session.Manager
+	controller     *control.Controller
+	opts           *base.WorkerOpt
+
+	sess *session.Session
+	ctx  context.Context
+}
+
+// New returns a new client for communicating with the buildkit controller.
+func New(root string) (*Client, error) {
+	// Native backend is fine, our images have just one layer. No need to depend on anything
+	backend := types.NativeBackend
+
+	// Create the root/
+	root = filepath.Join(root, "runc", backend)
+	if err := os.MkdirAll(root, 0700); err != nil {
+		return nil, err
+	}
+	c := &Client{
+		backend:   types.NativeBackend,
+		root:      root,
+		localDirs: nil,
+	}
+
+	if err := c.prepare(); err != nil {
+		return nil, errors.Wrapf(err, "failed preparing client")
+	}
+
+	// Create the start of the client.
+	return c, nil
+}
+
+func (c *Client) Close() {
+	c.sess.Close()
+}
+
+func (c *Client) prepare() error {
+	ctx := appcontext.Context()
+	sess, sessDialer, err := c.Session(ctx)
+	if err != nil {
+		return errors.Wrapf(err, "failed creating Session")
+	}
+	ctx = session.NewContext(ctx, sess.ID())
+	ctx = namespaces.WithNamespace(ctx, "buildkit")
+
+	c.ctx = ctx
+	c.sess = sess
+
+	go func() {
+		sess.Run(ctx, sessDialer)
+	}()
+	return nil
+}

pkg/installer/client/imgworker/pull.go

@@ -0,0 +1,129 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"fmt"
+
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
+	"github.com/docker/distribution/reference"
+	"github.com/moby/buildkit/cache"
+	"github.com/moby/buildkit/exporter"
+	imageexporter "github.com/moby/buildkit/exporter/containerimage"
+	"github.com/moby/buildkit/source"
+	"github.com/moby/buildkit/source/containerimage"
+)
+
+// ListedImage represents an image structure returuned from ListImages.
+// It extends containerd/images.Image with extra fields.
+type ListedImage struct {
+	images.Image
+	ContentSize int64
+}
+
+// Pull retrieves an image from a remote registry.
+func (c *Client) Pull(image string) (*ListedImage, error) {
+
+	ctx := c.ctx
+
+	sm, err := c.getSessionManager()
+	if err != nil {
+		return nil, err
+	}
+	// Parse the image name and tag.
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return nil, fmt.Errorf("parsing image name %q failed: %v", image, err)
+	}
+	// Add the latest lag if they did not provide one.
+	named = reference.TagNameOnly(named)
+	image = named.String()
+
+	// Get the identifier for the image.
+	identifier, err := source.NewImageIdentifier(image)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the worker opts.
+	opt, err := c.createWorkerOpt()
+	if err != nil {
+		return nil, fmt.Errorf("creating worker opt failed: %v", err)
+	}
+
+	cm, err := cache.NewManager(cache.ManagerOpt{
+		Snapshotter:    opt.Snapshotter,
+		MetadataStore:  opt.MetadataStore,
+		ContentStore:   opt.ContentStore,
+		LeaseManager:   opt.LeaseManager,
+		GarbageCollect: opt.GarbageCollect,
+		Applier:        opt.Applier,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the source for the pull.
+	srcOpt := containerimage.SourceOpt{
+		Snapshotter:   opt.Snapshotter,
+		ContentStore:  opt.ContentStore,
+		Applier:       opt.Applier,
+		CacheAccessor: cm,
+		ImageStore:    opt.ImageStore,
+		RegistryHosts: opt.RegistryHosts,
+		LeaseManager:  opt.LeaseManager,
+	}
+	src, err := containerimage.NewSource(srcOpt)
+	if err != nil {
+		return nil, err
+	}
+	s, err := src.Resolve(ctx, identifier, sm)
+	if err != nil {
+		return nil, err
+	}
+	ref, err := s.Snapshot(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the exporter for the pull.
+	iw, err := imageexporter.NewImageWriter(imageexporter.WriterOpt{
+		Snapshotter:  opt.Snapshotter,
+		ContentStore: opt.ContentStore,
+		Differ:       opt.Differ,
+	})
+	if err != nil {
+		return nil, err
+	}
+	expOpt := imageexporter.Opt{
+		SessionManager: sm,
+		ImageWriter:    iw,
+		Images:         opt.ImageStore,
+		RegistryHosts:  opt.RegistryHosts,
+		LeaseManager:   opt.LeaseManager,
+	}
+	exp, err := imageexporter.New(expOpt)
+	if err != nil {
+		return nil, err
+	}
+	e, err := exp.Resolve(ctx, map[string]string{"name": image})
+	if err != nil {
+		return nil, err
+	}
+	if _, err := e.Export(ctx, exporter.Source{Ref: ref}); err != nil {
+		return nil, err
+	}
+
+	// Get the image.
+	img, err := opt.ImageStore.Get(ctx, image)
+	if err != nil {
+		return nil, fmt.Errorf("getting image %s from image store failed: %v", image, err)
+	}
+	size, err := img.Size(ctx, opt.ContentStore, platforms.Default())
+	if err != nil {
+		return nil, fmt.Errorf("calculating size of image %s failed: %v", img.Name, err)
+	}
+
+	return &ListedImage{Image: img, ContentSize: size}, nil
+}

pkg/installer/client/imgworker/session.go

@@ -0,0 +1,51 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"context"
+	"os"
+
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/session/auth/authprovider"
+	"github.com/moby/buildkit/session/filesync"
+	"github.com/moby/buildkit/session/testutil"
+	"github.com/pkg/errors"
+)
+
+func (c *Client) getSessionManager() (*session.Manager, error) {
+	if c.sessionManager == nil {
+		var err error
+		c.sessionManager, err = session.NewManager()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return c.sessionManager, nil
+}
+
+// Session creates the session manager and returns the session and it's
+// dialer.
+func (c *Client) Session(ctx context.Context) (*session.Session, session.Dialer, error) {
+	m, err := c.getSessionManager()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to create session manager")
+	}
+	sessionName := "img"
+	s, err := session.NewSession(ctx, sessionName, "")
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to create session")
+	}
+	syncedDirs := make([]filesync.SyncedDir, 0, len(c.localDirs))
+	for name, d := range c.localDirs {
+		syncedDirs = append(syncedDirs, filesync.SyncedDir{Name: name, Dir: d})
+	}
+	s.Allow(filesync.NewFSSyncProvider(syncedDirs))
+	s.Allow(authprovider.NewDockerAuthProvider(os.Stderr))
+	return s, sessionDialer(s, m), err
+}
+
+func sessionDialer(s *session.Session, m *session.Manager) session.Dialer {
+	// FIXME: rename testutil
+	return session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn)))
+}

pkg/installer/client/imgworker/unpack.go

@@ -0,0 +1,82 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+)
+
+// TODO: this requires root permissions to mount/unmount layers, althrought it shouldn't be required.
+// See how backends are unpacking images without asking for root permissions.
+
+// Unpack exports an image to a rootfs destination directory.
+func (c *Client) Unpack(image, dest string) error {
+
+	ctx := c.ctx
+	if len(dest) < 1 {
+		return errors.New("destination directory for rootfs cannot be empty")
+	}
+
+	if _, err := os.Stat(dest); err == nil {
+		return fmt.Errorf("destination directory already exists: %s", dest)
+	}
+
+	// Parse the image name and tag.
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return fmt.Errorf("parsing image name %q failed: %v", image, err)
+	}
+	// Add the latest lag if they did not provide one.
+	named = reference.TagNameOnly(named)
+	image = named.String()
+
+	// Create the worker opts.
+	opt, err := c.createWorkerOpt()
+	if err != nil {
+		return fmt.Errorf("creating worker opt failed: %v", err)
+	}
+
+	if opt.ImageStore == nil {
+		return errors.New("image store is nil")
+	}
+
+	img, err := opt.ImageStore.Get(ctx, image)
+	if err != nil {
+		return fmt.Errorf("getting image %s from image store failed: %v", image, err)
+	}
+
+	manifest, err := images.Manifest(ctx, opt.ContentStore, img.Target, platforms.Default())
+	if err != nil {
+		return fmt.Errorf("getting image manifest failed: %v", err)
+	}
+
+	for _, desc := range manifest.Layers {
+		logrus.Debugf("Unpacking layer %s", desc.Digest.String())
+
+		// Read the blob from the content store.
+		layer, err := opt.ContentStore.ReaderAt(ctx, desc)
+		if err != nil {
+			return fmt.Errorf("getting reader for digest %s failed: %v", desc.Digest.String(), err)
+		}
+
+		// Unpack the tarfile to the rootfs path.
+		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
+		if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{
+			NoLchown:        true,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+		}); err != nil {
+			return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err)
+		}
+	}
+
+	return nil
+}

pkg/installer/client/imgworker/workeropts.go

@@ -0,0 +1,106 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/containerd/containerd/content/local"
+	"github.com/containerd/containerd/diff/apply"
+	"github.com/containerd/containerd/diff/walking"
+	ctdmetadata "github.com/containerd/containerd/metadata"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/remotes/docker"
+	ctdsnapshot "github.com/containerd/containerd/snapshots"
+	"github.com/containerd/containerd/snapshots/native"
+	"github.com/moby/buildkit/cache/metadata"
+	containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
+	"github.com/moby/buildkit/util/binfmt_misc"
+	"github.com/moby/buildkit/util/leaseutil"
+	"github.com/moby/buildkit/worker/base"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	bolt "go.etcd.io/bbolt"
+)
+
+// createWorkerOpt creates a base.WorkerOpt to be used for a new worker.
+func (c *Client) createWorkerOpt() (opt base.WorkerOpt, err error) {
+
+	if c.opts != nil {
+		return *c.opts, nil
+	}
+
+	// Create the metadata store.
+	md, err := metadata.NewStore(filepath.Join(c.root, "metadata.db"))
+	if err != nil {
+		return opt, err
+	}
+
+	snapshotRoot := filepath.Join(c.root, "snapshots")
+
+	s, err := native.NewSnapshotter(snapshotRoot)
+	if err != nil {
+		return opt, fmt.Errorf("creating %s snapshotter failed: %v", c.backend, err)
+	}
+
+	// Create the content store locally.
+	contentStore, err := local.NewStore(filepath.Join(c.root, "content"))
+	if err != nil {
+		return opt, err
+	}
+
+	// Open the bolt database for metadata.
+	db, err := bolt.Open(filepath.Join(c.root, "containerdmeta.db"), 0644, nil)
+	if err != nil {
+		return opt, err
+	}
+
+	// Create the new database for metadata.
+	mdb := ctdmetadata.NewDB(db, contentStore, map[string]ctdsnapshot.Snapshotter{
+		c.backend: s,
+	})
+	if err := mdb.Init(context.TODO()); err != nil {
+		return opt, err
+	}
+
+	// Create the image store.
+	imageStore := ctdmetadata.NewImageStore(mdb)
+
+	contentStore = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit")
+
+	id, err := base.ID(c.root)
+	if err != nil {
+		return opt, err
+	}
+
+	xlabels := base.Labels("oci", c.backend)
+
+	var supportedPlatforms []specs.Platform
+	for _, p := range binfmt_misc.SupportedPlatforms(false) {
+		parsed, err := platforms.Parse(p)
+		if err != nil {
+			return opt, err
+		}
+		supportedPlatforms = append(supportedPlatforms, platforms.Normalize(parsed))
+	}
+
+	opt = base.WorkerOpt{
+		ID:             id,
+		Labels:         xlabels,
+		MetadataStore:  md,
+		Snapshotter:    containerdsnapshot.NewSnapshotter(c.backend, mdb.Snapshotter(c.backend), "buildkit", nil),
+		ContentStore:   contentStore,
+		Applier:        apply.NewFileSystemApplier(contentStore),
+		Differ:         walking.NewWalkingDiff(contentStore),
+		ImageStore:     imageStore,
+		Platforms:      supportedPlatforms,
+		RegistryHosts:  docker.ConfigureDefaultRegistries(),
+		LeaseManager:   leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"),
+		GarbageCollect: mdb.GarbageCollect,
+	}
+
+	c.opts = &opt
+
+	return opt, err
+}

pkg/installer/client/local.go

@@ -38,15 +38,26 @@ func NewLocalClient(r RepoData) *LocalClient {
 func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
 	var err error
 
+	rootfs := ""
 	artifactName := path.Base(artifact.GetPath())
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 
+	if !config.LuetCfg.ConfigFromHost {
+		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Check if file is already in cache
 	if helpers.Exists(cacheFile) {
-		Info("Use artifact", artifactName, "from cache.")
+		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 		ok := false
 		for _, uri := range c.RepoData.Urls {
+
+			uri = filepath.Join(rootfs, uri)
+
 			Info("Downloading artifact", artifactName, "from", uri)
 
 			//defer os.Remove(file.Name())
@@ -72,8 +83,20 @@ func (c *LocalClient) DownloadFile(name string) (string, error) {
 	var err error
 	var file *os.File = nil
 
+	rootfs := ""
+
+	if !config.LuetCfg.ConfigFromHost {
+		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return "", err
+		}
+	}
+
 	ok := false
 	for _, uri := range c.RepoData.Urls {
+
+		uri = filepath.Join(rootfs, uri)
+
 		Info("Downloading file", name, "from", uri)
 		file, err = config.LuetCfg.GetSystem().TempFile("localclient")
 		if err != nil {

pkg/installer/config_protect.go

@@ -19,6 +19,7 @@ package installer
 import (
 	"io/ioutil"
 	"path"
+	"path/filepath"
 	"regexp"
 
 	"github.com/ghodss/yaml"
@@ -29,8 +30,21 @@ import (
 
 func LoadConfigProtectConfs(c *LuetConfig) error {
 	var regexConfs = regexp.MustCompile(`.yml$`)
+	var err error
+
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
 
 	for _, cdir := range c.ConfigProtectConfDir {
+		cdir = filepath.Join(rootfs, cdir)
+
 		Debug("Parsing Config Protect Directory", cdir, "...")
 
 		files, err := ioutil.ReadDir(cdir)

pkg/installer/finalizer.go

@@ -47,7 +47,7 @@ func (f *LuetFinalizer) RunInstall(s *System) error {
 
 	for _, c := range f.Install {
 		toRun := append(args, c)
-		Info("Executing finalizer on ", s.Target, cmd, toRun)
+		Info(":shell: Executing finalizer on ", s.Target, cmd, toRun)
 		if s.Target == "/" {
 			cmd := exec.Command(cmd, toRun...)
 			stdoutStderr, err := cmd.CombinedOutput()

pkg/installer/installer.go

@@ -24,13 +24,14 @@ import (
 	"strings"
 	"sync"
 
+	. "github.com/logrusorgru/aurora"
+	"github.com/mudler/luet/pkg/bus"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
-	"github.com/mudler/luet/pkg/tree"
 
 	"github.com/pkg/errors"
 )
@@ -45,6 +46,7 @@ type LuetInstallerOptions struct {
 	FullUninstall, FullCleanUninstall                              bool
 	CheckConflicts                                                 bool
 	SolverUpgrade, RemoveUnavailableOnUpgrade, UpgradeNewRevisions bool
+	Ask                                                            bool
 }
 
 type LuetInstaller struct {
@@ -63,86 +65,125 @@ func NewLuetInstaller(opts LuetInstallerOptions) Installer {
 	return &LuetInstaller{Options: opts}
 }
 
-func (l *LuetInstaller) Upgrade(s *System) error {
-	syncedRepos, err := l.SyncRepositories(true)
-	if err != nil {
-		return err
-	}
-
-	Info(":thinking: Computing upgrade, please hang tight")
+// computeUpgrade returns the packages to be uninstalled and installed in a system to perform an upgrade
+// based on the system repositories
+func (l *LuetInstaller) computeUpgrade(syncedRepos Repositories, s *System) (pkg.Packages, pkg.Packages, error) {
+	toInstall := pkg.Packages{}
+	var uninstall pkg.Packages
+	var err error
 	// First match packages against repositories by priority
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	// compute a "big" world
-	solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
-	var uninstall pkg.Packages
+	solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 	var solution solver.PackagesAssertions
 
 	if l.Options.SolverUpgrade {
 		uninstall, solution, err = solv.UpgradeUniverse(l.Options.RemoveUnavailableOnUpgrade)
 		if err != nil {
-			return errors.Wrap(err, "Failed solving solution for upgrade")
+			return uninstall, toInstall, errors.Wrap(err, "Failed solving solution for upgrade")
 		}
 	} else {
-		uninstall, solution, err = solv.Upgrade(!l.Options.FullUninstall, l.Options.NoDeps)
+		uninstall, solution, err = solv.Upgrade(l.Options.FullUninstall, true)
 		if err != nil {
-			return errors.Wrap(err, "Failed solving solution for upgrade")
+			return uninstall, toInstall, errors.Wrap(err, "Failed solving solution for upgrade")
 		}
 	}
 
-	if len(uninstall) > 0 {
-		Info("Packages marked for uninstall:")
-	}
-
-	for _, p := range uninstall {
-		Info(fmt.Sprintf("- %s", p.HumanReadableString()))
-	}
-
-	if len(solution) > 0 {
-		Info("Packages marked for upgrade:")
-	}
-
-	toInstall := pkg.Packages{}
 	for _, assertion := range solution {
 		// Be sure to filter from solutions packages already installed in the system
 		if _, err := s.Database.FindPackage(assertion.Package); err != nil && assertion.Value {
-			Info(fmt.Sprintf("- %s", assertion.Package.HumanReadableString()))
 			toInstall = append(toInstall, assertion.Package)
 		}
 	}
 
 	if l.Options.UpgradeNewRevisions {
-		Info("Checking packages with new revisions available")
 		for _, p := range s.Database.World() {
 			matches := syncedRepos.PackageMatches(pkg.Packages{p})
 			if len(matches) == 0 {
 				// Package missing. the user should run luet upgrade --universe
-				Info("Installed packages seems to be missing from remote repositories.")
-				Info("It is suggested to run 'luet upgrade --universe'")
 				continue
 			}
 			for _, artefact := range matches[0].Repo.GetIndex() {
 				if artefact.GetCompileSpec().GetPackage() == nil {
-					return errors.New("Package in compilespec empty")
+					return uninstall, toInstall, errors.New("Package in compilespec empty")
 
 				}
 				if artefact.GetCompileSpec().GetPackage().Matches(p) && artefact.GetCompileSpec().GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
 					toInstall = append(toInstall, matches[0].Package).Unique()
 					uninstall = append(uninstall, p).Unique()
-					Info(
-						fmt.Sprintf("- %s ( %s vs %s ) repo: %s (date: %s)",
-							p.HumanReadableString(),
-							artefact.GetCompileSpec().GetPackage().GetBuildTimestamp(),
-							p.GetBuildTimestamp(),
-							matches[0].Repo.GetName(),
-							matches[0].Repo.GetLastUpdate(),
-						))
 				}
 			}
 		}
 	}
 
-	return l.swap(syncedRepos, uninstall, toInstall, s)
+	return uninstall, toInstall, nil
+}
+
+func packsToList(p pkg.Packages) string {
+	var packs []string
+
+	for _, pp := range p {
+		packs = append(packs, pp.HumanReadableString())
+	}
+	return strings.Join(packs, " ")
+}
+
+func matchesToList(artefacts map[string]ArtifactMatch) string {
+	var packs []string
+
+	for fingerprint, match := range artefacts {
+		packs = append(packs, fmt.Sprintf("%s (%s)", fingerprint, match.Repository.GetName()))
+	}
+	return strings.Join(packs, " ")
+}
+
+// Upgrade upgrades a System based on the Installer options. Returns error in case of failure
+func (l *LuetInstaller) Upgrade(s *System) error {
+
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+
+	Info(":thinking: Computing upgrade, please hang tight... :zzz:")
+	if l.Options.UpgradeNewRevisions {
+		Info(":memo: note: will consider new build revisions while upgrading")
+	}
+
+	Spinner(32)
+	uninstall, toInstall, err := l.computeUpgrade(syncedRepos, s)
+	if err != nil {
+		return errors.Wrap(err, "failed computing upgrade")
+	}
+	SpinnerStop()
+
+	if len(uninstall) > 0 {
+		Info(":recycle: Packages that are going to be removed from the system:\n ", Yellow(packsToList(uninstall)).BgBlack().String())
+	}
+
+	if len(toInstall) > 0 {
+		Info(":zap:Packages that are going to be installed in the system:\n ", Green(packsToList(toInstall)).BgBlack().String())
+	}
+
+	if len(toInstall) == 0 && len(uninstall) == 0 {
+		Info("Nothing to do")
+		return nil
+	}
+
+	if l.Options.Ask {
+		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
+		if Ask() {
+			l.Options.Ask = false // Don't prompt anymore
+			return l.swap(syncedRepos, uninstall, toInstall, s, true)
+		} else {
+			return errors.New("Aborted by user")
+		}
+	}
+
+	Spinner(32)
+	defer SpinnerStop()
+	return l.swap(syncedRepos, uninstall, toInstall, s, true)
 }
 
 func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
@@ -172,19 +213,59 @@ func (l *LuetInstaller) Swap(toRemove pkg.Packages, toInstall pkg.Packages, s *S
 	if err != nil {
 		return err
 	}
-	return l.swap(syncedRepos, toRemove, toInstall, s)
+
+	toRemoveFinal := pkg.Packages{}
+	for _, p := range toRemove {
+		packs, _ := s.Database.FindPackages(p)
+		if len(packs) == 0 {
+			return errors.New("Package " + p.HumanReadableString() + " not found in the system")
+		}
+		for _, pp := range packs {
+			toRemoveFinal = append(toRemoveFinal, pp)
+		}
+	}
+
+	return l.swap(syncedRepos, toRemoveFinal, toInstall, s, false)
 }
 
-func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) error {
-	// First match packages against repositories by priority
+func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
+
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
+
 	toInstall = syncedRepos.ResolveSelectors(toInstall)
 
-	if err := l.download(syncedRepos, toInstall); err != nil {
-		return errors.Wrap(err, "Pre-downloading packages")
+	// First check what would have been done
+	installedtmp, err := s.Database.Copy()
+	if err != nil {
+		return nil, nil, nil, nil, errors.Wrap(err, "Failed create temporary in-memory db")
 	}
 
+	systemAfterChanges := &System{Database: installedtmp}
+
+	packs, err := l.computeUninstall(systemAfterChanges, toRemove...)
+	if err != nil && !l.Options.Force {
+		Error("Failed computing uninstall for ", packsToList(toRemove))
+		return nil, nil, nil, nil, errors.Wrap(err, "computing uninstall "+packsToList(toRemove))
+	}
+	for _, p := range packs {
+		err = systemAfterChanges.Database.RemovePackage(p)
+		if err != nil {
+			return nil, nil, nil, nil, errors.Wrap(err, "Failed removing package from database")
+		}
+	}
+
+	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, toInstall, systemAfterChanges)
+	for _, p := range toInstall {
+		assertions = append(assertions, solver.PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
+	}
+	return match, packages, assertions, allRepos, err
+}
+
+func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System, forceNodeps bool) error {
+	forced := l.Options.Force
+	nodeps := l.Options.NoDeps
+
 	// We don't want any conflict with the installed to raise during the upgrade.
 	// In this way we both force uninstalls and we avoid to check with conflicts
 	// against the current system state which is pending to deletion
@@ -192,23 +273,47 @@ func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, to
 	// if the old A results installed in the system. This is due to the fact that
 	// now the solver enforces the constraints and explictly denies two packages
 	// of the same version installed.
-	forced := l.Options.Force
-
 	l.Options.Force = true
+	if forceNodeps {
+		l.Options.NoDeps = true
+	}
 
-	for _, u := range toRemove {
-		Info(":package:", u.HumanReadableString(), "Marked for deletion")
+	match, packages, assertions, allRepos, err := l.computeSwap(syncedRepos, toRemove, toInstall, s)
+	if err != nil {
+		return errors.Wrap(err, "failed computing package replacement")
+	}
 
-		err := l.Uninstall(u, s)
-		if err != nil && !l.Options.Force {
-			Error("Failed uninstall for ", u.HumanReadableString())
-			return errors.Wrap(err, "uninstalling "+u.HumanReadableString())
+	if l.Options.Ask {
+		if len(toRemove) > 0 {
+			Info(":recycle: Packages that are going to be removed from the system:\n ", Yellow(packsToList(toRemove)).BgBlack().String())
 		}
 
-	}
-	l.Options.Force = forced
+		if len(match) > 0 {
+			Info("Packages that are going to be installed in the system: \n ", Green(matchesToList(match)).BgBlack().String())
+		}
 
-	return l.install(syncedRepos, toInstall, s)
+		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
+		if Ask() {
+			l.Options.Ask = false // Don't prompt anymore
+		} else {
+			return errors.New("Aborted by user")
+		}
+	}
+
+	// First match packages against repositories by priority
+	if err := l.download(syncedRepos, match); err != nil {
+		return errors.Wrap(err, "Pre-downloading packages")
+	}
+
+	err = l.Uninstall(s, toRemove...)
+	if err != nil && !l.Options.Force {
+		Error("Failed uninstall for ", packsToList(toRemove))
+		return errors.Wrap(err, "uninstalling "+packsToList(toRemove))
+	}
+
+	l.Options.Force = forced
+	l.Options.NoDeps = nodeps
+	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
 }
 
 func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
@@ -216,35 +321,54 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 	if err != nil {
 		return err
 	}
-	return l.install(syncedRepos, cp, s)
-}
 
-func (l *LuetInstaller) download(syncedRepos Repositories, cp pkg.Packages) error {
-	toDownload := map[string]ArtifactMatch{}
-
-	// FIXME: This can be optimized. We don't need to re-match this to the repository
-	// But we could just do it once
-
-	// Gathers things to download
-	for _, currentPack := range cp {
-		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
-		if len(matches) == 0 {
-			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
-		}
-	A:
-		for _, artefact := range matches[0].Repo.GetIndex() {
-			if artefact.GetCompileSpec().GetPackage() == nil {
-				return errors.New("Package in compilespec empty")
+	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, cp, s)
+	if err != nil {
+		return err
+	}
 
+	// Check if we have to process something, or return to the user an error
+	if len(match) == 0 {
+		Info("No packages to install")
+		return nil
+	}
+	// Resolvers might decide to remove some packages from being installed
+	if !l.Options.SolverOptions.ResolverIsSet() {
+		for _, p := range cp {
+			found := false
+			vers, _ := s.Database.FindPackageVersions(p) // If was installed, it is found, as it was filtered
+			if len(vers) >= 1 {
+				found = true
+				continue
 			}
-			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
 
-				toDownload[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
+			for _, m := range match {
+				if m.Package.GetName() == p.GetName() {
+					found = true
+				}
+			}
 
-				break A
+			if !found {
+				return fmt.Errorf("Package '%s' not found", p.HumanReadableString())
 			}
 		}
 	}
+	Info("Packages that are going to be installed in the system: \n ", Green(matchesToList(match)).BgBlack().String())
+
+	if l.Options.Ask {
+		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
+		if Ask() {
+			l.Options.Ask = false // Don't prompt anymore
+			return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+		} else {
+			return errors.New("Aborted by user")
+		}
+	}
+	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+}
+
+func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string]ArtifactMatch) error {
+
 	// Download packages into cache in parallel.
 	all := make(chan ArtifactMatch)
 
@@ -287,7 +411,7 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 					if err != nil {
 						return err
 					}
-					Info("Found package:", p.HumanReadableString())
+					Info(":mag: Found package:", p.HumanReadableString())
 					toMerge = append(toMerge, ArtifactMatch{Artifact: artefact, Package: p})
 					break FILES
 				}
@@ -308,23 +432,25 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 			return errors.Wrap(err, "Failed creating package")
 		}
 		s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.GetFiles()})
-		Info("Reclaimed package:", pack.HumanReadableString())
+		Info(":zap:Reclaimed package:", pack.HumanReadableString())
 	}
 	Info("Done!")
 
 	return nil
 }
 
-func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *System) error {
+func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
 	var p pkg.Packages
+	toInstall := map[string]ArtifactMatch{}
+	allRepos := pkg.NewInMemoryDatabase(false)
+	var solution solver.PackagesAssertions
 
 	// Check if the package is installed first
 	for _, pi := range cp {
-
 		vers, _ := s.Database.FindPackageVersions(pi)
 
 		if len(vers) >= 1 {
-			Warning("Filtering out package " + pi.HumanReadableString() + ", it has other versions already installed. Uninstall one of them first ")
+			//	Warning("Filtering out package " + pi.HumanReadableString() + ", it has other versions already installed. Uninstall one of them first ")
 			continue
 			//return errors.New("Package " + pi.GetFingerPrint() + " has other versions already installed. Uninstall one of them first: " + strings.Join(vers, " "))
 
@@ -333,8 +459,7 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 	}
 
 	if len(p) == 0 {
-		Warning("No package to install, bailing out with no errors")
-		return nil
+		return toInstall, p, solution, allRepos, nil
 	}
 	// First get metas from all repos (and decodes trees)
 
@@ -342,80 +467,72 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 	//	matches := syncedRepos.PackageMatches(p)
 
 	// compute a "big" world
-	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	p = syncedRepos.ResolveSelectors(p)
-	toInstall := map[string]ArtifactMatch{}
 	var packagesToInstall pkg.Packages
 	var err error
-	var solution solver.PackagesAssertions
 
 	if !l.Options.NoDeps {
-		solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		solution, err = solv.Install(p)
+		/// TODO: PackageAssertions needs to be a map[fingerprint]pack so lookup is in O(1)
 		if err != nil && !l.Options.Force {
-			return errors.Wrap(err, "Failed solving solution for package")
+			return toInstall, p, solution, allRepos, errors.Wrap(err, "Failed solving solution for package")
 		}
 		// Gathers things to install
 		for _, assertion := range solution {
 			if assertion.Value {
+				if _, err := s.Database.FindPackage(assertion.Package); err == nil {
+					// skip matching if it is installed already
+					continue
+				}
 				packagesToInstall = append(packagesToInstall, assertion.Package)
 			}
 		}
 	} else if !l.Options.OnlyDeps {
 		for _, currentPack := range p {
+			if _, err := s.Database.FindPackage(currentPack); err == nil {
+				// skip matching if it is installed already
+				continue
+			}
 			packagesToInstall = append(packagesToInstall, currentPack)
 		}
 	}
-	Info(":deciduous_tree: Finding packages to install")
 	// Gathers things to install
 	for _, currentPack := range packagesToInstall {
 		// Check if package is already installed.
-		if _, err := s.Database.FindPackage(currentPack); err == nil {
-			// skip matching if it is installed already
-			continue
-		}
+
 		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
 		if len(matches) == 0 {
-			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
+			return toInstall, p, solution, allRepos, errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
 		}
 	A:
 		for _, artefact := range matches[0].Repo.GetIndex() {
 			if artefact.GetCompileSpec().GetPackage() == nil {
-				return errors.New("Package in compilespec empty")
-
+				return toInstall, p, solution, allRepos, errors.New("Package in compilespec empty")
 			}
 			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
 				currentPack.SetBuildTimestamp(artefact.GetCompileSpec().GetPackage().GetBuildTimestamp())
 				// Filter out already installed
 				if _, err := s.Database.FindPackage(currentPack); err != nil {
 					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
-					Info("\t:package:", currentPack.HumanReadableString(), "from repository", matches[0].Repo.GetName())
 				}
 				break A
 			}
 		}
 	}
+	return toInstall, p, solution, allRepos, nil
+}
+
+func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
 	// Install packages into rootfs in parallel.
+	if err := l.download(syncedRepos, toInstall); err != nil {
+		return errors.Wrap(err, "Downloading packages")
+	}
+
 	all := make(chan ArtifactMatch)
 
-	var wg = new(sync.WaitGroup)
-
-	// Download first
-	for i := 0; i < l.Options.Concurrency; i++ {
-		wg.Add(1)
-		go l.downloadWorker(i, wg, all)
-	}
-
-	for _, c := range toInstall {
-		all <- c
-	}
-	close(all)
-	wg.Wait()
-
-	all = make(chan ArtifactMatch)
-
-	wg = new(sync.WaitGroup)
+	wg := new(sync.WaitGroup)
 
 	// Do the real install
 	for i := 0; i < l.Options.Concurrency; i++ {
@@ -435,9 +552,9 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Failed creating package")
 		}
+		bus.Manager.Publish(bus.EventPackageInstall, c)
 	}
-	executedFinalizer := map[string]bool{}
-
+	var toFinalize []pkg.Package
 	if !l.Options.NoDeps {
 		// TODO: Lower those errors as warning
 		for _, w := range p {
@@ -449,36 +566,17 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 		ORDER:
 			for _, ass := range ordered {
 				if ass.Value {
-
 					installed, ok := toInstall[ass.Package.GetFingerPrint()]
 					if !ok {
 						// It was a dep already installed in the system, so we can skip it safely
 						continue ORDER
 					}
-
 					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
 					if err != nil {
 						return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
 					}
-					if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
-						Info("Executing finalizer for " + ass.Package.HumanReadableString())
-						finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
-						if err != nil && !l.Options.Force {
-							return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
-						}
-						if _, exists := executedFinalizer[ass.Package.GetFingerPrint()]; !exists {
-							finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
-							if err != nil && !l.Options.Force {
-								return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
-							}
-							err = finalizer.RunInstall(s)
-							if err != nil && !l.Options.Force {
-								return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
-							}
-							executedFinalizer[ass.Package.GetFingerPrint()] = true
-						}
-					}
 
+					toFinalize = append(toFinalize, treePackage)
 				}
 			}
 
@@ -489,29 +587,11 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 			if err != nil {
 				return errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
 			}
-			if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
-				Info("Executing finalizer for " + c.Package.HumanReadableString())
-				finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
-				if err != nil && !l.Options.Force {
-					return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
-				}
-				if _, exists := executedFinalizer[c.Package.GetFingerPrint()]; !exists {
-					finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
-					if err != nil && !l.Options.Force {
-						return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
-					}
-					err = finalizer.RunInstall(s)
-					if err != nil && !l.Options.Force {
-						return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
-					}
-					executedFinalizer[c.Package.GetFingerPrint()] = true
-				}
-			}
+			toFinalize = append(toFinalize, treePackage)
 		}
 	}
 
-	return nil
-
+	return s.ExecuteFinalizers(toFinalize)
 }
 
 func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) {
@@ -522,7 +602,7 @@ func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, err
 	}
 
 	err = artifact.Verify()
-	if err != nil && !l.Options.Force {
+	if err != nil {
 		return nil, errors.Wrap(err, "Artifact integrity check failure")
 	}
 	return artifact, nil
@@ -556,15 +636,11 @@ func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan Artif
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
 		_, err := l.downloadPackage(p)
-		if err != nil && !l.Options.Force {
-			//TODO: Uninstall, rollback.
-			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
-			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
-		}
-		if err == nil {
-			Info(":package: ", p.Package.HumanReadableString(), "downloaded")
-		} else if err != nil && l.Options.Force {
-			Info(":package: ", p.Package.HumanReadableString(), "downloaded with failures (force download)")
+		if err != nil {
+			Fatal("Failed downloading package "+p.Package.GetName(), err.Error())
+			return errors.Wrap(err, "Failed downloading package "+p.Package.GetName())
+		} else {
+			Info(":package: Package ", p.Package.HumanReadableString(), "downloaded")
 		}
 	}
 
@@ -583,9 +659,9 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
 		}
 		if err == nil {
-			Info(":package: ", p.Package.HumanReadableString(), "installed")
+			Info(":package: Package ", p.Package.HumanReadableString(), "installed")
 		} else if err != nil && l.Options.Force {
-			Info(":package: ", p.Package.HumanReadableString(), "installed with failures (force install)")
+			Info(":package: Package ", p.Package.HumanReadableString(), "installed with failures (forced install)")
 		}
 	}
 
@@ -593,16 +669,39 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 }
 
 func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
+	var cp *config.ConfigProtect
+	annotationDir := ""
+
 	files, err := s.Database.GetPackageFiles(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed getting installed files")
 	}
 
-	// Remove from target
-	for _, f := range files {
-		target := filepath.Join(s.Target, f)
-		Debug("Removing", target)
+	if !config.LuetCfg.ConfigProtectSkip {
 
+		if p.HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := p.GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
+			}
+		}
+
+		cp = config.NewConfigProtect(annotationDir)
+		cp.Map(files)
+	}
+
+	toRemove, notPresent := helpers.OrderFiles(s.Target, files)
+
+	// Remove from target
+	for _, f := range toRemove {
+		target := filepath.Join(s.Target, f)
+
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Preserving protected file:", f)
+			continue
+		}
+
+		Debug("Removing", target)
 		if l.Options.PreserveSystemEssentialData &&
 			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) ||
 			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath()) {
@@ -610,11 +709,41 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 			continue
 		}
 
-		err := os.Remove(target)
+		fi, err := os.Lstat(target)
 		if err != nil {
-			Warning("Failed removing file (not present in the system target ?)", target)
+			Warning("File not found (it was before?) ", err.Error())
+			continue
+		}
+		switch mode := fi.Mode(); {
+		case mode.IsDir():
+			files, err := ioutil.ReadDir(target)
+			if err != nil {
+				Warning("Failed reading folder", target, err.Error())
+			}
+			if len(files) != 0 {
+				Debug("Preserving not-empty folder", target)
+				continue
+			}
+		}
+
+		if err = os.Remove(target); err != nil {
+			Warning("Failed removing file (maybe not present in the system target anymore ?)", target, err.Error())
 		}
 	}
+
+	for _, f := range notPresent {
+		target := filepath.Join(s.Target, f)
+
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Preserving protected file:", f)
+			continue
+		}
+
+		if err = os.Remove(target); err != nil {
+			Debug("Failed removing file (not present in the system target)", target, err.Error())
+		}
+	}
+
 	err = s.Database.RemovePackageFiles(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed removing package files from database")
@@ -624,16 +753,15 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		return errors.Wrap(err, "Failed removing package from database")
 	}
 
-	Info(p.GetFingerPrint(), "Removed")
+	bus.Manager.Publish(bus.EventPackageUnInstall, p)
+
+	Info(":recycle: ", p.GetFingerPrint(), "Removed :heavy_check_mark:")
 	return nil
 }
 
-func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
-	Spinner(32)
-	defer SpinnerStop()
-
-	Info("Uninstalling :package:", p.HumanReadableString(), "hang tight")
+func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.Packages, error) {
 
+	var toUninstall pkg.Packages
 	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) TODO - mark the uninstallation in db
 	// Get installed definition
 	checkConflicts := l.Options.CheckConflicts
@@ -645,49 +773,78 @@ func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
 
 	// Create a temporary DB with the installed packages
 	// so the solver is much faster finding the deptree
-	installedtmp := pkg.NewInMemoryDatabase(false)
-
-	for _, i := range s.Database.World() {
-		_, err := installedtmp.CreatePackage(i)
-		if err != nil {
-			return errors.Wrap(err, "Failed create temporary in-memory db")
-		}
+	// First check what would have been done
+	installedtmp, err := s.Database.Copy()
+	if err != nil {
+		return toUninstall, errors.Wrap(err, "Failed create temporary in-memory db")
 	}
 
 	if !l.Options.NoDeps {
-		Info("Finding :package:", p.HumanReadableString(), "dependency graph :deciduous_tree:")
-		solv := solver.NewResolver(installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		var solution pkg.Packages
 		var err error
 		if l.Options.FullCleanUninstall {
-			solution, err = solv.UninstallUniverse(pkg.Packages{p})
+			solution, err = solv.UninstallUniverse(packs)
 			if err != nil {
-				return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+				return toUninstall, errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
 			}
 		} else {
-			solution, err = solv.Uninstall(p, checkConflicts, full)
+			solution, err = solv.Uninstall(checkConflicts, full, packs...)
 			if err != nil && !l.Options.Force {
-				return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+				return toUninstall, errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
 			}
 		}
 
 		for _, p := range solution {
-			Info("Uninstalling", p.HumanReadableString())
+			toUninstall = append(toUninstall, p)
+		}
+	} else {
+		toUninstall = append(toUninstall, packs...)
+	}
+
+	return toUninstall, nil
+}
+func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
+
+	for _, p := range packs {
+		if packs, _ := s.Database.FindPackages(p); len(packs) == 0 {
+			return errors.New("Package not found in the system")
+		}
+
+	}
+
+	Spinner(32)
+	toUninstall, err := l.computeUninstall(s, packs...)
+	if err != nil {
+		return errors.Wrap(err, "while computing uninstall")
+	}
+	SpinnerStop()
+
+	uninstall := func() error {
+		for _, p := range toUninstall {
 			err := l.uninstall(p, s)
 			if err != nil && !l.Options.Force {
 				return errors.Wrap(err, "Uninstall failed")
 			}
 		}
-	} else {
-		Info("Uninstalling", p.HumanReadableString(), "without deps")
-		err := l.uninstall(p, s)
-		if err != nil && !l.Options.Force {
-			return errors.Wrap(err, "Uninstall failed")
-		}
-		Info(":package:", p.HumanReadableString(), "uninstalled")
+		return nil
 	}
-	return nil
 
+	if len(toUninstall) == 0 {
+		Info("Nothing to do")
+		return nil
+	}
+
+	if l.Options.Ask {
+		Info(":recycle: Packages that are going to be removed from the system:\n   ", Yellow(packsToList(toUninstall)).BgBlack().String())
+		if Ask() {
+			l.Options.Ask = false // Don't prompt anymore
+			return uninstall()
+		} else {
+			return errors.New("Aborted by user")
+		}
+	}
+	return uninstall()
 }
 
 func (l *LuetInstaller) Repositories(r []Repository) { l.PackageRepositories = r }

pkg/installer/installer_test.go

@@ -24,6 +24,9 @@ import (
 	compiler "github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/installer"
+	solver "github.com/mudler/luet/pkg/solver"
+
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
@@ -31,6 +34,18 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+func stubRepo(tmpdir, tree string) (installer.Repository, error) {
+	return GenerateRepository(
+		"test",
+		"description",
+		"disk",
+		[]string{tmpdir},
+		1,
+		tmpdir,
+		[]string{tree},
+		pkg.NewInMemoryDatabase(false), nil, "", false, false)
+}
+
 var _ = Describe("Installer", func() {
 	Context("Writes a repository definition", func() {
 		It("Writes a repo and can install packages from it", func() {
@@ -47,7 +62,7 @@ var _ = Describe("Installer", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -58,8 +73,8 @@ var _ = Describe("Installer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -82,13 +97,13 @@ var _ = Describe("Installer", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -132,7 +147,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p.GetName()).To(Equal("b"))
 
-			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			err = inst.Uninstall(system, &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
@@ -163,7 +178,7 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
 			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
-				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -174,8 +189,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -198,7 +213,9 @@ urls:
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
 			treeFile := NewDefaultTreeRepositoryFile()
 			treeFile.SetCompressionType(compiler.None)
 			repo.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
@@ -207,7 +224,7 @@ urls:
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -251,7 +268,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p.GetName()).To(Equal("b"))
 
-			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			err = inst.Uninstall(system, &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
@@ -281,7 +298,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -292,8 +309,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -316,13 +333,20 @@ urls:
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository(
+				"test",
+				"description",
+				"disk",
+				[]string{tmpdir}, 1,
+				tmpdir,
+				[]string{"../../tests/fixtures/buildable"},
+				pkg.NewInMemoryDatabase(false), nil, "", false, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -371,7 +395,7 @@ urls:
 			Expect(files).To(Equal([]string{"artifact42", "test5", "test6"}))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			err = inst.Uninstall(system, &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
@@ -399,7 +423,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -410,8 +434,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -434,13 +458,21 @@ urls:
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository(
+				"test",
+				"description",
+				"disk",
+				[]string{tmpdir},
+				1,
+				tmpdir,
+				[]string{"../../tests/fixtures/buildable"},
+				pkg.NewInMemoryDatabase(false), nil, "", false, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -486,7 +518,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -500,9 +532,9 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 
-			repo, err = GenerateRepository("test", "description", "disk", []string{tmpdir2}, 1, tmpdir2, []string{"../../tests/fixtures/alpine"}, pkg.NewInMemoryDatabase(false))
+			repo, err = stubRepo(tmpdir2, "../../tests/fixtures/alpine")
 			Expect(err).ToNot(HaveOccurred())
-			err = repo.Write(tmpdir2, false)
+			err = repo.Write(tmpdir2, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			fakeroot, err = ioutil.TempDir("", "fakeroot")
@@ -545,7 +577,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -569,13 +601,13 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -663,8 +695,8 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 			Expect(len(generalRecipeNewRepo.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
-			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -695,18 +727,18 @@ urls:
 			_, errs = c2.CompileParallel(false, compiler.NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade_old_repo"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			repoupgrade, err := GenerateRepository("test", "description", "disk", []string{tmpdirnewrepo}, 1, tmpdirnewrepo, []string{"../../tests/fixtures/upgrade_new_repo"}, pkg.NewInMemoryDatabase(false))
+			repoupgrade, err := stubRepo(tmpdirnewrepo, "../../tests/fixtures/upgrade_new_repo")
 			Expect(err).ToNot(HaveOccurred())
-			err = repoupgrade.Write(tmpdirnewrepo, false)
+			err = repoupgrade.Write(tmpdirnewrepo, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
@@ -795,7 +827,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -818,13 +850,13 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
@@ -897,6 +929,47 @@ urls:
 
 	})
 
+	Context("Uninstallation", func() {
+		It("fails if package is required by others which are installed", func() {
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1, CheckConflicts: true})
+
+			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B := pkg.NewPackage("calamares", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+			C := pkg.NewPackage("kpmcore", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+			Z := pkg.NewPackage("chromium", "", []*pkg.DefaultPackage{A}, []*pkg.DefaultPackage{})
+			F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{Z, B}, []*pkg.DefaultPackage{})
+
+			Z.SetVersion("86.0.4240.193+2")
+			Z.SetCategory("www-client")
+			B.SetVersion("3.2.32.1+5")
+			B.SetCategory("app-admin")
+			C.SetVersion("4.2.0+2")
+			C.SetCategory("sys-libs-5")
+			D.SetVersion("5.19.5+9")
+			D.SetCategory("layers")
+
+			for _, p := range []pkg.Package{A, B, C, D, Z, F} {
+				_, err := systemDB.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = inst.Uninstall(system, D)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
 	Context("Existing files", func() {
 		It("Reclaims them", func() {
 			//repo:=NewLuetSystemRepository()
@@ -912,7 +985,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -935,13 +1008,13 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
@@ -1017,7 +1090,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1037,13 +1110,13 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade_old_repo"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -1109,7 +1182,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(3))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1123,10 +1196,10 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err = GenerateRepository("test", "description", "disk", []string{tmpdir2}, 1, tmpdir2, []string{"../../tests/fixtures/upgrade_new_repo"}, pkg.NewInMemoryDatabase(false))
+			repo, err = stubRepo(tmpdir2, "../../tests/fixtures/upgrade_new_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			err = repo.Write(tmpdir2, false)
+			err = repo.Write(tmpdir2, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			inst = NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})

pkg/installer/interface.go

@@ -24,7 +24,7 @@ import (
 
 type Installer interface {
 	Install(pkg.Packages, *System) error
-	Uninstall(pkg.Package, *System) error
+	Uninstall(*System, ...pkg.Package) error
 	Upgrade(s *System) error
 	Reclaim(s *System) error
 
@@ -51,7 +51,7 @@ type Repository interface {
 	SetIndex(i compiler.ArtifactIndex)
 	GetTree() tree.Builder
 	SetTree(tree.Builder)
-	Write(path string, resetRevision bool) error
+	Write(path string, resetRevision, force bool) error
 	Sync(bool) (Repository, error)
 	GetTreePath() string
 	SetTreePath(string)
@@ -72,4 +72,6 @@ type Repository interface {
 	SetRepositoryFile(string, LuetRepositoryFile)
 	SetName(p string)
 	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
+	GetBackend() compiler.CompilerBackend
+	SetBackend(b compiler.CompilerBackend)
 }

pkg/installer/repository.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mudler/luet/pkg/bus"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
@@ -45,6 +46,10 @@ const (
 
 	REPOFILE_TREE_KEY = "tree"
 	REPOFILE_META_KEY = "meta"
+
+	DiskRepositoryType   = "disk"
+	HttpRepositoryType   = "http"
+	DockerRepositoryType = "docker"
 )
 
 type LuetRepositoryFile struct {
@@ -59,6 +64,9 @@ type LuetSystemRepository struct {
 	Index           compiler.ArtifactIndex        `json:"index"`
 	Tree            tree.Builder                  `json:"-"`
 	RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"`
+	Backend         compiler.CompilerBackend      `json:"-"`
+	PushImages      bool                          `json:"-"`
+	ForcePush       bool                          `json:"-"`
 }
 
 type LuetSystemRepositorySerialized struct {
@@ -156,27 +164,54 @@ func NewDefaultMetaRepositoryFile() LuetRepositoryFile {
 	}
 }
 
+// SetFileName sets the name of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this set the filename that the client will pull
 func (f *LuetRepositoryFile) SetFileName(n string) {
 	f.FileName = n
 }
 
+// GetFileName returns the name of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this gets the filename that the client will pull
 func (f *LuetRepositoryFile) GetFileName() string {
 	return f.FileName
 }
+
+// SetCompressionType sets the compression type of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this sets the compression type that the client will use to uncompress the artifact
 func (f *LuetRepositoryFile) SetCompressionType(c compiler.CompressionImplementation) {
 	f.CompressionType = c
 }
+
+// GetCompressionType gets the compression type of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this gets the compression type that the client will use to uncompress the artifact
 func (f *LuetRepositoryFile) GetCompressionType() compiler.CompressionImplementation {
 	return f.CompressionType
 }
+
+// SetChecksums sets the checksum of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this sets the checksums that the client will use to verify the artifact
 func (f *LuetRepositoryFile) SetChecksums(c compiler.Checksums) {
 	f.Checksums = c
 }
+
+// GetChecksums gets the checksum of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this gets the checksums that the client will use to verify the artifact
 func (f *LuetRepositoryFile) GetChecksums() compiler.Checksums {
 	return f.Checksums
 }
 
-func GenerateRepository(name, descr, t string, urls []string, priority int, src string, treesDir []string, db pkg.PackageDatabase) (Repository, error) {
+// GenerateRepository generates a new repository from the given argument.
+// If the repository is of the docker type, it will also push the package images.
+// In case the repository is local, it will build the package Index
+func GenerateRepository(name, descr, t string, urls []string,
+	priority int, src string, treesDir []string, db pkg.PackageDatabase,
+	b compiler.CompilerBackend, imagePrefix string, pushImages, force bool) (Repository, error) {
 
 	tr := tree.NewInstallerRecipe(db)
 
@@ -187,14 +222,31 @@ func GenerateRepository(name, descr, t string, urls []string, priority int, src
 		}
 	}
 
-	art, err := buildPackageIndex(src, tr.GetDatabase())
-	if err != nil {
-		return nil, err
+	// if !strings.HasSuffix(imagePrefix, "/") {
+	// 	imagePrefix = imagePrefix + "/"
+	// }
+
+	var art []compiler.Artifact
+	var err error
+	switch t {
+	case DiskRepositoryType, HttpRepositoryType:
+		art, err = buildPackageIndex(src, tr.GetDatabase())
+		if err != nil {
+			return nil, err
+		}
+
+	case DockerRepositoryType:
+		art, err = generatePackageImages(b, imagePrefix, src, tr.GetDatabase(), pushImages, force)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	return NewLuetSystemRepository(
+	repo := NewLuetSystemRepository(
 		config.NewLuetRepository(name, t, descr, urls, priority, true, false),
-		art, tr), nil
+		art, tr, pushImages, force)
+	repo.SetBackend(b)
+	return repo, nil
 }
 
 func NewSystemRepository(repo config.LuetRepository) Repository {
@@ -204,12 +256,14 @@ func NewSystemRepository(repo config.LuetRepository) Repository {
 	}
 }
 
-func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder) Repository {
+func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder, pushImages, force bool) Repository {
 	return &LuetSystemRepository{
 		LuetRepository:  repo,
 		Index:           art,
 		Tree:            builder,
 		RepositoryFiles: map[string]LuetRepositoryFile{},
+		PushImages:      pushImages,
+		ForcePush:       force,
 	}
 }
 
@@ -243,6 +297,71 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 	return r, err
 }
 
+func pushImage(b compiler.CompilerBackend, image string, force bool) error {
+	if b.ImageAvailable(image) && !force {
+		Debug("Image", image, "already present, skipping")
+		return nil
+	}
+	return b.Push(compiler.CompilerBackendOptions{ImageName: image})
+}
+
+func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]compiler.Artifact, error) {
+	Info("Generating docker images for packages in", imagePrefix)
+	var art []compiler.Artifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil // Skip with no errors
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		artifact, err := compiler.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				artifact.GetCompileSpec().GetPackage().HumanReadableString()))
+			return nil
+		}
+
+		packageImage := fmt.Sprintf("%s:%s", imagePrefix, artifact.GetCompileSpec().GetPackage().GetFingerPrint())
+
+		if imagePush && b.ImageAvailable(packageImage) && !force {
+			Info("Image", packageImage, "already present, skipping. use --force-push to override")
+		} else {
+			Info("Generating final image", packageImage,
+				"for package ", artifact.GetCompileSpec().GetPackage().HumanReadableString())
+			if opts, err := artifact.GenerateFinalImage(packageImage, b, true); err != nil {
+				return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+			}
+		}
+		if imagePush {
+			if err := pushImage(b, packageImage, force); err != nil {
+				return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage)
+			}
+		}
+
+		art = append(art, artifact)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
 func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) {
 
 	var art []compiler.Artifact
@@ -265,6 +384,8 @@ func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact
 		// We want to include packages that are ONLY referenced in the tree.
 		// the ones which aren't should be deleted. (TODO: by another cli command?)
 		if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				artifact.GetCompileSpec().GetPackage().HumanReadableString()))
 			return nil
 		}
 
@@ -303,6 +424,13 @@ func (r *LuetSystemRepository) SetType(p string) {
 	r.LuetRepository.Type = p
 }
 
+func (r *LuetSystemRepository) GetBackend() compiler.CompilerBackend {
+	return r.Backend
+}
+func (r *LuetSystemRepository) SetBackend(b compiler.CompilerBackend) {
+	r.Backend = b
+}
+
 func (r *LuetSystemRepository) SetName(p string) {
 	r.LuetRepository.Name = p
 }
@@ -397,7 +525,7 @@ func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repos
 	return repo, err
 }
 
-func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
+func (r *LuetSystemRepository) genLocalRepo(dst string, resetRevision bool) error {
 	err := os.MkdirAll(dst, os.ModePerm)
 	if err != nil {
 		return err
@@ -424,6 +552,14 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 		r.Name, r.Revision, r.LastUpdate,
 	))
 
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
+
 	// Create tree and repository file
 	archive, err := config.LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
@@ -504,23 +640,247 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 		return err
 	}
 
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
 	return nil
 }
 
+func (r *LuetSystemRepository) genDockerRepo(imagePrefix string, resetRevision, force bool) error {
+	// - Iterate over meta, build final images, push them if necessary
+	//   - while pushing, check if image already exists, and if exist push them only if --force is supplied
+	// - Generate final images for metadata and push
+
+	imageRepository := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
+
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
+
+	repoTemp, err := config.LuetCfg.GetSystem().TempDir("repo")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for repository")
+	}
+	defer os.RemoveAll(repoTemp) // clean up
+
+	if r.GetBackend().ImageAvailable(imageRepository) {
+		if err := r.GetBackend().DownloadImage(compiler.CompilerBackendOptions{ImageName: imageRepository}); err != nil {
+			return errors.Wrapf(err, "while downloading '%s'", imageRepository)
+		}
+
+		if err := r.GetBackend().ExtractRootfs(compiler.CompilerBackendOptions{ImageName: imageRepository, Destination: repoTemp}, false); err != nil {
+			return errors.Wrapf(err, "while extracting '%s'", imageRepository)
+		}
+	}
+
+	repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE)
+	if resetRevision {
+		r.Revision = 0
+	} else {
+		if _, err := os.Stat(repospec); !os.IsNotExist(err) {
+			// Read existing file for retrieve revision
+			spec, err := r.ReadSpecFile(repospec, false)
+			if err != nil {
+				return err
+			}
+			r.Revision = spec.GetRevision()
+		}
+	}
+	r.Revision++
+
+	Info(fmt.Sprintf(
+		"For repository %s creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imageRepository,
+	})
+
+	// Create tree and repository file
+	archive, err := config.LuetCfg.GetSystem().TempDir("archive")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for archive")
+	}
+	defer os.RemoveAll(archive) // clean up
+	err = r.GetTree().Save(archive)
+	if err != nil {
+		return errors.Wrap(err, "Error met while saving the tree")
+	}
+
+	treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY)
+	if err != nil {
+		treeFile = NewDefaultTreeRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
+	}
+
+	a := compiler.NewPackageArtifact(filepath.Join(repoTemp, treeFile.GetFileName()))
+	a.SetCompressionType(treeFile.GetCompressionType())
+	err = a.Compress(archive, 1)
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating package archive")
+	}
+
+	// Update the tree name with the name created by compression selected.
+	treeFile.SetFileName(a.GetFileName())
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	treeFile.SetChecksums(a.GetChecksums())
+	r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
+
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	treeArchive, err := compiler.CreateArtifactForFile(a.GetPath())
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	imageTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName())
+	Debug("Generating image", imageTree)
+	if opts, err := treeArchive.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
+	}
+	if r.PushImages {
+		if err := pushImage(r.GetBackend(), imageTree, true); err != nil {
+			return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree)
+		}
+	}
+
+	// Create Metadata struct and serialized repository
+	meta, serialized := r.Serialize()
+
+	// Create metadata file and repository file
+	metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaTmpDir) // clean up
+
+	metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY)
+	if err != nil {
+		metaFile = NewDefaultMetaRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	}
+
+	repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE)
+	// Create repository.meta.yaml file
+	err = meta.WriteFile(repoMetaSpec)
+	if err != nil {
+		return err
+	}
+
+	// create temp dir for metafile
+	metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaDir) // clean up
+
+	a = compiler.NewPackageArtifact(filepath.Join(metaDir, metaFile.GetFileName()))
+	a.SetCompressionType(metaFile.GetCompressionType())
+	err = a.Compress(metaTmpDir, 1)
+	if err != nil {
+		return errors.Wrap(err, "Error met while archiving repository metadata")
+	}
+
+	metaFile.SetFileName(a.GetFileName())
+	r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for metadata")
+	}
+	metaFile.SetChecksums(a.GetChecksums())
+
+	// Files are downloaded as-is from docker images
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	metaArchive, err := compiler.CreateArtifactForFile(a.GetPath())
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	imageMetaTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName())
+	if opts, err := metaArchive.GenerateFinalImage(imageMetaTree, r.GetBackend(), false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+	}
+	if r.PushImages {
+		if err := pushImage(r.GetBackend(), imageMetaTree, true); err != nil {
+			return errors.Wrapf(err, "Failed while pushing image: '%s'", imageMetaTree)
+		}
+	}
+	data, err := yaml.Marshal(serialized)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(repospec, data, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	tempRepoFile := filepath.Join(metaDir, REPOSITORY_SPECFILE+".tar")
+	if err := helpers.Tar(repospec, tempRepoFile); err != nil {
+		return errors.Wrap(err, "Error met while archiving repository file")
+	}
+
+	a = compiler.NewPackageArtifact(tempRepoFile)
+	imageRepo := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
+	if opts, err := a.GenerateFinalImage(imageRepo, r.GetBackend(), false); err != nil {
+		return errors.Wrap(err, "Failed generating repository image"+opts.ImageName)
+	}
+	if r.PushImages {
+		if err := pushImage(r.GetBackend(), imageRepo, true); err != nil {
+			return errors.Wrapf(err, "Failed while pushing image: '%s'", imageRepo)
+		}
+	}
+
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imagePrefix,
+	})
+	return nil
+}
+
+// Write writes the repository metadata to the supplied destination
+func (r *LuetSystemRepository) Write(dst string, resetRevision, force bool) error {
+	switch r.GetType() {
+	case DiskRepositoryType, HttpRepositoryType:
+		return r.genLocalRepo(dst, resetRevision)
+	case DockerRepositoryType:
+		return r.genDockerRepo(dst, resetRevision, force)
+	}
+	return errors.New("invalid repository type")
+}
+
 func (r *LuetSystemRepository) Client() Client {
 	switch r.GetType() {
-	case "disk":
+	case DiskRepositoryType:
 		return client.NewLocalClient(client.RepoData{Urls: r.GetUrls()})
-	case "http":
+	case HttpRepositoryType:
 		return client.NewHttpClient(
 			client.RepoData{
 				Urls:           r.GetUrls(),
 				Authentication: r.GetAuthentication(),
 			})
-	}
 
+	case DockerRepositoryType:
+		return client.NewDockerClient(
+			client.RepoData{
+				Urls:           r.GetUrls(),
+				Authentication: r.GetAuthentication(),
+			})
+	}
 	return nil
 }
+
 func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	var repoUpdated bool = false
 	var treefs, metafs string
@@ -529,7 +889,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	Debug("Sync of the repository", r.Name, "in progress...")
 	c := r.Client()
 	if c == nil {
-		return nil, errors.New("No client could be generated from repository.")
+		return nil, errors.New("no client could be generated from repository")
 	}
 
 	// Retrieve remote repository.yaml for retrieve revision and date
@@ -543,7 +903,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	if err != nil {
 		return nil, err
 	}
-	// Remove temporary file that contains repository.html.
+	// Remove temporary file that contains repository.yaml
 	// Example: /tmp/HttpClient236052003
 	defer os.RemoveAll(file)
 
@@ -573,15 +933,10 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 		}
-		// Note: If we always remove them, later on, no other structure can access
-		// to the tree for e.g. to retrieve finalizers
-
 		metafs, err = config.LuetCfg.GetSystem().TempDir("metafs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met whilte creating tempdir for metafs")
 		}
-		//defer os.RemoveAll(metafs)
-
 	}
 
 	// POST: treeFile and metaFile are present. I check this inside
@@ -592,17 +947,18 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	if !repoUpdated {
 
 		// Get Tree
-		a := compiler.NewPackageArtifact(treeFile.GetFileName())
-		artifactTree, err := c.DownloadArtifact(a)
+		downloadedTreeFile, err := c.DownloadFile(treeFile.GetFileName())
 		if err != nil {
 			return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName())
 		}
-		defer os.Remove(artifactTree.GetPath())
+		defer os.Remove(downloadedTreeFile)
 
-		artifactTree.SetChecksums(treeFile.GetChecksums())
-		artifactTree.SetCompressionType(treeFile.GetCompressionType())
+		// Treat the file as artifact, in order to verify it
+		treeFileArtifact := compiler.NewPackageArtifact(downloadedTreeFile)
+		treeFileArtifact.SetChecksums(treeFile.GetChecksums())
+		treeFileArtifact.SetCompressionType(treeFile.GetCompressionType())
 
-		err = artifactTree.Verify()
+		err = treeFileArtifact.Verify()
 		if err != nil {
 			return nil, errors.Wrap(err, "Tree integrity check failure")
 		}
@@ -610,17 +966,17 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
 
 		// Get Repository Metadata
-		a = compiler.NewPackageArtifact(metaFile.GetFileName())
-		artifactMeta, err := c.DownloadArtifact(a)
+		downloadedMeta, err := c.DownloadFile(metaFile.GetFileName())
 		if err != nil {
 			return nil, errors.Wrap(err, "While downloading "+metaFile.GetFileName())
 		}
-		defer os.Remove(artifactMeta.GetPath())
+		defer os.Remove(downloadedMeta)
 
-		artifactMeta.SetChecksums(metaFile.GetChecksums())
-		artifactMeta.SetCompressionType(metaFile.GetCompressionType())
+		metaFileArtifact := compiler.NewPackageArtifact(downloadedMeta)
+		metaFileArtifact.SetChecksums(metaFile.GetChecksums())
+		metaFileArtifact.SetCompressionType(metaFile.GetCompressionType())
 
-		err = artifactMeta.Verify()
+		err = metaFileArtifact.Verify()
 		if err != nil {
 			return nil, errors.Wrap(err, "Metadata integrity check failure")
 		}
@@ -640,7 +996,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		}
 		Debug("Decompress tree of the repository " + r.Name + "...")
 
-		err = artifactTree.Unpack(treefs, true)
+		err = treeFileArtifact.Unpack(treefs, true)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while unpacking tree")
 		}
@@ -648,7 +1004,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		// FIXME: It seems that tar with only one file doesn't create destination
 		//       directory. I create directory directly for now.
 		os.MkdirAll(metafs, os.ModePerm)
-		err = artifactMeta.Unpack(metafs, true)
+		err = metaFileArtifact.Unpack(metafs, true)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while unpacking metadata")
 		}
@@ -692,9 +1048,12 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	repo.SetPriority(r.GetPriority())
 	repo.SetName(r.GetName())
 	InfoC(
-		aurora.Bold(
-			aurora.Yellow(":information_source: Repository "+repo.GetName()+" priority: ")).String() +
-			aurora.Bold(aurora.Green(repo.GetPriority())).String() + " - type " +
+		aurora.Yellow(":information_source:").String() +
+			aurora.Magenta("Repository: ").String() +
+			aurora.Green(aurora.Bold(repo.GetName()).String()).String() +
+			aurora.Magenta(" Priority: ").String() +
+			aurora.Bold(aurora.Green(repo.GetPriority())).String() +
+			aurora.Magenta(" Type: ").String() +
 			aurora.Bold(aurora.Green(repo.GetType())).String(),
 	)
 	return repo, nil
@@ -807,13 +1166,11 @@ PACKAGE:
 				c, err := r.GetTree().GetDatabase().FindPackageCandidate(pack)
 				// If FindPackageCandidate returns the same package, it means it couldn't find one.
 				// Skip this repository and keep looking.
-				if c.String() == pack.String() {
+				if err != nil { //c.String() == pack.String() {
 					continue REPOSITORY
 				}
-				if err == nil {
-					matches = append(matches, c)
-					continue PACKAGE
-				}
+				matches = append(matches, c)
+				continue PACKAGE
 			} else {
 				// If it's not a selector, just append it
 				matches = append(matches, pack)

pkg/installer/repository_test.go

@@ -19,20 +19,36 @@ import (
 
 	//	. "github.com/mudler/luet/pkg/installer"
 
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
 	"github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	config "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
+func dockerStubRepo(tmpdir, tree, image string, push, force bool) (installer.Repository, error) {
+	return GenerateRepository(
+		"test",
+		"description",
+		"docker",
+		[]string{image},
+		1,
+		tmpdir,
+		[]string{tree},
+		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force)
+}
+
 var _ = Describe("Repository", func() {
 	Context("Generation", func() {
 		It("Generate repository metadata", func() {
@@ -48,7 +64,7 @@ var _ = Describe("Repository", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -59,8 +75,8 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			compiler.SetConcurrency(1)
@@ -83,13 +99,13 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
@@ -116,11 +132,11 @@ var _ = Describe("Repository", func() {
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -132,8 +148,8 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
@@ -165,13 +181,13 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
@@ -215,6 +231,89 @@ urls:
 			Expect(matches).To(Equal([]PackageMatch{{Repo: repo1, Package: package1}}))
 
 		})
+	})
+	Context("Docker repository", func() {
+		repoImage := os.Getenv("UNIT_TEST_DOCKER_IMAGE_REPOSITORY")
 
+		BeforeEach(func() {
+			if repoImage == "" {
+				Skip("UNIT_TEST_DOCKER_IMAGE_REPOSITORY not specified")
+			}
+		})
+
+		It("generates images", func() {
+			b := backend.NewSimpleDockerBackend()
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			localcompiler.SetConcurrency(1)
+
+			artifact, err := localcompiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/buildable", repoImage, true, true)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(repoImage, false, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "tree.tar.gz"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.meta.yaml.tar"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.yaml"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "b-test-1.0"))).To(BeTrue())
+
+			extracted, err := ioutil.TempDir("", "extracted")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(extracted) // clean up
+
+			c := repo.Client()
+
+			f, err := c.DownloadFile("repository.yaml")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(f)).To(ContainSubstring("name: test"))
+
+			a, err := c.DownloadArtifact(&compiler.PackageArtifact{
+				Path: "test.tar",
+				CompileSpec: &compiler.LuetCompilationSpec{
+					Package: &pkg.DefaultPackage{
+						Name:     "b",
+						Category: "test",
+						Version:  "1.0",
+					},
+				},
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(a.Unpack(extracted, false)).ToNot(HaveOccurred())
+			Expect(helpers.Read(filepath.Join(extracted, "test6"))).To(Equal("artifact6\n"))
+		})
 	})
 })

pkg/installer/system.go

@@ -1,7 +1,11 @@
 package installer
 
 import (
+	"github.com/hashicorp/go-multierror"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
 )
 
 type System struct {
@@ -12,3 +16,38 @@ type System struct {
 func (s *System) World() (pkg.Packages, error) {
 	return s.Database.World(), nil
 }
+
+type templatedata map[string]interface{}
+
+func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
+	var errs error
+	executedFinalizer := map[string]bool{}
+	for _, p := range packs {
+		if helpers.Exists(p.Rel(tree.FinalizerFile)) {
+			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile), "")
+			if err != nil {
+				Warning("Failed rendering finalizer for ", p.HumanReadableString(), err.Error())
+				errs = multierror.Append(errs, err)
+				continue
+			}
+
+			if _, exists := executedFinalizer[p.GetFingerPrint()]; !exists {
+				executedFinalizer[p.GetFingerPrint()] = true
+				Info("Executing finalizer for " + p.HumanReadableString())
+				finalizer, err := NewLuetFinalizerFromYaml([]byte(out))
+				if err != nil {
+					Warning("Failed reading finalizer for ", p.HumanReadableString(), err.Error())
+					errs = multierror.Append(errs, err)
+					continue
+				}
+				err = finalizer.RunInstall(s)
+				if err != nil {
+					Warning("Failed running finalizer for ", p.HumanReadableString(), err.Error())
+					errs = multierror.Append(errs, err)
+					continue
+				}
+			}
+		}
+	}
+	return errs
+}

pkg/logger/logger.go

@@ -3,7 +3,10 @@ package logger
 import (
 	"fmt"
 	"os"
+	"path"
 	"regexp"
+	"runtime"
+	"strings"
 
 	. "github.com/mudler/luet/pkg/config"
 
@@ -36,6 +39,22 @@ func GetAurora() Aurora {
 	return aurora
 }
 
+func Ask() bool {
+	var input string
+
+	Info("Do you want to continue with this operation? [y/N]: ")
+	_, err := fmt.Scanln(&input)
+	if err != nil {
+		return false
+	}
+	input = strings.ToLower(input)
+
+	if input == "y" || input == "yes" {
+		return true
+	}
+	return false
+}
+
 func ZapLogger() error {
 	var err error
 	if z == nil {
@@ -179,13 +198,13 @@ func msg(level string, withoutColor bool, msg ...interface{}) {
 	} else {
 		switch level {
 		case "warning":
-			levelMsg = Bold(Yellow(":construction: " + message)).BgBlack().String()
+			levelMsg = Yellow(":construction: warning" + message).BgBlack().String()
 		case "debug":
 			levelMsg = White(message).BgBlack().String()
 		case "info":
-			levelMsg = Bold(White(message)).BgBlack().String()
+			levelMsg = message
 		case "error":
-			levelMsg = Bold(Red(":bomb: " + message + ":fire:")).BgBlack().String()
+			levelMsg = Red(message).String()
 		}
 	}
 
@@ -211,6 +230,11 @@ func Warning(mess ...interface{}) {
 }
 
 func Debug(mess ...interface{}) {
+	pc, file, line, ok := runtime.Caller(1)
+	if ok {
+		mess = append([]interface{}{fmt.Sprintf("DEBUG (%s:#%d:%v)",
+			path.Base(file), line, runtime.FuncForPC(pc).Name())}, mess...)
+	}
 	msg("debug", false, mess...)
 }
 
@@ -231,6 +255,6 @@ func Error(mess ...interface{}) {
 }
 
 func Fatal(mess ...interface{}) {
-	Error(mess)
+	Error(mess...)
 	os.Exit(1)
 }

pkg/package/database.go

@@ -28,6 +28,10 @@ type PackageDatabase interface {
 }
 
 type PackageSet interface {
+	Clone(PackageDatabase) error
+	Copy() (PackageDatabase, error)
+
+	GetRevdeps(p Package) (Packages, error)
 	GetPackages() []string //Ids
 	CreatePackage(pkg Package) (string, error)
 	GetPackage(ID string) (Package, error)

pkg/package/database_bench_test.go

@@ -0,0 +1,116 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	"io/ioutil"
+	"os"
+	"strconv"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Database  benchmark", func() {
+
+	Context("BoltDB", func() {
+
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		var db PackageSet
+
+		BeforeEach(func() {
+
+			tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			db = NewBoltDatabase(tmpfile.Name())
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+
+		Measure("it should be fast in computing world from a 50000 dataset", func(b Benchmarker) {
+			for i := 0; i < 50000; i++ {
+				a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			runtime := b.Time("runtime", func() {
+				packs := db.World()
+				Expect(len(packs)).To(Equal(50000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 30), "World() shouldn't take too long.")
+
+		}, 1)
+
+		Measure("it should be fast in computing world from a 100000 dataset", func(b Benchmarker) {
+			for i := 0; i < 100000; i++ {
+				a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			runtime := b.Time("runtime", func() {
+				packs := db.World()
+				Expect(len(packs)).To(Equal(100000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 30), "World() shouldn't take too long.")
+
+		}, 1)
+	})
+
+	Context("InMemory", func() {
+
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		var db PackageSet
+
+		BeforeEach(func() {
+
+			tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			db = NewInMemoryDatabase(false)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+
+		Measure("it should be fast in computing world from a 100000 dataset", func(b Benchmarker) {
+
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 100000; i++ {
+					a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					_, err := db.CreatePackage(a)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				packs := db.World()
+				Expect(len(packs)).To(Equal(100000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 10), "World() shouldn't take too long.")
+
+		}, 2)
+	})
+})

pkg/package/database_boltdb.go

@@ -47,6 +47,14 @@ func NewBoltDatabase(path string) PackageDatabase {
 	return &BoltDatabase{Path: path, ProvidesDatabase: map[string]map[string]Package{}}
 }
 
+func (db *BoltDatabase) Clone(to PackageDatabase) error {
+	return clone(db, to)
+}
+
+func (db *BoltDatabase) Copy() (PackageDatabase, error) {
+	return copy(db)
+}
+
 func (db *BoltDatabase) Get(s string) (string, error) {
 	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
@@ -86,6 +94,17 @@ func (db *BoltDatabase) Retrieve(ID string) ([]byte, error) {
 	return enc, nil
 }
 
+// GetRevdeps uses a new inmemory db to calcuate revdeps
+// TODO: Have a memory instance for boltdb, so we don't compute each time we get called
+// as this is REALLY expensive. But we don't perform usually those operations in a file db.
+func (db *BoltDatabase) GetRevdeps(p Package) (Packages, error) {
+	memory, err := db.Copy()
+	if err != nil {
+		return nil, errors.New("Failed copying bolt db to memory")
+	}
+	return memory.GetRevdeps(p)
+}
+
 func (db *BoltDatabase) FindPackage(tofind Package) (Package, error) {
 	// Provides: Return the replaced package here
 	if provided, err := db.getProvide(tofind); err == nil {
@@ -162,26 +181,17 @@ func (db *BoltDatabase) GetAllPackages(packages chan Package) error {
 		return err
 	}
 	defer bolt.Close()
-	// Fetching records one by one (useful when the bucket contains a lot of records)
-	//query := bolt.Select()
-
-	var packs []Package
+	var packs []DefaultPackage
 	err = bolt.All(&packs)
 	if err != nil {
 		return err
 	}
 
 	for _, r := range packs {
-		packages <- r
+		packages <- &r
 	}
 
 	return nil
-
-	// return query.Each(new(DefaultPackage), func(record interface{}) error {
-	// 	u := record.(*DefaultPackage)
-	// 	packages <- u
-	// 	return err
-	// })
 }
 
 // Encode encodes the package to string.
@@ -316,33 +326,41 @@ func (db *BoltDatabase) RemovePackage(p Package) error {
 }
 
 func (db *BoltDatabase) World() Packages {
+	var packs []DefaultPackage
 
-	var all []Package
-	// FIXME: This should all be locked in the db - for now forbid the solver to be run in threads.
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err == nil {
-			all = append(all, pack)
-		}
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return Packages([]Package{})
 	}
-	return Packages(all)
+	defer bolt.Close()
+	err = bolt.All(&packs)
+	if err != nil {
+		return Packages([]Package{})
+	}
+	models := make([]Package, len(packs))
+	for i, _ := range packs {
+		models[i] = &packs[i]
+	}
+
+	return Packages(models)
 }
 
 func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
 
 	required, err := db.FindPackage(p)
 	if err != nil {
-
+		err = nil
 		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
 		packages, err := p.Expand(db)
 		//	Info("Expanded", packages, err)
 		if err != nil || len(packages) == 0 {
 			required = p
+			err = errors.Wrap(err, "Candidate not found")
 		} else {
 			required = packages.Best(nil)
 
 		}
-		return required, nil
+		return required, err
 		//required = &DefaultPackage{Name: "test"}
 	}
 
@@ -353,10 +371,22 @@ func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
 // FindPackages return the list of the packages beloging to cat/name  (any versions in requested range)
 // FIXME: Optimize, see inmemorydb
 func (db *BoltDatabase) FindPackages(p Package) (Packages, error) {
+	if !p.IsSelector() {
+		pack, err := db.FindPackage(p)
+		if err != nil {
+			return []Package{}, err
+		}
+		return []Package{pack}, nil
+	}
+
 	// Provides: Treat as the replaced package here
 	if provided, err := db.getProvide(p); err == nil {
 		p = provided
+		if !provided.IsSelector() {
+			return Packages{provided}, nil
+		}
 	}
+
 	var versionsInWorld []Package
 	for _, w := range db.World() {
 		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
@@ -376,6 +406,11 @@ func (db *BoltDatabase) FindPackages(p Package) (Packages, error) {
 
 // FindPackageVersions return the list of the packages beloging to cat/name
 func (db *BoltDatabase) FindPackageVersions(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+
 	var versionsInWorld []Package
 	for _, w := range db.World() {
 		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
@@ -390,11 +425,7 @@ func (db *BoltDatabase) FindPackageVersions(p Package) (Packages, error) {
 func (db *BoltDatabase) FindPackageLabel(labelKey string) (Packages, error) {
 	var ans []Package
 
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err != nil {
-			return ans, err
-		}
+	for _, pack := range db.World() {
 		if pack.HasLabel(labelKey) {
 			ans = append(ans, pack)
 		}
@@ -410,11 +441,7 @@ func (db *BoltDatabase) FindPackageLabelMatch(pattern string) (Packages, error)
 		return nil, errors.New("Invalid regex " + pattern + "!")
 	}
 
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err != nil {
-			return ans, err
-		}
+	for _, pack := range db.World() {
 		if pack.MatchLabel(re) {
 			ans = append(ans, pack)
 		}
@@ -431,12 +458,7 @@ func (db *BoltDatabase) FindPackageMatch(pattern string) (Packages, error) {
 		return nil, errors.New("Invalid regex " + pattern + "!")
 	}
 
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err != nil {
-			return ans, err
-		}
-
+	for _, pack := range db.World() {
 		if re.MatchString(pack.HumanReadableString()) {
 			ans = append(ans, pack)
 		}

pkg/package/database_boltdb_test.go

@@ -0,0 +1,287 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or ItNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	"io/ioutil"
+	"os"
+	"regexp"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("BoltDB Database", func() {
+
+	tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+	defer os.Remove(tmpfile.Name()) // clean up
+	var db PackageDatabase
+
+	BeforeEach(func() {
+		tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		db = NewBoltDatabase(tmpfile.Name())
+	})
+	Context("Simple package", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		It("Find packages", func() {
+			ID, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.GetPackage(ID)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack).To(Equal(a))
+			ids := db.GetPackages()
+
+			Expect(ids).To(Equal([]string{"1"}))
+
+			pack, err = db.FindPackage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Expands correctly", func() {
+
+			a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+			re := regexp.MustCompile("project[0-9][=].*")
+			for _, p := range []Package{a1, a11, a01} {
+				_, err := db.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			lst, err := a.Expand(db)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(lst).To(ContainElement(a11))
+			Expect(lst).To(ContainElement(a1))
+			Expect(lst).ToNot(ContainElement(a01))
+			Expect(len(lst)).To(Equal(2))
+			p := lst.Best(nil)
+			Expect(p).To(Equal(a11))
+			// Test annotation with null map
+			Expect(a.MatchAnnotation(re)).To(Equal(false))
+		})
+
+		It("Find best package candidate", func() {
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+
+		})
+
+		It("Find specific package candidate", func() {
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Provides replaces definitions", func() {
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+			a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+			Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+
+			s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackage(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+		})
+
+		Context("Provides", func() {
+
+			It("replaces definitions", func() {
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				pack, err := db.FindPackage(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(pack).To(Equal(a3))
+			})
+
+			It("replaces definitions", func() {
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(a3))
+			})
+
+			It("replaces definitions", func() {
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of unexisting packages", func() {
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of a required package", func() {
+
+				c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = db.CreatePackage(c)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			When("Searching with selectors", func() {
+				It("replaces definitions of a required package", func() {
+
+					c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+					z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+					z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+					Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+					_, err := db.CreatePackage(z)
+					Expect(err).ToNot(HaveOccurred())
+					_, err = db.CreatePackage(c)
+					Expect(err).ToNot(HaveOccurred())
+
+					s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					packs, err := db.FindPackages(s)
+					Expect(err).ToNot(HaveOccurred())
+					Expect(packs).To(ContainElement(z))
+				})
+			})
+
+		})
+
+	})
+
+})

pkg/package/database_common.go

@@ -0,0 +1,38 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+import "github.com/pkg/errors"
+
+func clone(src, dst PackageDatabase) error {
+	for _, i := range src.World() {
+		_, err := dst.CreatePackage(i)
+		if err != nil {
+			return errors.Wrap(err, "Failed create package "+i.HumanReadableString())
+		}
+	}
+	return nil
+}
+
+func copy(src PackageDatabase) (PackageDatabase, error) {
+	dst := NewInMemoryDatabase(false)
+
+	if err := clone(src, dst); err != nil {
+		return dst, errors.Wrap(err, "Failed create temporary in-memory db")
+	}
+
+	return dst, nil
+}

pkg/package/database_mem.go

@@ -31,6 +31,7 @@ var DBInMemoryInstance = &InMemoryDatabase{
 	Database:         map[string]string{},
 	CacheNoVersion:   map[string]map[string]interface{}{},
 	ProvidesDatabase: map[string]map[string]Package{},
+	RevDepsDatabase:  map[string]map[string]Package{},
 }
 
 type InMemoryDatabase struct {
@@ -39,6 +40,8 @@ type InMemoryDatabase struct {
 	FileDatabase     map[string][]string
 	CacheNoVersion   map[string]map[string]interface{}
 	ProvidesDatabase map[string]map[string]Package
+	RevDepsDatabase  map[string]map[string]Package
+	cached           map[string]interface{}
 }
 
 func NewInMemoryDatabase(singleton bool) PackageDatabase {
@@ -50,6 +53,8 @@ func NewInMemoryDatabase(singleton bool) PackageDatabase {
 			Database:         map[string]string{},
 			CacheNoVersion:   map[string]map[string]interface{}{},
 			ProvidesDatabase: map[string]map[string]Package{},
+			RevDepsDatabase:  map[string]map[string]Package{},
+			cached:           map[string]interface{}{},
 		}
 	}
 	return DBInMemoryInstance
@@ -125,6 +130,47 @@ func (db *InMemoryDatabase) GetAllPackages(packages chan Package) error {
 	return nil
 }
 
+func (db *InMemoryDatabase) getRevdeps(p Package, visited map[string]interface{}) (Packages, error) {
+	var versionsInWorld Packages
+	if _, ok := visited[p.HumanReadableString()]; ok {
+		return versionsInWorld, nil
+	}
+	visited[p.HumanReadableString()] = true
+
+	var res Packages
+	packs, err := db.FindPackages(p)
+	if err != nil {
+		return res, err
+	}
+	for _, pp := range packs {
+		//	db.Lock()
+		list := db.RevDepsDatabase[pp.GetFingerPrint()]
+		//	db.Unlock()
+		for _, revdep := range list {
+			dep, err := db.FindPackage(revdep)
+			if err != nil {
+				return res, err
+			}
+			res = append(res, dep)
+
+			packs, err := db.getRevdeps(dep, visited)
+			if err != nil {
+				return res, err
+			}
+			res = append(res, packs...)
+
+		}
+	}
+	return res.Unique(), nil
+}
+
+// GetRevdeps returns the package reverse dependencies,
+// matching also selectors in versions (>, <, >=, <=)
+// TODO: Code should use db explictly
+func (db *InMemoryDatabase) GetRevdeps(p Package) (Packages, error) {
+	return db.getRevdeps(p, make(map[string]interface{}))
+}
+
 // Encode encodes the package to string.
 // It returns an ID which can be used to retrieve the package later on.
 func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
@@ -143,9 +189,21 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 		return "", err
 	}
 
+	db.populateCaches(pd)
+
+	return ID, nil
+}
+
+func (db *InMemoryDatabase) populateCaches(p Package) {
+	pd, _ := p.(*DefaultPackage)
+
 	// Create extra cache between package -> []versions
 	db.Lock()
-	defer db.Unlock()
+	if _, ok := db.cached[p.GetFingerPrint()]; ok {
+		db.Unlock()
+		return
+	}
+	db.cached[p.GetFingerPrint()] = nil
 
 	// Provides: Store package provides, we will reuse this when walking deps
 	for _, provide := range pd.Provides {
@@ -157,21 +215,71 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 		db.ProvidesDatabase[provide.GetPackageName()][provide.GetVersion()] = p
 	}
 
-	_, ok = db.CacheNoVersion[p.GetPackageName()]
+	_, ok := db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
 		db.CacheNoVersion[p.GetPackageName()] = make(map[string]interface{})
 	}
 	db.CacheNoVersion[p.GetPackageName()][p.GetVersion()] = nil
 
-	return ID, nil
+	// Updating Revdeps
+	// Given that when we populate the cache we don't have the full db at hand
+	// We cycle over reverse dependency of a package to update their entry if they are matching
+	// the version selector
+	toUpdate, ok := db.RevDepsDatabase[pd.GetPackageName()]
+	if ok {
+		for _, pp := range toUpdate {
+			for _, re := range pp.GetRequires() {
+				if match, _ := pd.VersionMatchSelector(re.GetVersion(), nil); match {
+					_, ok = db.RevDepsDatabase[pd.GetFingerPrint()]
+					if !ok {
+						db.RevDepsDatabase[pd.GetFingerPrint()] = make(map[string]Package)
+					}
+					db.RevDepsDatabase[pd.GetFingerPrint()][pp.GetFingerPrint()] = pp
+				}
+			}
+		}
+	}
+	db.Unlock()
+
+	for _, re := range pd.GetRequires() {
+		packages, _ := db.FindPackages(re)
+		db.Lock()
+
+		for _, pa := range packages {
+			_, ok := db.RevDepsDatabase[pa.GetFingerPrint()]
+			if !ok {
+				db.RevDepsDatabase[pa.GetFingerPrint()] = make(map[string]Package)
+			}
+			db.RevDepsDatabase[pa.GetFingerPrint()][pd.GetFingerPrint()] = pd
+			_, ok = db.RevDepsDatabase[pa.GetPackageName()]
+			if !ok {
+				db.RevDepsDatabase[pa.GetPackageName()] = make(map[string]Package)
+			}
+			db.RevDepsDatabase[pa.GetPackageName()][pd.GetPackageName()] = pd
+		}
+		_, ok := db.RevDepsDatabase[re.GetFingerPrint()]
+		if !ok {
+			db.RevDepsDatabase[re.GetFingerPrint()] = make(map[string]Package)
+		}
+		db.RevDepsDatabase[re.GetFingerPrint()][pd.GetFingerPrint()] = pd
+		_, ok = db.RevDepsDatabase[re.GetPackageName()]
+		if !ok {
+			db.RevDepsDatabase[re.GetPackageName()] = make(map[string]Package)
+		}
+		db.RevDepsDatabase[re.GetPackageName()][pd.GetPackageName()] = pd
+
+		db.Unlock()
+	}
 }
 
 func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
+
 	db.Lock()
+
 	pa, ok := db.ProvidesDatabase[p.GetPackageName()][p.GetVersion()]
 	if !ok {
 		versions, ok := db.ProvidesDatabase[p.GetPackageName()]
-		db.Unlock()
+		defer db.Unlock()
 
 		if !ok {
 			return nil, errors.New("No versions found for package")
@@ -195,9 +303,18 @@ func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
 		return nil, errors.New("No package provides this")
 	}
 	db.Unlock()
+
 	return db.FindPackage(pa)
 }
 
+func (db *InMemoryDatabase) Clone(to PackageDatabase) error {
+	return clone(db, to)
+}
+
+func (db *InMemoryDatabase) Copy() (PackageDatabase, error) {
+	return copy(db)
+}
+
 func (db *InMemoryDatabase) encodePackage(p Package) (string, string, error) {
 	pd, ok := p.(*DefaultPackage)
 	if !ok {
@@ -225,7 +342,13 @@ func (db *InMemoryDatabase) FindPackage(p Package) (Package, error) {
 
 // FindPackages return the list of the packages beloging to cat/name
 func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+	db.Lock()
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
+	db.Unlock()
 	if !ok {
 		return nil, errors.New("No versions found for package")
 	}
@@ -242,29 +365,41 @@ func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) {
 
 // FindPackages return the list of the packages beloging to cat/name (any versions in requested range)
 func (db *InMemoryDatabase) FindPackages(p Package) (Packages, error) {
-
+	if !p.IsSelector() {
+		pack, err := db.FindPackage(p)
+		if err != nil {
+			return []Package{}, err
+		}
+		return []Package{pack}, nil
+	}
 	// Provides: Treat as the replaced package here
 	if provided, err := db.getProvide(p); err == nil {
 		p = provided
+		if !provided.IsSelector() {
+			return Packages{provided}, nil
+		}
 	}
+
+	db.Lock()
+	var matches []*DefaultPackage
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
+	for ve := range versions {
+		match, _ := p.SelectorMatchVersion(ve, nil)
+		if match {
+			matches = append(matches, &DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
+		}
+	}
+	db.Unlock()
 	if !ok {
 		return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 	}
 	var versionsInWorld []Package
-	for ve, _ := range versions {
-		match, err := p.SelectorMatchVersion(ve, nil)
+	for _, p := range matches {
+		w, err := db.FindPackage(p)
 		if err != nil {
-			return nil, errors.Wrap(err, "Error on match selector")
-		}
-
-		if match {
-			w, err := db.FindPackage(&DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
-			if err != nil {
-				return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")
-			}
-			versionsInWorld = append(versionsInWorld, w)
+			return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")
 		}
+		versionsInWorld = append(versionsInWorld, w)
 	}
 	return Packages(versionsInWorld), nil
 }
@@ -344,16 +479,17 @@ func (db *InMemoryDatabase) FindPackageCandidate(p Package) (Package, error) {
 
 	required, err := db.FindPackage(p)
 	if err != nil {
+		err = nil
 		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
 		packages, err := p.Expand(db)
 		//	Info("Expanded", packages, err)
 		if err != nil || len(packages) == 0 {
 			required = p
+			err = errors.Wrap(err, "Candidate not found")
 		} else {
 			required = packages.Best(nil)
-
 		}
-		return required, nil
+		return required, err
 		//required = &DefaultPackage{Name: "test"}
 	}
 

pkg/package/database_mem_test.go

@@ -0,0 +1,247 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Database", func() {
+
+	db := NewInMemoryDatabase(false)
+	Context("Simple package", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		//	a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		//		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+		//		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+		It("Saves and get data back correctly", func() {
+
+			ID, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.GetPackage(ID)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Gets all", func() {
+
+			ids := db.GetPackages()
+
+			Expect(ids).To(Equal([]string{"A-->=1.0"}))
+
+		})
+		It("Find packages", func() {
+
+			pack, err := db.FindPackage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Find best package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+
+		})
+
+		It("Find specific package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		Context("Provides", func() {
+
+			It("replaces definitions", func() {
+				db := NewInMemoryDatabase(false)
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				pack, err := db.FindPackage(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(pack).To(Equal(a3))
+			})
+
+			It("replaces definitions", func() {
+				db := NewInMemoryDatabase(false)
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(a3))
+			})
+
+			It("replaces definitions", func() {
+				db := NewInMemoryDatabase(false)
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of unexisting packages", func() {
+				db := NewInMemoryDatabase(false)
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of a required package", func() {
+				db := NewInMemoryDatabase(false)
+
+				c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = db.CreatePackage(c)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			When("Searching with selectors", func() {
+				It("replaces definitions of a required package", func() {
+					db := NewInMemoryDatabase(false)
+
+					c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+					z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+					z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+					Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+					_, err := db.CreatePackage(z)
+					Expect(err).ToNot(HaveOccurred())
+					_, err = db.CreatePackage(c)
+					Expect(err).ToNot(HaveOccurred())
+
+					s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					packs, err := db.FindPackages(s)
+					Expect(err).ToNot(HaveOccurred())
+					Expect(packs).To(ContainElement(z))
+				})
+			})
+
+		})
+
+	})
+
+})

pkg/package/database_test.go

@@ -1,127 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package pkg_test
-
-import (
-	. "github.com/mudler/luet/pkg/package"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Database", func() {
-
-	db := NewInMemoryDatabase(false)
-	Context("Simple package", func() {
-		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
-		//	a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-		//		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
-		//		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
-		It("Saves and get data back correctly", func() {
-
-			ID, err := db.CreatePackage(a)
-			Expect(err).ToNot(HaveOccurred())
-
-			pack, err := db.GetPackage(ID)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(pack).To(Equal(a))
-
-		})
-
-		It("Gets all", func() {
-
-			ids := db.GetPackages()
-
-			Expect(ids).To(Equal([]string{"A-->=1.0"}))
-
-		})
-		It("Find packages", func() {
-
-			pack, err := db.FindPackage(a)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(pack).To(Equal(a))
-
-		})
-
-		It("Find best package candidate", func() {
-			db := NewInMemoryDatabase(false)
-			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
-			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
-			_, err := db.CreatePackage(a)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a1)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a3)
-			Expect(err).ToNot(HaveOccurred())
-			s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
-
-			pack, err := db.FindPackageCandidate(s)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(pack).To(Equal(a3))
-
-		})
-
-		It("Find specific package candidate", func() {
-			db := NewInMemoryDatabase(false)
-			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
-			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
-			_, err := db.CreatePackage(a)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a1)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a3)
-			Expect(err).ToNot(HaveOccurred())
-			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
-
-			pack, err := db.FindPackageCandidate(s)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(pack).To(Equal(a))
-
-		})
-
-		It("Provides replaces definitions", func() {
-			db := NewInMemoryDatabase(false)
-			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
-			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
-
-			a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
-			Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
-
-			_, err := db.CreatePackage(a)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a1)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a3)
-			Expect(err).ToNot(HaveOccurred())
-
-			s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-
-			pack, err := db.FindPackage(s)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(pack).To(Equal(a3))
-		})
-	})
-
-})

pkg/package/package.go

@@ -40,15 +40,16 @@ import (
 // FIXME: Currently some of the methods are returning DefaultPackages due to JSON serialization of the package
 type Package interface {
 	Encode(PackageDatabase) (string, error)
+	Related(definitiondb PackageDatabase) Packages
 
 	BuildFormula(PackageDatabase, PackageDatabase) ([]bf.Formula, error)
 
 	GetFingerPrint() string
 	GetPackageName() string
+	GetPackageImageName() string
 	Requires([]*DefaultPackage) Package
 	Conflicts([]*DefaultPackage) Package
 	Revdeps(PackageDatabase) Packages
-	ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages
 	LabelDeps(PackageDatabase, string) Packages
 
 	GetProvides() []*DefaultPackage
@@ -146,6 +147,60 @@ func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) {
 	return unescaped, nil
 }
 
+type rawPackages []map[string]interface{}
+
+func (r rawPackages) Find(name, category, version string) map[string]interface{} {
+	for _, v := range r {
+		if v["name"] == name && v["category"] == category && v["version"] == version {
+			return v
+		}
+	}
+	return map[string]interface{}{}
+}
+
+func GetRawPackages(yml []byte) (rawPackages, error) {
+	var rawPackages struct {
+		Packages []map[string]interface{} `yaml:"packages"`
+	}
+	source, err := yaml.YAMLToJSON(yml)
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+	err = json.Unmarshal(bytes, &rawPackages)
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+	return rawPackages.Packages, nil
+
+}
+func DefaultPackagesFromYaml(yml []byte) ([]DefaultPackage, error) {
+
+	var unescaped struct {
+		Packages []DefaultPackage `json:"packages"`
+	}
+	source, err := yaml.YAMLToJSON(yml)
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+	err = json.Unmarshal(bytes, &unescaped)
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+	return unescaped.Packages, nil
+}
+
 // Major and minor gets escaped when marshalling in JSON, making compiler fails recognizing selectors for expansion
 func (t *DefaultPackage) JSON() ([]byte, error) {
 	buffer := &bytes.Buffer{}
@@ -234,6 +289,10 @@ func (p *DefaultPackage) GetPackageName() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.Category)
 }
 
+func (p *DefaultPackage) GetPackageImageName() string {
+	return fmt.Sprintf("%s-%s:%s", p.Name, p.Category, p.Version)
+}
+
 // GetBuildTimestamp returns the package build timestamp
 func (p *DefaultPackage) GetBuildTimestamp() string {
 	return p.BuildTimestamp
@@ -451,52 +510,47 @@ func (p *DefaultPackage) Revdeps(definitiondb PackageDatabase) Packages {
 	return versionsInWorld
 }
 
-// ExpandedRevdeps returns the package reverse dependencies,
-// matching also selectors in versions (>, <, >=, <=)
-func (p *DefaultPackage) ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages {
+func walkPackage(p Package, definitiondb PackageDatabase, visited map[string]interface{}) Packages {
 	var versionsInWorld Packages
 	if _, ok := visited[p.HumanReadableString()]; ok {
 		return versionsInWorld
 	}
 	visited[p.HumanReadableString()] = true
 
-	for _, w := range definitiondb.World() {
-		if w.Matches(p) {
-			continue
+	revdeps, _ := definitiondb.GetRevdeps(p)
+	for _, r := range revdeps {
+		versionsInWorld = append(versionsInWorld, r)
+	}
+
+	if !p.IsSelector() {
+		versionsInWorld = append(versionsInWorld, p)
+	}
+
+	for _, re := range p.GetRequires() {
+		versions, _ := re.Expand(definitiondb)
+		for _, r := range versions {
+
+			versionsInWorld = append(versionsInWorld, r)
+			versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...)
 		}
-		match := false
-
-		for _, re := range w.GetRequires() {
-			if re.Matches(p) {
-				match = true
-			}
-			if !match {
-
-				packages, _ := re.Expand(definitiondb)
-				for _, pa := range packages {
-					if pa.Matches(p) {
-						match = true
-					}
-				}
-			}
-
-			//	if ok, _ := w.RequiresContains(definitiondb, p); ok {
-
-		}
-
-		if match {
-			versionsInWorld = append(versionsInWorld, w)
-
-			versionsInWorld = append(versionsInWorld, w.ExpandedRevdeps(definitiondb, visited).Unique()...)
-
-		}
-		//	}
 
 	}
-	//visited[p.HumanReadableString()] = true
+	for _, re := range p.GetConflicts() {
+		versions, _ := re.Expand(definitiondb)
+		for _, r := range versions {
+
+			versionsInWorld = append(versionsInWorld, r)
+			versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...)
+
+		}
+	}
 	return versionsInWorld.Unique()
 }
 
+func (p *DefaultPackage) Related(definitiondb PackageDatabase) Packages {
+	return walkPackage(p, definitiondb, map[string]interface{}{})
+}
+
 func (p *DefaultPackage) LabelDeps(definitiondb PackageDatabase, labelKey string) Packages {
 	var pkgsWithLabelInWorld Packages
 	// TODO: check if integrate some index to improve
@@ -656,7 +710,7 @@ func (pack *DefaultPackage) buildFormula(definitiondb PackageDatabase, db Packag
 					C = bf.Var(encodedC)
 					// Or the Candidate is true, or all the others might be not true
 					// This forces the CDCL sat implementation to look first at a solution with C=true
-					formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Or(C, bf.Or(priorityConstraints...)), bf.Or(bf.Not(C), bf.Or(priorityALO...)))))
+					formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.And(C, bf.Or(priorityConstraints...)), bf.And(bf.Not(C), bf.Or(priorityALO...)))))
 				}
 
 				// AMO - At most one

pkg/package/package_test.go

@@ -220,8 +220,8 @@ var _ = Describe("Package", func() {
 				_, err := definitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			visited := make(map[string]interface{})
-			lst := a.ExpandedRevdeps(definitions, visited)
+			lst, err := definitions.GetRevdeps(a)
+			Expect(err).ToNot(HaveOccurred())
 			Expect(lst).To(ContainElement(c))
 			Expect(lst).To(ContainElement(d))
 			Expect(lst).To(ContainElement(e))
@@ -242,9 +242,9 @@ var _ = Describe("Package", func() {
 				_, err := definitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			visited := make(map[string]interface{})
 
-			lst := a.ExpandedRevdeps(definitions, visited)
+			lst, err := definitions.GetRevdeps(a)
+			Expect(err).ToNot(HaveOccurred())
 			Expect(lst).To(ContainElement(b))
 			Expect(lst).To(ContainElement(c))
 			Expect(lst).To(ContainElement(d))
@@ -266,9 +266,8 @@ var _ = Describe("Package", func() {
 				_, err := definitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			visited := make(map[string]interface{})
-
-			lst := a.ExpandedRevdeps(definitions, visited)
+			lst, err := definitions.GetRevdeps(a)
+			Expect(err).ToNot(HaveOccurred())
 			Expect(lst).To(ContainElement(b))
 			Expect(lst).To(ContainElement(c))
 			Expect(lst).To(ContainElement(d))

pkg/repository/loader.go

@@ -19,6 +19,7 @@ package repository
 import (
 	"io/ioutil"
 	"path"
+	"path/filepath"
 	"regexp"
 
 	"github.com/ghodss/yaml"
@@ -29,8 +30,21 @@ import (
 
 func LoadRepositories(c *LuetConfig) error {
 	var regexRepo = regexp.MustCompile(`.yml$|.yaml$`)
+	var err error
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
 
 	for _, rdir := range c.RepositoriesConfDir {
+
+		rdir = filepath.Join(rootfs, rdir)
+
 		Debug("Parsing Repository Directory", rdir, "...")
 
 		files, err := ioutil.ReadDir(rdir)

pkg/solver/benchmark_test.go

@@ -0,0 +1,298 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/tests/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "github.com/mudler/luet/pkg/solver"
+)
+
+var _ = Describe("Solver Benchmarks", func() {
+	db := pkg.NewInMemoryDatabase(false)
+	dbInstalled := pkg.NewInMemoryDatabase(false)
+	dbDefinitions := pkg.NewInMemoryDatabase(false)
+	var s PackageSolver
+
+	Context("Complex data sets", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+		Measure("it should be fast in resolution from a 50000 dataset", func(b Benchmarker) {
+
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 50000; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					_, err := dbInstalled.CreatePackage(C)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for i := 0; i < 1; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+					solution, err := s.Install([]pkg.Package{A})
+					Expect(err).ToNot(HaveOccurred())
+
+					Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: H, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: G, Value: true}))
+				}
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 120), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+	Context("Complex data sets - Parallel", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: ParallelSimple, Concurrency: 10}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+		Measure("it should be fast in resolution from a 50000 dataset", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 50000; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					_, err := dbInstalled.CreatePackage(C)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				for i := 0; i < 1; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+					solution, err := s.Install([]pkg.Package{A})
+					Expect(err).ToNot(HaveOccurred())
+
+					Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: H, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: G, Value: true}))
+
+					//	Expect(len(solution)).To(Equal(6))
+				}
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+	Context("Complex data sets - Parallel Upgrades", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			//	dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: ParallelSimple, Concurrency: 100}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+			tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			dbInstalled = pkg.NewBoltDatabase(tmpfile.Name())
+		})
+
+		Measure("it should be fast in resolution from a 10000*8 dataset", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 2; i < 10000; i++ {
+					C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G, H} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+				}
+
+				//C := pkg.NewPackage("C", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				G := pkg.NewPackage("G", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H := pkg.NewPackage("H", "1", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "1", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "1", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				_, err := dbInstalled.CreatePackage(A)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(B)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(D)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(H)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(G)
+				Expect(err).ToNot(HaveOccurred())
+				fmt.Println("Upgrade starts")
+
+				packages, ass, err := s.Upgrade(false, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(packages).To(ContainElement(A))
+
+				G = pkg.NewPackage("G", "9999", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H = pkg.NewPackage("H", "9999", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D = pkg.NewPackage("D", "9999", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B = pkg.NewPackage("B", "9999", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A = pkg.NewPackage("A", "9999", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				Expect(ass).To(ContainElement(PackageAssert{Package: A, Value: true}))
+
+				Expect(len(packages)).To(Equal(5))
+				//	Expect(len(solution)).To(Equal(6))
+
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+
+		Measure("it should be fast in installation with 12000 packages installed and 2000*8 available", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 2000; i++ {
+					C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					fmt.Println("Creating package, run", i)
+				}
+
+				for i := 0; i < 12000; i++ {
+					x := helpers.RandomPackage()
+					_, err := dbInstalled.CreatePackage(x)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				G := pkg.NewPackage("G", strconv.Itoa(50000), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H := pkg.NewPackage("H", strconv.Itoa(50000), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", strconv.Itoa(50000), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", strconv.Itoa(50000), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", strconv.Itoa(50000), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				ass, err := s.Install([]pkg.Package{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(ass).To(ContainElement(PackageAssert{Package: pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}), Value: true}))
+				//Expect(ass).To(Equal(5))
+				//	Expect(len(solution)).To(Equal(6))
+
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+
+		PMeasure("it should be fast in resolution from a 50000 dataset with upgrade universe", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 2; i++ {
+					C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					fmt.Println("Creating package, run", i)
+				}
+
+				G := pkg.NewPackage("G", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H := pkg.NewPackage("H", "1", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "1", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "1", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				_, err := dbInstalled.CreatePackage(A)
+				Expect(err).ToNot(HaveOccurred())
+				fmt.Println("Upgrade starts")
+
+				packages, ass, err := s.UpgradeUniverse(true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(ass).To(ContainElement(PackageAssert{Package: pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}), Value: true}))
+				Expect(packages).To(ContainElement(pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})))
+				Expect(packages).To(Equal(5))
+				//	Expect(len(solution)).To(Equal(6))
+
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+})

pkg/solver/decoder.go

@@ -22,9 +22,9 @@ import (
 	"unicode"
 
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/topsort"
 	toposort "github.com/philopon/go-toposort"
 	"github.com/pkg/errors"
-	"github.com/stevenle/topsort"
 )
 
 type PackagesAssertions []PackageAssert
@@ -150,22 +150,13 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 
 	orderedAssertions := PackagesAssertions{}
 	unorderedAssertions := PackagesAssertions{}
-	fingerprints := []string{}
 
 	tmpMap := map[string]PackageAssert{}
 	graph := topsort.NewGraph()
-
 	for _, a := range assertions {
 		graph.AddNode(a.Package.GetFingerPrint())
 		tmpMap[a.Package.GetFingerPrint()] = a
-		fingerprints = append(fingerprints, a.Package.GetFingerPrint())
 		unorderedAssertions = append(unorderedAssertions, a) // Build a list of the ones that must be ordered
-
-		if a.Value {
-			unorderedAssertions = append(unorderedAssertions, a) // Build a list of the ones that must be ordered
-		} else {
-			orderedAssertions = append(orderedAssertions, a) // Keep last the ones which are not meant to be installed
-		}
 	}
 
 	sort.Sort(unorderedAssertions)
@@ -190,7 +181,7 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 			}
 			// Expand also here, as we need to order them (or instead the solver should give back the dep correctly?)
 			graph.AddEdge(currentPkg.GetFingerPrint(), requiredDef.GetFingerPrint())
-			added[requiredDef.GetFingerPrint()] = nil
+			added[requiredDef.GetFingerPrint()] = true
 		}
 	}
 	result, err := graph.TopSort(fingerprint)
@@ -200,8 +191,11 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 	for _, res := range result {
 		a, ok := tmpMap[res]
 		if !ok {
-			return nil, errors.New("fail looking for " + res)
-			//	continue
+			//return nil, errors.New("fail looking for " + res)
+			// Since now we don't return the entire world as part of assertions
+			// if we don't find any reference must be because fingerprint we are analyzing (which is the one we are ordering against)
+			// is not part of the assertions, thus we can omit it from the result
+			continue
 		}
 		orderedAssertions = append(orderedAssertions, a)
 		//	orderedAssertions = append(PackagesAssertions{a}, orderedAssertions...) // push upfront

pkg/solver/decoder_test.go

@@ -30,13 +30,13 @@ var _ = Describe("Decoder", func() {
 	db := pkg.NewInMemoryDatabase(false)
 	dbInstalled := pkg.NewInMemoryDatabase(false)
 	dbDefinitions := pkg.NewInMemoryDatabase(false)
-	s := NewSolver(dbInstalled, dbDefinitions, db)
+	s := NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 	BeforeEach(func() {
 		db = pkg.NewInMemoryDatabase(false)
 		dbInstalled = pkg.NewInMemoryDatabase(false)
 		dbDefinitions = pkg.NewInMemoryDatabase(false)
-		s = NewSolver(dbInstalled, dbDefinitions, db)
+		s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 	})
 
 	Context("Assertion ordering", func() {
@@ -214,12 +214,10 @@ var _ = Describe("Decoder", func() {
 			hash2 := solution.AssertionHash()
 
 			//	Expect(len(solution)).To(Equal(6))
-			Expect(solution[0].Package.GetName()).To(Equal("A"))
-			Expect(solution[1].Package.GetName()).To(Equal("G"))
-			Expect(solution[2].Package.GetName()).To(Equal("H"))
-			Expect(solution[3].Package.GetName()).To(Equal("D"))
-			Expect(solution[4].Package.GetName()).To(Equal("B"))
-			Expect(solution[0].Value).ToNot(BeTrue())
+			Expect(solution[0].Package.GetName()).To(Equal("G"))
+			Expect(solution[1].Package.GetName()).To(Equal("H"))
+			Expect(solution[2].Package.GetName()).To(Equal("D"))
+			Expect(solution[3].Package.GetName()).To(Equal("B"))
 
 			Expect(hash).ToNot(Equal(""))
 			Expect(hash2).ToNot(Equal(""))
@@ -390,5 +388,37 @@ var _ = Describe("Decoder", func() {
 
 			Expect(solution4.Drop(Y).AssertionHash()).To(Equal(solution4.HashFrom(Y)))
 		})
+		for index := 0; index < 300; index++ { // Just to make sure we don't have false positives
+
+			It("Always same solution", func() {
+
+				X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+				Z := pkg.NewPackage("Z", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+				W := pkg.NewPackage("W", "", []*pkg.DefaultPackage{Z, Y}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{X, Y, Z} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{W})
+				Expect(err).ToNot(HaveOccurred())
+
+				orderW, err := solution.Order(dbDefinitions, W.GetFingerPrint())
+				Expect(err).ToNot(HaveOccurred())
+				Expect(orderW[0].Package.GetName()).To(Equal("X"))
+				Expect(orderW[1].Package.GetName()).To(Equal("Y"))
+				Expect(orderW[2].Package.GetName()).To(Equal("Z"))
+				Expect(orderW[3].Package.GetName()).To(Equal("W"))
+			})
+
+		}
+
 	})
 })

pkg/solver/parallel.go

@@ -0,0 +1,860 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver
+
+import (
+
+	//. "github.com/mudler/luet/pkg/logger"
+	"fmt"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"github.com/crillab/gophersat/bf"
+	pkg "github.com/mudler/luet/pkg/package"
+)
+
+// Parallel is the default Parallel for luet
+type Parallel struct {
+	Concurrency        int
+	DefinitionDatabase pkg.PackageDatabase
+	ParallelDatabase   pkg.PackageDatabase
+	Wanted             pkg.Packages
+	InstalledDatabase  pkg.PackageDatabase
+
+	Resolver PackageResolver
+}
+
+func (s *Parallel) SetDefinitionDatabase(db pkg.PackageDatabase) {
+	s.DefinitionDatabase = db
+}
+
+// SetReSolver is a setter for the unsat ReSolver backend
+func (s *Parallel) SetResolver(r PackageResolver) {
+	s.Resolver = r
+}
+
+func (s *Parallel) World() pkg.Packages {
+	return s.DefinitionDatabase.World()
+}
+
+func (s *Parallel) Installed() pkg.Packages {
+
+	return s.InstalledDatabase.World()
+}
+
+func (s *Parallel) noRulesWorld() bool {
+	for _, p := range s.World() {
+		if len(p.GetConflicts()) != 0 || len(p.GetRequires()) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (s *Parallel) noRulesInstalled() bool {
+	for _, p := range s.Installed() {
+		if len(p.GetConflicts()) != 0 || len(p.GetRequires()) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (s *Parallel) buildParallelFormula(db pkg.PackageDatabase, formulas []bf.Formula, packages pkg.Packages) (bf.Formula, error) {
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				solvable, err := p.BuildFormula(db, s.ParallelDatabase)
+				if err != nil {
+					panic(err)
+				}
+				for _, s := range solvable {
+					results <- s
+				}
+			}
+		}(wg, all)
+	}
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	for _, p := range packages {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	if len(formulas) != 0 {
+		return bf.And(formulas...), nil
+	}
+	return bf.True, nil
+}
+
+func (s *Parallel) BuildInstalled() (bf.Formula, error) {
+	var formulas []bf.Formula
+
+	var packages pkg.Packages
+	for _, p := range s.Installed() {
+		packages = append(packages, p)
+		for _, dep := range p.Related(s.InstalledDatabase) {
+			packages = append(packages, dep)
+		}
+
+	}
+
+	return s.buildParallelFormula(s.InstalledDatabase, formulas, packages)
+}
+
+// BuildWorld builds the formula which olds the requirements from the package definitions
+// which are available (global state)
+func (s *Parallel) BuildWorld(includeInstalled bool) (bf.Formula, error) {
+	var formulas []bf.Formula
+	// NOTE: This block should be enabled in case of very old systems with outdated world sets
+	if includeInstalled {
+		solvable, err := s.BuildInstalled()
+		if err != nil {
+			return nil, err
+		}
+		//f = bf.And(f, solvable)
+		formulas = append(formulas, solvable)
+	}
+	return s.buildParallelFormula(s.DefinitionDatabase, formulas, s.World())
+}
+
+// BuildWorld builds the formula which olds the requirements from the package definitions
+// which are available (global state)
+func (s *Parallel) BuildPartialWorld(includeInstalled bool) (bf.Formula, error) {
+	var formulas []bf.Formula
+	// NOTE: This block should be enabled in case of very old systems with outdated world sets
+	if includeInstalled {
+		solvable, err := s.BuildInstalled()
+		if err != nil {
+			return nil, err
+		}
+		//f = bf.And(f, solvable)
+		formulas = append(formulas, solvable)
+	}
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+	var packages pkg.Packages
+
+	all := make(chan pkg.Package)
+	results := make(chan pkg.Package, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				for _, dep := range p.Related(s.DefinitionDatabase) {
+					results <- dep
+				}
+
+			}
+		}(wg, all)
+	}
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			packages = append(packages, t)
+		}
+	}()
+
+	for _, p := range s.Wanted {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	return s.buildParallelFormula(s.DefinitionDatabase, formulas, packages)
+
+	//return s.buildParallelFormula(formulas, s.World())
+}
+
+func (s *Parallel) getList(db pkg.PackageDatabase, lsp pkg.Packages) (pkg.Packages, error) {
+	var ls pkg.Packages
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan pkg.Package, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				cp, err := db.FindPackage(p)
+				if err != nil {
+					packages, err := p.Expand(db)
+					// Expand, and relax search - if not found pick the same one
+					if err != nil || len(packages) == 0 {
+						cp = p
+					} else {
+						cp = packages.Best(nil)
+					}
+				}
+				results <- cp
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func(wg *sync.WaitGroup) {
+		defer wg2.Done()
+		for t := range results {
+			ls = append(ls, t)
+		}
+	}(wg)
+
+	for _, pp := range lsp {
+		all <- pp
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	return ls, nil
+}
+
+// Conflicts acts like ConflictsWith, but uses package's reverse dependencies to
+// determine if it conflicts with the given set
+func (s *Parallel) Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error) {
+	p, err := s.DefinitionDatabase.FindPackage(pack)
+	if err != nil {
+		p = pack
+	}
+
+	ls, err := s.getList(s.DefinitionDatabase, lsp)
+	if err != nil {
+		return false, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	if s.noRulesWorld() {
+		return false, nil
+	}
+
+	temporarySet := pkg.NewInMemoryDatabase(false)
+	for _, p := range ls {
+		temporarySet.CreatePackage(p)
+	}
+
+	revdeps, err := temporarySet.GetRevdeps(p)
+	if err != nil {
+		return false, errors.Wrap(err, "error scanning revdeps")
+	}
+	var revdepsErr error
+	for _, r := range revdeps {
+		if revdepsErr == nil {
+			revdepsErr = errors.New("")
+		}
+		revdepsErr = errors.New(fmt.Sprintf("%s\n%s", revdepsErr.Error(), r.HumanReadableString()))
+	}
+
+	return len(revdeps) != 0, revdepsErr
+}
+
+// ConflictsWith return true if a package is part of the requirement set of a list of package
+// return false otherwise (and thus it is NOT relevant to the given list)
+func (s *Parallel) ConflictsWith(pack pkg.Package, lsp pkg.Packages) (bool, error) {
+	p, err := s.DefinitionDatabase.FindPackage(pack)
+	if err != nil {
+		p = pack //Relax search, otherwise we cannot compute solutions for packages not in definitions
+	}
+
+	ls, err := s.getList(s.DefinitionDatabase, lsp)
+	if err != nil {
+		return false, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	var formulas []bf.Formula
+
+	if s.noRulesWorld() {
+		return false, nil
+	}
+
+	encodedP, err := p.Encode(s.ParallelDatabase)
+	if err != nil {
+		return false, err
+	}
+	P := bf.Var(encodedP)
+
+	r, err := s.BuildWorld(false)
+	if err != nil {
+		return false, err
+	}
+	formulas = append(formulas, bf.And(bf.Not(P), r))
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for i := range c {
+				if i.Matches(p) {
+					continue
+				}
+
+				// XXX: Skip check on any of its requires ?  ( Drop to avoid removing system packages when selecting an uninstall)
+				// if i.RequiresContains(p) {
+				// 	fmt.Println("Requires found")
+				// 	continue
+				// }
+
+				encodedI, err := i.Encode(s.ParallelDatabase)
+				if err != nil {
+					panic(err)
+				}
+				I := bf.Var(encodedI)
+
+				results <- bf.And(I, r)
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	for _, p := range ls {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return true, nil
+	}
+
+	return false, nil
+
+}
+
+func (s *Parallel) ConflictsWithInstalled(p pkg.Package) (bool, error) {
+	return s.ConflictsWith(p, s.Installed())
+}
+
+// UninstallUniverse takes a list of candidate package and return a list of packages that would be removed
+// in order to purge the candidate. Uses the Parallel to check constraints and nothing else
+//
+// It can be compared to the counterpart Uninstall as this method acts like a uninstall --full
+// it removes all the packages and its deps. taking also in consideration other packages that might have
+// revdeps
+func (s *Parallel) UninstallUniverse(toremove pkg.Packages) (pkg.Packages, error) {
+
+	if s.noRulesInstalled() {
+		return s.getList(s.InstalledDatabase, toremove)
+	}
+
+	// resolve to packages from the db
+	toRemove, err := s.getList(s.InstalledDatabase, toremove)
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	var formulas []bf.Formula
+	r, err := s.BuildInstalled()
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	// SAT encode the clauses against the world
+	for _, p := range toRemove.Unique() {
+		encodedP, err := p.Encode(s.InstalledDatabase)
+		if err != nil {
+			return nil, errors.Wrap(err, "Package not found in definition db")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(bf.Not(P), r))
+	}
+
+	markedForRemoval := pkg.Packages{}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, errors.New("Failed finding a solution")
+	}
+	assertion, err := DecodeModel(model, s.InstalledDatabase)
+	if err != nil {
+		return nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+	}
+	return markedForRemoval, nil
+}
+
+// UpgradeUniverse mark packages for removal and returns a solution. It considers
+// the Universe db as authoritative
+// See also on the subject: https://arxiv.org/pdf/1007.1021.pdf
+func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssertions, error) {
+	var formulas []bf.Formula
+	// we first figure out which aren't up-to-date
+	// which has to be removed
+	// and which needs to be upgraded
+	removed := pkg.Packages{}
+
+	// TODO: this is memory expensive, we need to optimize this
+	universe, err := s.DefinitionDatabase.Copy()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't build world copy")
+	}
+	for _, p := range s.Installed() {
+		universe.CreatePackage(p)
+	}
+
+	// Build constraints for the whole defdb
+	r, err := s.BuildWorld(true)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't build world constraints")
+	}
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				available, err := s.DefinitionDatabase.FindPackageVersions(p)
+				if len(available) == 0 || err != nil {
+					removed = append(removed, p)
+					continue
+				}
+
+				bestmatch := available.Best(nil)
+				// Found a better version available
+				if !bestmatch.Matches(p) {
+					oldP, _ := p.Encode(universe)
+					toreplaceP := bf.Var(oldP)
+					best, _ := bestmatch.Encode(universe)
+					toUpgrade := bf.Var(best)
+
+					solvablenew, _ := bestmatch.BuildFormula(s.DefinitionDatabase, s.ParallelDatabase)
+					results <- bf.And(bf.Not(toreplaceP), bf.And(append(solvablenew, toUpgrade)...))
+				}
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	// Grab all the installed ones, see if they are eligible for update
+	for _, p := range s.Installed() {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	// Treat removed packages from universe as marked for deletion
+	if dropremoved {
+
+		for _, p := range removed.Unique() {
+			encodedP, err := p.Encode(universe)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "couldn't encode package")
+			}
+			P := bf.Var(encodedP)
+			formulas = append(formulas, bf.And(bf.Not(P), r))
+		}
+	}
+
+	markedForRemoval := pkg.Packages{}
+	if len(formulas) == 0 {
+		return pkg.Packages{}, PackagesAssertions{}, nil
+	}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, nil, errors.New("Failed finding a solution")
+	}
+
+	assertion, err := DecodeModel(model, universe)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+
+	}
+	return markedForRemoval, assertion, nil
+}
+
+// Upgrade compute upgrades of the package against the world definition.
+// It accepts two boolean indicating if it has to check for conflicts or try to attempt a full upgrade
+func (s *Parallel) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAssertions, error) {
+
+	// First get candidates that needs to be upgraded..
+
+	toUninstall := pkg.Packages{}
+	toInstall := pkg.Packages{}
+
+	// we do this in memory so we take into account of provides
+	universe, err := s.DefinitionDatabase.Copy()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Could not copy def db")
+	}
+
+	installedcopy := pkg.NewInMemoryDatabase(false)
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan []pkg.Package, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for p := range c {
+				installedcopy.CreatePackage(p)
+				packages, err := universe.FindPackageVersions(p)
+				if err == nil && len(packages) != 0 {
+					best := packages.Best(nil)
+					if !best.Matches(p) {
+						results <- []pkg.Package{p, best}
+					}
+				}
+			}
+		}(wg, all)
+	}
+
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			toUninstall = append(toUninstall, t[0])
+			toInstall = append(toInstall, t[1])
+		}
+	}()
+
+	for _, p := range s.InstalledDatabase.World() {
+		all <- p
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	s2 := &Parallel{Concurrency: s.Concurrency, InstalledDatabase: installedcopy, DefinitionDatabase: s.DefinitionDatabase, ParallelDatabase: pkg.NewInMemoryDatabase(false)}
+	s2.SetResolver(s.Resolver)
+	if !full {
+		ass := PackagesAssertions{}
+		for _, i := range toInstall {
+			ass = append(ass, PackageAssert{Package: i.(*pkg.DefaultPackage), Value: true})
+		}
+	}
+
+	// Then try to uninstall the versions in the system, and store that tree
+	r, err := s.Uninstall(checkconflicts, false, toUninstall...)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't uninstall candidates ")
+	}
+	for _, z := range r {
+		err = installedcopy.RemovePackage(z)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't remove copy of package targetted for removal")
+		}
+	}
+
+	if len(toInstall) == 0 {
+		return toUninstall, PackagesAssertions{}, nil
+	}
+	assertions, e := s2.Install(toInstall)
+	return toUninstall, assertions, e
+	// To that tree, ask to install the versions that should be upgraded, and try to solve
+	// Return the solution
+
+}
+
+// Uninstall takes a candidate package and return a list of packages that would be removed
+// in order to purge the candidate. Returns error if unsat.
+func (s *Parallel) Uninstall(checkconflicts, full bool, packs ...pkg.Package) (pkg.Packages, error) {
+	if len(packs) == 0 {
+		return pkg.Packages{}, nil
+	}
+	var res pkg.Packages
+	toRemove := pkg.Packages{}
+
+	for _, c := range packs {
+		candidate, err := s.InstalledDatabase.FindPackage(c)
+		if err != nil {
+
+			//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
+			packages, err := c.Expand(s.InstalledDatabase)
+			//	Info("Expanded", packages, err)
+			if err != nil || len(packages) == 0 {
+				candidate = c
+			} else {
+				candidate = packages.Best(nil)
+			}
+			//Relax search, otherwise we cannot compute solutions for packages not in definitions
+			//	return nil, errors.Wrap(err, "Package not found between installed")
+		}
+
+		toRemove = append(toRemove, candidate)
+	}
+	// Build a fake "Installed" - Candidate and its requires tree
+	var InstalledMinusCandidate pkg.Packages
+
+	// We are asked to not perform a full uninstall (checking all the possible requires that could
+	// be removed). Let's only check if we can remove the selected package
+	if !full && checkconflicts {
+		for _, candidate := range toRemove {
+			if conflicts, err := s.Conflicts(candidate, s.Installed()); conflicts {
+				return nil, err
+			}
+		}
+		return toRemove, nil
+	}
+
+	// TODO: Can be optimized
+	for _, i := range s.Installed() {
+		matched := false
+		for _, candidate := range toRemove {
+			if !i.Matches(candidate) {
+				contains, err := candidate.RequiresContains(s.ParallelDatabase, i)
+				if err != nil {
+					return nil, errors.Wrap(err, "Failed getting installed list")
+				}
+				if !contains {
+					matched = true
+				}
+
+			}
+		}
+		if matched {
+			InstalledMinusCandidate = append(InstalledMinusCandidate, i)
+		}
+	}
+
+	s2 := &Parallel{Concurrency: s.Concurrency, InstalledDatabase: pkg.NewInMemoryDatabase(false), DefinitionDatabase: s.InstalledDatabase, ParallelDatabase: pkg.NewInMemoryDatabase(false)}
+	s2.SetResolver(s.Resolver)
+	// Get the requirements to install the candidate
+	asserts, err := s2.Install(toRemove)
+	if err != nil {
+		return nil, err
+	}
+	for _, a := range asserts {
+		if a.Value {
+			if !checkconflicts {
+				res = append(res, a.Package)
+				continue
+			}
+
+			c, err := s.ConflictsWithInstalled(a.Package)
+			if err != nil {
+				return nil, err
+			}
+
+			// If doesn't conflict with installed we just consider it for removal and look for the next one
+			if !c {
+				res = append(res, a.Package)
+				continue
+			}
+
+			// If does conflicts, give it another chance by checking conflicts if in case we didn't installed our candidate and all the required packages in the system
+			c, err = s.ConflictsWith(a.Package, InstalledMinusCandidate)
+			if err != nil {
+				return nil, err
+			}
+			if !c {
+				res = append(res, a.Package)
+			}
+
+		}
+
+	}
+
+	return res, nil
+}
+
+// BuildFormula builds the main solving formula that is evaluated by the sat Parallel.
+func (s *Parallel) BuildFormula() (bf.Formula, error) {
+	var formulas []bf.Formula
+
+	r, err := s.BuildWorld(false)
+	if err != nil {
+		return nil, err
+	}
+
+	var wg = new(sync.WaitGroup)
+	var wg2 = new(sync.WaitGroup)
+
+	all := make(chan pkg.Package)
+	results := make(chan bf.Formula, 1)
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
+			defer wg.Done()
+			for wanted := range c {
+				encodedW, err := wanted.Encode(s.ParallelDatabase)
+				if err != nil {
+					panic(err)
+				}
+				W := bf.Var(encodedW)
+				installedWorld := s.Installed()
+				//TODO:Optimize
+				if len(installedWorld) == 0 {
+					results <- W
+					continue
+				}
+
+				for _, installed := range installedWorld {
+					encodedI, err := installed.Encode(s.ParallelDatabase)
+					if err != nil {
+						panic(err)
+					}
+					I := bf.Var(encodedI)
+					results <- bf.And(W, I)
+				}
+			}
+		}(wg, all)
+	}
+	wg2.Add(1)
+	go func() {
+		defer wg2.Done()
+		for t := range results {
+			formulas = append(formulas, t)
+		}
+	}()
+
+	for _, wanted := range s.Wanted {
+		all <- wanted
+	}
+
+	close(all)
+	wg.Wait()
+	close(results)
+	wg2.Wait()
+
+	formulas = append(formulas, r)
+
+	return bf.And(formulas...), nil
+}
+
+func (s *Parallel) solve(f bf.Formula) (map[string]bool, bf.Formula, error) {
+	model := bf.Solve(f)
+	if model == nil {
+		return model, f, errors.New("Unsolvable")
+	}
+
+	return model, f, nil
+}
+
+// Solve builds the formula given the current state and returns package assertions
+func (s *Parallel) Solve() (PackagesAssertions, error) {
+	var model map[string]bool
+	var err error
+
+	f, err := s.BuildFormula()
+
+	if err != nil {
+		return nil, err
+	}
+
+	model, _, err = s.solve(f)
+	if err != nil && s.Resolver != nil {
+		return s.Resolver.Solve(f, s)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return DecodeModel(model, s.ParallelDatabase)
+}
+
+// Install given a list of packages, returns package assertions to indicate the packages that must be installed in the system in order
+// to statisfy all the constraints
+func (s *Parallel) Install(c pkg.Packages) (PackagesAssertions, error) {
+
+	coll, err := s.getList(s.DefinitionDatabase, c)
+	if err != nil {
+		return nil, errors.Wrap(err, "Packages not found in definition db")
+	}
+
+	s.Wanted = coll
+
+	if s.noRulesWorld() {
+		var ass PackagesAssertions
+		for _, p := range s.Installed() {
+			ass = append(ass, PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
+
+		}
+		for _, p := range s.Wanted {
+			ass = append(ass, PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
+		}
+		return ass, nil
+	}
+
+	return s.Solve()
+}

pkg/solver/resolver_test.go

@@ -28,13 +28,13 @@ var _ = Describe("Resolver", func() {
 	db := pkg.NewInMemoryDatabase(false)
 	dbInstalled := pkg.NewInMemoryDatabase(false)
 	dbDefinitions := pkg.NewInMemoryDatabase(false)
-	s := NewSolver(dbInstalled, dbDefinitions, db)
+	s := NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 	BeforeEach(func() {
 		db = pkg.NewInMemoryDatabase(false)
 		dbInstalled = pkg.NewInMemoryDatabase(false)
 		dbDefinitions = pkg.NewInMemoryDatabase(false)
-		s = NewSolver(dbInstalled, dbDefinitions, db)
+		s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 	})
 
 	Context("Conflict set", func() {
@@ -81,11 +81,11 @@ var _ = Describe("Resolver", func() {
 
 				Expect(len(solution)).To(Equal(6))
 
-				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
-				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
-				Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
 
 			})
@@ -112,8 +112,8 @@ var _ = Describe("Resolver", func() {
 				solution, err := s.Install([]pkg.Package{A, D})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
-				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
 
@@ -142,14 +142,13 @@ var _ = Describe("Resolver", func() {
 				solution, err := s.Install([]pkg.Package{A, D, E, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
-				Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) // Was already installed
 				Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
-				Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
+				Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 				Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
 				Expect(len(solution)).To(Equal(6))
-
 			})
 		})
 

pkg/solver/solver.go

@@ -26,11 +26,18 @@ import (
 	pkg "github.com/mudler/luet/pkg/package"
 )
 
+type SolverType int
+
+const (
+	SingleCoreSimple = 0
+	ParallelSimple   = iota
+)
+
 // PackageSolver is an interface to a generic package solving algorithm
 type PackageSolver interface {
 	SetDefinitionDatabase(pkg.PackageDatabase)
 	Install(p pkg.Packages) (PackagesAssertions, error)
-	Uninstall(candidate pkg.Package, checkconflicts, full bool) (pkg.Packages, error)
+	Uninstall(checkconflicts, full bool, candidate ...pkg.Package) (pkg.Packages, error)
 	ConflictsWithInstalled(p pkg.Package) (bool, error)
 	ConflictsWith(p pkg.Package, ls pkg.Packages) (bool, error)
 	Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error)
@@ -56,16 +63,30 @@ type Solver struct {
 	Resolver PackageResolver
 }
 
-// NewSolver accepts as argument two lists of packages, the first is the initial set,
-// the second represent all the known packages.
-func NewSolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase) PackageSolver {
-	return NewResolver(installed, definitiondb, solverdb, &DummyPackageResolver{})
+type Options struct {
+	Type        SolverType
+	Concurrency int
 }
 
-// NewReSolver accepts as argument two lists of packages, the first is the initial set,
+// NewSolver accepts as argument two lists of packages, the first is the initial set,
 // the second represent all the known packages.
-func NewResolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase, re PackageResolver) PackageSolver {
-	return &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb, Resolver: re}
+func NewSolver(t Options, installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase) PackageSolver {
+	return NewResolver(t, installed, definitiondb, solverdb, &DummyPackageResolver{})
+}
+
+// NewResolver accepts as argument two lists of packages, the first is the initial set,
+// the second represent all the known packages.
+// Using constructors as in the future we foresee warmups for hot-restore solver cache
+func NewResolver(t Options, installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase, re PackageResolver) PackageSolver {
+	var s PackageSolver
+	switch t.Type {
+	case SingleCoreSimple:
+		s = &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb, Resolver: re}
+	case ParallelSimple:
+		s = &Parallel{InstalledDatabase: installed, DefinitionDatabase: definitiondb, ParallelDatabase: solverdb, Resolver: re, Concurrency: t.Concurrency}
+	}
+
+	return s
 }
 
 // SetDefinitionDatabase is a setter for the definition Database
@@ -110,8 +131,16 @@ func (s *Solver) noRulesInstalled() bool {
 
 func (s *Solver) BuildInstalled() (bf.Formula, error) {
 	var formulas []bf.Formula
+	var packages pkg.Packages
 	for _, p := range s.Installed() {
-		solvable, err := p.BuildFormula(s.DefinitionDatabase, s.SolverDatabase)
+		packages = append(packages, p)
+		for _, dep := range p.Related(s.InstalledDatabase) {
+			packages = append(packages, dep)
+		}
+	}
+
+	for _, p := range packages {
+		solvable, err := p.BuildFormula(s.InstalledDatabase, s.SolverDatabase)
 		if err != nil {
 			return nil, err
 		}
@@ -147,6 +176,44 @@ func (s *Solver) BuildWorld(includeInstalled bool) (bf.Formula, error) {
 	return bf.And(formulas...), nil
 }
 
+// BuildWorld builds the formula which olds the requirements from the package definitions
+// which are available (global state)
+func (s *Solver) BuildPartialWorld(includeInstalled bool) (bf.Formula, error) {
+	var formulas []bf.Formula
+	// NOTE: This block shouldf be enabled in case of very old systems with outdated world sets
+	if includeInstalled {
+		solvable, err := s.BuildInstalled()
+		if err != nil {
+			return nil, err
+		}
+		//f = bf.And(f, solvable)
+		formulas = append(formulas, solvable)
+	}
+
+	var packages pkg.Packages
+	for _, p := range s.Wanted {
+		//	packages = append(packages, p)
+		for _, dep := range p.Related(s.DefinitionDatabase) {
+			packages = append(packages, dep)
+		}
+
+	}
+
+	for _, p := range packages {
+		solvable, err := p.BuildFormula(s.DefinitionDatabase, s.SolverDatabase)
+		if err != nil {
+			return nil, err
+		}
+		formulas = append(formulas, solvable...)
+	}
+
+	if len(formulas) != 0 {
+		return bf.And(formulas...), nil
+	}
+
+	return bf.True, nil
+}
+
 func (s *Solver) getList(db pkg.PackageDatabase, lsp pkg.Packages) (pkg.Packages, error) {
 	var ls pkg.Packages
 
@@ -187,15 +254,18 @@ func (s *Solver) Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error) {
 	for _, p := range ls {
 		temporarySet.CreatePackage(p)
 	}
-	visited := make(map[string]interface{})
-	revdeps := p.ExpandedRevdeps(temporarySet, visited)
+
+	revdeps, err := temporarySet.GetRevdeps(p)
+	if err != nil {
+		return false, errors.Wrap(err, "error scanning revdeps")
+	}
 
 	var revdepsErr error
 	for _, r := range revdeps {
 		if revdepsErr == nil {
 			revdepsErr = errors.New("")
 		}
-		revdepsErr = errors.New(fmt.Sprintf("%s\n%s", revdepsErr.Error(), r.HumanReadableString()))
+		revdepsErr = fmt.Errorf("%s\n%s", revdepsErr.Error(), r.HumanReadableString())
 	}
 
 	return len(revdeps) != 0, revdepsErr
@@ -207,7 +277,6 @@ func (s *Solver) ConflictsWith(pack pkg.Package, lsp pkg.Packages) (bool, error)
 	p, err := s.DefinitionDatabase.FindPackage(pack)
 	if err != nil {
 		p = pack //Relax search, otherwise we cannot compute solutions for packages not in definitions
-
 		//	return false, errors.Wrap(err, "Package not found in definition db")
 	}
 
@@ -328,23 +397,23 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert
 	notUptodate := pkg.Packages{}
 	removed := pkg.Packages{}
 	toUpgrade := pkg.Packages{}
+	replacements := map[pkg.Package]pkg.Package{}
 
 	// TODO: this is memory expensive, we need to optimize this
-	universe := pkg.NewInMemoryDatabase(false)
-	for _, p := range s.DefinitionDatabase.World() {
-		universe.CreatePackage(p)
+	universe, err := s.DefinitionDatabase.Copy()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Failed copying db")
 	}
+
 	for _, p := range s.Installed() {
 		universe.CreatePackage(p)
 	}
 
 	// Grab all the installed ones, see if they are eligible for update
 	for _, p := range s.Installed() {
-		available, err := universe.FindPackageVersions(p)
-		if err != nil {
+		available, err := s.DefinitionDatabase.FindPackageVersions(p)
+		if len(available) == 0 || err != nil {
 			removed = append(removed, p)
-		}
-		if len(available) == 0 {
 			continue
 		}
 
@@ -353,20 +422,10 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert
 		if !bestmatch.Matches(p) {
 			notUptodate = append(notUptodate, p)
 			toUpgrade = append(toUpgrade, bestmatch)
+			replacements[p] = bestmatch
 		}
 	}
 
-	// resolve to packages from the db to be able to encode correctly
-	oldPackages, err := s.getList(universe, notUptodate)
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "couldn't get package marked for removal from universe")
-	}
-
-	updates, err := s.getList(universe, toUpgrade)
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "couldn't get package marked for update from universe")
-	}
-
 	var formulas []bf.Formula
 
 	// Build constraints for the whole defdb
@@ -377,29 +436,42 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert
 
 	// Treat removed packages from universe as marked for deletion
 	if dropremoved {
-		oldPackages = append(oldPackages, removed...)
+		// SAT encode the clauses against the world
+		for _, p := range removed.Unique() {
+			encodedP, err := p.Encode(universe)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "couldn't encode package")
+			}
+			P := bf.Var(encodedP)
+			formulas = append(formulas, bf.And(bf.Not(P), r))
+		}
 	}
 
-	// SAT encode the clauses against the world
-	for _, p := range oldPackages.Unique() {
-		encodedP, err := p.Encode(universe)
+	for old, new := range replacements {
+		oldP, err := old.Encode(universe)
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "couldn't encode package")
 		}
-		P := bf.Var(encodedP)
-		formulas = append(formulas, bf.And(bf.Not(P), r))
-	}
-
-	for _, p := range updates {
-		encodedP, err := p.Encode(universe)
+		oldencodedP := bf.Var(oldP)
+		newP, err := new.Encode(universe)
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "couldn't encode package")
 		}
-		P := bf.Var(encodedP)
-		formulas = append(formulas, bf.And(P, r))
+		newEncodedP := bf.Var(newP)
+
+		//solvable, err := old.BuildFormula(s.DefinitionDatabase, s.SolverDatabase)
+		solvablenew, err := new.BuildFormula(s.DefinitionDatabase, s.SolverDatabase)
+
+		formulas = append(formulas, bf.And(bf.Not(oldencodedP), bf.And(append(solvablenew, newEncodedP)...)))
 	}
 
+	//formulas = append(formulas, r)
+
 	markedForRemoval := pkg.Packages{}
+
+	if len(formulas) == 0 {
+		return pkg.Packages{}, PackagesAssertions{}, nil
+	}
 	model := bf.Solve(bf.And(formulas...))
 	if model == nil {
 		return nil, nil, errors.New("Failed finding a solution")
@@ -428,27 +500,27 @@ func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAsser
 	toUninstall := pkg.Packages{}
 	toInstall := pkg.Packages{}
 
-	availableCache := map[string]pkg.Packages{}
-	for _, p := range s.DefinitionDatabase.World() {
-		// Each one, should be expanded
-		availableCache[p.GetName()+p.GetCategory()] = append(availableCache[p.GetName()+p.GetCategory()], p)
+	// we do this in memory so we take into account of provides, and its faster
+	universe, err := s.DefinitionDatabase.Copy()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed creating db copy")
 	}
 
 	installedcopy := pkg.NewInMemoryDatabase(false)
 
 	for _, p := range s.InstalledDatabase.World() {
 		installedcopy.CreatePackage(p)
-		packages, ok := availableCache[p.GetName()+p.GetCategory()]
-		if ok && len(packages) != 0 {
+		packages, err := universe.FindPackageVersions(p)
+		if err == nil && len(packages) != 0 {
 			best := packages.Best(nil)
-			if best.GetVersion() != p.GetVersion() {
+			if !best.Matches(p) {
 				toUninstall = append(toUninstall, p)
 				toInstall = append(toInstall, best)
 			}
 		}
 	}
 
-	s2 := NewSolver(installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+	s2 := NewSolver(Options{Type: SingleCoreSimple}, installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
 	s2.SetResolver(s.Resolver)
 	if !full {
 		ass := PackagesAssertions{}
@@ -456,23 +528,25 @@ func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAsser
 			ass = append(ass, PackageAssert{Package: i.(*pkg.DefaultPackage), Value: true})
 		}
 	}
-
 	// Then try to uninstall the versions in the system, and store that tree
-	for _, p := range toUninstall {
-		r, err := s.Uninstall(p, checkconflicts, false)
+	r, err := s.Uninstall(checkconflicts, false, toUninstall.Unique()...)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't uninstall candidates ")
+	}
+	for _, z := range r {
+		err = installedcopy.RemovePackage(z)
 		if err != nil {
-			return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't uninstall selected candidate "+p.GetFingerPrint())
-		}
-		for _, z := range r {
-			err = installedcopy.RemovePackage(z)
-			if err != nil {
-				return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't remove copy of package targetted for removal")
-			}
+			return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't remove copy of package targetted for removal")
 		}
 	}
 
-	r, e := s2.Install(toInstall)
-	return toUninstall, r, e
+	if len(toInstall) == 0 {
+		return toUninstall, PackagesAssertions{}, nil
+	}
+
+	assertions, err := s2.Install(toInstall.Unique())
+
+	return toUninstall, assertions, err
 	// To that tree, ask to install the versions that should be upgraded, and try to solve
 	// Return the solution
 
@@ -480,52 +554,72 @@ func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAsser
 
 // Uninstall takes a candidate package and return a list of packages that would be removed
 // in order to purge the candidate. Returns error if unsat.
-func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packages, error) {
-	var res pkg.Packages
-	candidate, err := s.InstalledDatabase.FindPackage(c)
-	if err != nil {
-
-		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
-		packages, err := c.Expand(s.InstalledDatabase)
-		//	Info("Expanded", packages, err)
-		if err != nil || len(packages) == 0 {
-			candidate = c
-		} else {
-			candidate = packages.Best(nil)
-		}
-		//Relax search, otherwise we cannot compute solutions for packages not in definitions
-		//	return nil, errors.Wrap(err, "Package not found between installed")
+func (s *Solver) Uninstall(checkconflicts, full bool, packs ...pkg.Package) (pkg.Packages, error) {
+	if len(packs) == 0 {
+		return pkg.Packages{}, nil
 	}
+	var res pkg.Packages
+
+	toRemove := pkg.Packages{}
+
+	for _, c := range packs {
+		candidate, err := s.InstalledDatabase.FindPackage(c)
+		if err != nil {
+
+			//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
+			packages, err := c.Expand(s.InstalledDatabase)
+			//	Info("Expanded", packages, err)
+			if err != nil || len(packages) == 0 {
+				candidate = c
+			} else {
+				candidate = packages.Best(nil)
+			}
+			//Relax search, otherwise we cannot compute solutions for packages not in definitions
+			//	return nil, errors.Wrap(err, "Package not found between installed")
+		}
+
+		toRemove = append(toRemove, candidate)
+	}
+
 	// Build a fake "Installed" - Candidate and its requires tree
 	var InstalledMinusCandidate pkg.Packages
 
 	// We are asked to not perform a full uninstall (checking all the possible requires that could
 	// be removed). Let's only check if we can remove the selected package
 	if !full && checkconflicts {
-		if conflicts, err := s.Conflicts(candidate, s.Installed()); conflicts {
-			return nil, err
-		} else {
-			return pkg.Packages{candidate}, nil
+		for _, candidate := range toRemove {
+			if conflicts, err := s.Conflicts(candidate, s.Installed()); conflicts {
+				return nil, errors.Wrap(err, "while searching for "+candidate.HumanReadableString()+" conflicts")
+			}
 		}
+		return toRemove, nil
 	}
 
 	// TODO: Can be optimized
 	for _, i := range s.Installed() {
-		if !i.Matches(candidate) {
-			contains, err := candidate.RequiresContains(s.SolverDatabase, i)
-			if err != nil {
-				return nil, errors.Wrap(err, "Failed getting installed list")
-			}
-			if !contains {
-				InstalledMinusCandidate = append(InstalledMinusCandidate, i)
+		matched := false
+		for _, candidate := range toRemove {
+			if !i.Matches(candidate) {
+				contains, err := candidate.RequiresContains(s.SolverDatabase, i)
+				if err != nil {
+					return nil, errors.Wrap(err, "Failed getting installed list")
+				}
+				if !contains {
+					matched = true
+				}
+
 			}
 		}
+		if matched {
+			InstalledMinusCandidate = append(InstalledMinusCandidate, i)
+		}
 	}
 
-	s2 := NewSolver(pkg.NewInMemoryDatabase(false), s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+	s2 := NewSolver(Options{Type: SingleCoreSimple}, pkg.NewInMemoryDatabase(false), s.InstalledDatabase, pkg.NewInMemoryDatabase(false))
 	s2.SetResolver(s.Resolver)
+
 	// Get the requirements to install the candidate
-	asserts, err := s2.Install(pkg.Packages{candidate})
+	asserts, err := s2.Install(toRemove)
 	if err != nil {
 		return nil, err
 	}
@@ -570,12 +664,15 @@ func (s *Solver) BuildFormula() (bf.Formula, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	for _, wanted := range s.Wanted {
+
 		encodedW, err := wanted.Encode(s.SolverDatabase)
 		if err != nil {
 			return nil, err
 		}
 		W := bf.Var(encodedW)
+		//	allW = append(allW, W)
 		installedWorld := s.Installed()
 		//TODO:Optimize
 		if len(installedWorld) == 0 {
@@ -593,8 +690,8 @@ func (s *Solver) BuildFormula() (bf.Formula, error) {
 		}
 
 	}
-	formulas = append(formulas, r)
 
+	formulas = append(formulas, r)
 	return bf.And(formulas...), nil
 }
 

pkg/solver/solver_test.go

@@ -28,13 +28,13 @@ var _ = Describe("Solver", func() {
 	db := pkg.NewInMemoryDatabase(false)
 	dbInstalled := pkg.NewInMemoryDatabase(false)
 	dbDefinitions := pkg.NewInMemoryDatabase(false)
-	s := NewSolver(dbInstalled, dbDefinitions, db)
+	s := NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 	BeforeEach(func() {
 		db = pkg.NewInMemoryDatabase(false)
 		dbInstalled = pkg.NewInMemoryDatabase(false)
 		dbDefinitions = pkg.NewInMemoryDatabase(false)
-		s = NewSolver(dbInstalled, dbDefinitions, db)
+		s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 	})
 	Context("Simple set", func() {
 		It("Solves correctly if the selected package has no requirements or conflicts and we have nothing installed yet", func() {
@@ -52,7 +52,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
@@ -75,7 +75,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{B})
 			Expect(err).ToNot(HaveOccurred())
@@ -101,15 +101,15 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: true}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false}))
+			//	Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+			//Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false}))
 
 			Expect(len(solution)).To(Equal(5))
 		})
@@ -130,7 +130,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
@@ -156,7 +156,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(err).ToNot(HaveOccurred())
@@ -181,7 +181,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -209,7 +209,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -236,7 +236,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -263,7 +263,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{C})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -291,7 +291,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{&pkg.DefaultPackage{Name: "c", Version: ">1.0", Category: "test"}})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -317,7 +317,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -345,7 +345,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A, B})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -391,7 +391,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{C})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -431,7 +431,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A, B})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
@@ -476,7 +476,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A2, B})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
@@ -514,7 +514,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A2})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
@@ -555,7 +555,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 			solution, err := s.Install([]pkg.Package{A2})
 			Expect(solution).To(ContainElement(PackageAssert{Package: A2, Value: true}))
@@ -591,9 +591,9 @@ var _ = Describe("Solver", func() {
 					_, err := dbInstalled.CreatePackage(p)
 					Expect(err).ToNot(HaveOccurred())
 				}
-				s = NewSolver(dbInstalled, dbDefinitions, db)
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
-				solution, err := s.Uninstall(A, true, true)
+				solution, err := s.Uninstall(true, true, A)
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(solution).To(ContainElement(A))
@@ -617,9 +617,9 @@ var _ = Describe("Solver", func() {
 					_, err := dbInstalled.CreatePackage(p)
 					Expect(err).ToNot(HaveOccurred())
 				}
-				s = NewSolver(dbInstalled, dbDefinitions, db)
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
-				solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"}, true, true)
+				solution, err := s.Uninstall(true, true, &pkg.DefaultPackage{Name: "A", Version: ">1.0"})
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(solution).To(ContainElement(A))
@@ -643,7 +643,7 @@ var _ = Describe("Solver", func() {
 					_, err := dbInstalled.CreatePackage(p)
 					Expect(err).ToNot(HaveOccurred())
 				}
-				solution, err := s.Uninstall(A, true, true)
+				solution, err := s.Uninstall(true, true, A)
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(solution).To(ContainElement(A))
@@ -667,12 +667,13 @@ var _ = Describe("Solver", func() {
 					_, err := dbInstalled.CreatePackage(p)
 					Expect(err).ToNot(HaveOccurred())
 				}
-				solution, err := s.Uninstall(A, true, true)
+				solution, err := s.Uninstall(true, true, A)
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
 
-				Expect(len(solution)).To(Equal(1))
+				Expect(len(solution)).To(Equal(2))
 			})
 
 			It("Uninstalls complex packages correctly, even if shared deps are required by system packages", func() {
@@ -690,7 +691,7 @@ var _ = Describe("Solver", func() {
 					_, err := dbInstalled.CreatePackage(p)
 					Expect(err).ToNot(HaveOccurred())
 				}
-				solution, err := s.Uninstall(A, true, true)
+				solution, err := s.Uninstall(true, true, A)
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(solution).To(ContainElement(A))
@@ -715,7 +716,7 @@ var _ = Describe("Solver", func() {
 					Expect(err).ToNot(HaveOccurred())
 				}
 
-				solution, err := s.Uninstall(A, true, true)
+				solution, err := s.Uninstall(true, true, A)
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(solution).To(ContainElement(A))
@@ -741,7 +742,7 @@ var _ = Describe("Solver", func() {
 					Expect(err).ToNot(HaveOccurred())
 				}
 
-				solution, err := s.Uninstall(A, true, true)
+				solution, err := s.Uninstall(true, true, A)
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(solution).To(ContainElement(A))
@@ -752,6 +753,33 @@ var _ = Describe("Solver", func() {
 
 			})
 
+			It("Fails to uninstall if a package is required", func() {
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				Z := pkg.NewPackage("Z", "", []*pkg.DefaultPackage{A}, []*pkg.DefaultPackage{})
+				F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{Z, B}, []*pkg.DefaultPackage{})
+
+				Z.SetVersion("1.4101.dvw.dqc.")
+				B.SetVersion("1.4101qe.eq.ff..dvw.dqc.")
+				C.SetVersion("1.aaaa.eq.ff..dvw.dqc.")
+
+				for _, p := range []pkg.Package{A, B, C, D, Z, F} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D, Z, F} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Uninstall(true, false, B)
+				Expect(err).To(HaveOccurred())
+				Expect(len(solution)).To(Equal(0))
+			})
+
 			It("UninstallUniverse simple package correctly", func() {
 
 				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
@@ -768,7 +796,7 @@ var _ = Describe("Solver", func() {
 					_, err := dbInstalled.CreatePackage(p)
 					Expect(err).ToNot(HaveOccurred())
 				}
-				s = NewSolver(dbInstalled, dbDefinitions, db)
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
@@ -794,7 +822,7 @@ var _ = Describe("Solver", func() {
 					_, err := dbInstalled.CreatePackage(p)
 					Expect(err).ToNot(HaveOccurred())
 				}
-				s = NewSolver(dbInstalled, dbDefinitions, db)
+				s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 				solution, err := s.UninstallUniverse(pkg.Packages{
 					&pkg.DefaultPackage{Name: "A", Version: ">1.0"}})
@@ -1070,7 +1098,7 @@ var _ = Describe("Solver", func() {
 			}
 
 			val, err := s.Conflicts(D, dbInstalled.World())
-			Expect(err.Error()).To(Equal("\n/A-\n/B-"))
+			Expect(err.Error()).To(Or(Equal("\n/A-\n/B-"), Equal("\n/B-\n/A-")))
 			Expect(val).To(BeTrue())
 		})
 
@@ -1209,7 +1237,8 @@ var _ = Describe("Solver", func() {
 		})
 	})
 	Context("Upgrades", func() {
-
+		E := pkg.NewPackage("e", "1.5", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		E.SetCategory("test")
 		C := pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
 		C.SetCategory("test")
 		B := pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
@@ -1219,6 +1248,17 @@ var _ = Describe("Solver", func() {
 		A1 := pkg.NewPackage("a", "1.2", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
 		A1.SetCategory("test")
 
+		BeforeEach(func() {
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+			A1 = pkg.NewPackage("a", "1.2", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A1.SetCategory("test")
+		})
+
 		It("upgrades correctly", func() {
 			for _, p := range []pkg.Package{A1, B, C} {
 				_, err := dbDefinitions.CreatePackage(p)
@@ -1240,7 +1280,60 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
 			Expect(len(solution)).To(Equal(3))
+		})
 
+		It("upgrades correctly with provides", func() {
+			B.SetProvides([]*pkg.DefaultPackage{
+				&pkg.DefaultPackage{Name: "a", Version: ">=0", Category: "test"},
+				&pkg.DefaultPackage{Name: "c", Version: ">=0", Category: "test"},
+			})
+
+			for _, p := range []pkg.Package{B} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, C} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.Upgrade(true, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(2))
+
+			Expect(uninstall).To(ContainElement(C))
+			Expect(uninstall).To(ContainElement(A))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(len(solution)).To(Equal(1))
+		})
+
+		PIt("upgrades correctly with provides, also if definitiondb contains both a provide, and the package to be provided", func() {
+			B.SetProvides([]*pkg.DefaultPackage{
+				&pkg.DefaultPackage{Name: "a", Version: ">=0", Category: "test"},
+				&pkg.DefaultPackage{Name: "c", Version: ">=0", Category: "test"},
+			})
+
+			for _, p := range []pkg.Package{A1, B} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, C} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.Upgrade(true, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(2))
+
+			Expect(uninstall).To(ContainElement(C))
+			Expect(uninstall).To(ContainElement(A))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(len(solution)).To(Equal(1))
 		})
 
 		It("UpgradeUniverse upgrades correctly", func() {
@@ -1262,10 +1355,37 @@ var _ = Describe("Solver", func() {
 
 			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
 
-			Expect(len(solution)).To(Equal(4))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: C, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
+
+			Expect(len(solution)).To(Equal(3))
+		})
+
+		It("Suggests to remove untracked packages", func() {
+			for _, p := range []pkg.Package{E} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B, C, E} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.UpgradeUniverse(true)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(uninstall)).To(Equal(3))
+
+			Expect(uninstall).To(ContainElement(B))
+			Expect(uninstall).To(ContainElement(A))
+			Expect(uninstall).To(ContainElement(C))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+
+			Expect(len(solution)).To(Equal(3))
 		})
 	})
 })

pkg/spectooling/definition.go

@@ -44,6 +44,14 @@ type DefaultPackageSanitized struct {
 	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
 }
 
+func NewDefaultPackageSanitizedFromYaml(data []byte) (*DefaultPackageSanitized, error) {
+	ans := &DefaultPackageSanitized{}
+	if err := yaml.Unmarshal(data, ans); err != nil {
+		return nil, err
+	}
+	return ans, nil
+}
+
 func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
 	ans := &DefaultPackageSanitized{
 		Name:        p.GetName(),
@@ -110,3 +118,12 @@ func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
 func (p *DefaultPackageSanitized) Yaml() ([]byte, error) {
 	return yaml.Marshal(p)
 }
+
+func (p *DefaultPackageSanitized) Clone() (*DefaultPackageSanitized, error) {
+	data, err := p.Yaml()
+	if err != nil {
+		return nil, err
+	}
+
+	return NewDefaultPackageSanitizedFromYaml(data)
+}

pkg/tree/builder/gentoo/gentoo.go

@@ -1,142 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package gentoo
-
-// NOTE: Look here as an example of the builder definition executor
-// https://gist.github.com/adnaan/6ca68c7985c6f851def3
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-
-	. "github.com/mudler/luet/pkg/logger"
-	tree "github.com/mudler/luet/pkg/tree"
-
-	pkg "github.com/mudler/luet/pkg/package"
-)
-
-type MemoryDB int
-
-const (
-	InMemory MemoryDB = iota
-	BoltDB   MemoryDB = iota
-)
-
-func NewGentooBuilder(e EbuildParser, concurrency int, db MemoryDB) tree.Parser {
-	return &GentooBuilder{EbuildParser: e, Concurrency: concurrency}
-}
-
-type GentooBuilder struct {
-	EbuildParser EbuildParser
-	Concurrency  int
-	DBType       MemoryDB
-}
-
-type EbuildParser interface {
-	ScanEbuild(string) (pkg.Packages, error)
-}
-
-func (gb *GentooBuilder) scanEbuild(path string, db pkg.PackageDatabase) error {
-	defer func() {
-		if r := recover(); r != nil {
-			Error(r)
-		}
-	}()
-	pkgs, err := gb.EbuildParser.ScanEbuild(path)
-	if err != nil {
-		return err
-	}
-	for _, p := range pkgs {
-		_, err := db.FindPackage(p)
-		if err != nil {
-			_, err := db.CreatePackage(p)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (gb *GentooBuilder) worker(i int, wg *sync.WaitGroup, s <-chan string, db pkg.PackageDatabase) {
-	defer wg.Done()
-
-	for path := range s {
-		Info("#"+strconv.Itoa(i), "parsing", path)
-		err := gb.scanEbuild(path, db)
-		if err != nil {
-			Error(path, ":", err.Error())
-		}
-	}
-
-}
-
-func (gb *GentooBuilder) Generate(dir string) (pkg.PackageDatabase, error) {
-
-	var toScan = make(chan string)
-	Spinner(27)
-	defer SpinnerStop()
-	var db pkg.PackageDatabase
-	// Support for
-	switch gb.DBType {
-	case InMemory:
-		db = pkg.NewInMemoryDatabase(false)
-	case BoltDB:
-		tmpfile, err := ioutil.TempFile("", "boltdb")
-		if err != nil {
-			return nil, err
-		}
-		db = pkg.NewBoltDatabase(tmpfile.Name())
-	default:
-		db = pkg.NewInMemoryDatabase(false)
-	}
-
-	Debug("Concurrency", gb.Concurrency)
-	// the waitgroup will allow us to wait for all the goroutines to finish at the end
-	var wg = new(sync.WaitGroup)
-	for i := 0; i < gb.Concurrency; i++ {
-		wg.Add(1)
-		go gb.worker(i, wg, toScan, db)
-	}
-
-	// TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean()
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.IsDir() {
-			return nil
-		}
-		// Ensure that only file with suffix .ebuild are elaborated.
-		// and ignore .swp files or files with string ebuild on name
-		if strings.HasSuffix(info.Name(), ".ebuild") {
-			toScan <- path
-		}
-		return nil
-	})
-
-	close(toScan)
-	wg.Wait()
-	if err != nil {
-		return db, err
-	}
-
-	return db, nil
-}

pkg/tree/builder/gentoo/gentoo_test.go

@@ -1,177 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package gentoo_test
-
-import (
-	"fmt"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	pkg "github.com/mudler/luet/pkg/package"
-	. "github.com/mudler/luet/pkg/tree/builder/gentoo"
-)
-
-type FakeParser struct {
-}
-
-func (f *FakeParser) ScanEbuild(path string) (pkg.Packages, error) {
-	return pkg.Packages{&pkg.DefaultPackage{Name: path}}, nil
-}
-
-var _ = Describe("GentooBuilder", func() {
-
-	Context("Simple test", func() {
-		for _, dbType := range []MemoryDB{InMemory, BoltDB} {
-			It("parses correctly deps", func() {
-				gb := NewGentooBuilder(&FakeParser{}, 20, dbType)
-				tree, err := gb.Generate("../../../../tests/fixtures/overlay")
-				defer func() {
-					Expect(tree.Clean()).ToNot(HaveOccurred())
-				}()
-				Expect(err).ToNot(HaveOccurred())
-				Expect(len(tree.GetPackages())).To(Equal(10))
-			})
-		}
-	})
-
-	Context("Parse ebuild1", func() {
-		parser := &SimpleEbuildParser{}
-		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/overlay/app-crypt/pinentry-gnome/pinentry-gnome-1.0.0-r2.ebuild")
-		It("parses correctly deps", func() {
-			Expect(err).ToNot(HaveOccurred())
-			fmt.Println("PKG ", pkgs[0])
-			Expect(pkgs[0].GetLicense()).To(Equal("GPL-2"))
-			Expect(pkgs[0].GetDescription()).To(Equal("GNOME 3 frontend for pinentry"))
-		})
-	})
-
-	Context("Parse ebuild2", func() {
-		parser := &SimpleEbuildParser{}
-		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/mod_dav_svn-1.12.2.ebuild")
-
-		It("Parsing ebuild2", func() {
-			Expect(err).ToNot(HaveOccurred())
-			fmt.Println("PKG ", pkgs[0])
-			Expect(pkgs[0].GetLicense()).To(Equal("Subversion"))
-			Expect(pkgs[0].GetDescription()).To(Equal("Subversion WebDAV support"))
-		})
-	})
-
-	Context("Parse ebuild3", func() {
-		parser := &SimpleEbuildParser{}
-		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/linux-sources-1.ebuild")
-
-		It("Check parsing of the ebuild3", func() {
-			Expect(err).ToNot(HaveOccurred())
-			fmt.Println("PKG ", pkgs[0])
-			Expect(len(pkgs[0].GetRequires())).To(Equal(0))
-			Expect(pkgs[0].GetLicense()).To(Equal(""))
-			Expect(pkgs[0].GetDescription()).To(Equal("Virtual for Linux kernel sources"))
-		})
-	})
-
-	Context("Parse ebuild4", func() {
-		parser := &SimpleEbuildParser{}
-		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/sabayon-mce-1.1-r5.ebuild")
-
-		It("Check parsing of the ebuild4", func() {
-			Expect(err).ToNot(HaveOccurred())
-			fmt.Println("PKG ", pkgs[0])
-			Expect(len(pkgs[0].GetRequires())).To(Equal(2))
-			Expect(pkgs[0].GetLicense()).To(Equal("GPL-2"))
-			Expect(pkgs[0].GetDescription()).To(Equal("Sabayon Linux Media Center Infrastructure"))
-		})
-	})
-
-	Context("Parse ebuild5", func() {
-		parser := &SimpleEbuildParser{}
-		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/libreoffice-l10n-meta-6.2.8.2.ebuild")
-
-		It("Check parsing of the ebuild5", func() {
-			Expect(err).ToNot(HaveOccurred())
-			fmt.Println("PKG ", pkgs[0])
-			Expect(len(pkgs[0].GetRequires())).To(Equal(146))
-			Expect(pkgs[0].GetLicense()).To(Equal("LGPL-2"))
-			Expect(pkgs[0].GetDescription()).To(Equal("LibreOffice.org localisation meta-package"))
-		})
-	})
-
-	Context("Parse ebuild6", func() {
-		parser := &SimpleEbuildParser{}
-		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/pkgs-checker-0.2.0.ebuild")
-
-		It("Check parsing of the ebuild6", func() {
-			Expect(err).ToNot(HaveOccurred())
-			fmt.Println("PKG ", pkgs[0])
-			Expect(len(pkgs[0].GetRequires())).To(Equal(0))
-			Expect(pkgs[0].GetLicense()).To(Equal("GPL-3"))
-			Expect(pkgs[0].GetDescription()).To(Equal("Sabayon Packages Checker"))
-		})
-	})
-
-	Context("Parse ebuild7", func() {
-		parser := &SimpleEbuildParser{}
-		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/calamares-sabayon-base-modules-1.15.ebuild")
-
-		It("Check parsing of the ebuild7", func() {
-			Expect(err).ToNot(HaveOccurred())
-			fmt.Println("PKG ", pkgs[0])
-			Expect(len(pkgs[0].GetRequires())).To(Equal(2))
-			Expect(pkgs[0].GetLicense()).To(Equal("CC-BY-SA-4.0"))
-			Expect(pkgs[0].GetDescription()).To(Equal("Sabayon Official Calamares base modules"))
-		})
-	})
-
-	Context("Parse ebuild8", func() {
-		parser := &SimpleEbuildParser{}
-		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/subversion-1.12.0.ebuild")
-
-		It("Check parsing of the ebuild8", func() {
-			Expect(err).ToNot(HaveOccurred())
-			fmt.Println("PKG ", pkgs[0])
-			Expect(len(pkgs[0].GetRequires())).To(Equal(25))
-			Expect(pkgs[0].GetLicense()).To(Equal("Subversion GPL-2"))
-			Expect(pkgs[0].GetDescription()).To(Equal("Advanced version control system"))
-		})
-	})
-
-	Context("Parse ebuild9", func() {
-		parser := &SimpleEbuildParser{}
-		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/kodi-raspberrypi-16.0.ebuild")
-
-		PIt("Check parsing of the ebuild9", func() {
-			Expect(err).ToNot(HaveOccurred())
-			fmt.Println("PKG ", pkgs[0])
-			Expect(len(pkgs[0].GetRequires())).To(Equal(66))
-			Expect(pkgs[0].GetLicense()).To(Equal("GPL-2"))
-			Expect(pkgs[0].GetDescription()).To(Equal("Kodi is a free and open source media-player and entertainment hub"))
-		})
-	})
-
-	Context("Parse ebuild10", func() {
-		parser := &SimpleEbuildParser{}
-		pkgs, err := parser.ScanEbuild("../../../../tests/fixtures/parser/tango-icon-theme-0.8.90-r1.ebuild")
-
-		It("Check parsing of the ebuild10", func() {
-			Expect(err).ToNot(HaveOccurred())
-			fmt.Println("PKG ", pkgs[0])
-			Expect(len(pkgs[0].GetRequires())).To(Equal(2))
-			Expect(pkgs[0].GetLicense()).To(Equal("public-domain"))
-			Expect(pkgs[0].GetDescription()).To(Equal("SVG and PNG icon theme from the Tango project"))
-		})
-	})
-
-})

pkg/tree/builder/gentoo/simpleparser.go

@@ -1,440 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package gentoo
-
-// NOTE: Look here as an example of the builder definition executor
-// https://gist.github.com/adnaan/6ca68c7985c6f851def3
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"path/filepath"
-	"regexp"
-	"strings"
-	"time"
-
-	. "github.com/mudler/luet/pkg/logger"
-
-	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
-	pkg "github.com/mudler/luet/pkg/package"
-	"mvdan.cc/sh/v3/expand"
-	"mvdan.cc/sh/v3/shell"
-	"mvdan.cc/sh/v3/syntax"
-)
-
-const (
-	uriRegex = "(.*[.]tar[.].*|.*[.]zip|.*[.]run|.*[.]png|.*[.]rpm|.*[.]gz)"
-)
-
-// SimpleEbuildParser ignores USE flags and generates just 1-1 package
-type SimpleEbuildParser struct {
-	World pkg.PackageDatabase
-}
-
-type GentooDependency struct {
-	Use          string
-	UseCondition _gentoo.PackageCond
-	SubDeps      []*GentooDependency
-	Dep          *_gentoo.GentooPackage
-}
-
-type GentooRDEPEND struct {
-	Dependencies []*GentooDependency
-}
-
-func NewGentooDependency(pkg, use string) (*GentooDependency, error) {
-	var err error
-	ans := &GentooDependency{
-		Use:     use,
-		SubDeps: make([]*GentooDependency, 0),
-	}
-
-	if strings.HasPrefix(use, "!") {
-		ans.Use = ans.Use[1:]
-		ans.UseCondition = _gentoo.PkgCondNot
-	}
-
-	if pkg != "" {
-		ans.Dep, err = _gentoo.ParsePackageStr(pkg)
-		if err != nil {
-			return nil, err
-		}
-
-		// TODO: Fix this on parsing phase for handle correctly ${PV}
-		if strings.HasSuffix(ans.Dep.Name, "-") {
-			ans.Dep.Name = ans.Dep.Name[:len(ans.Dep.Name)-1]
-		}
-
-	}
-
-	return ans, nil
-}
-
-func (d *GentooDependency) String() string {
-	if d.Dep != nil {
-		return fmt.Sprintf("%s", d.Dep)
-	} else {
-		return fmt.Sprintf("%s %d %s", d.Use, d.UseCondition, d.SubDeps)
-	}
-}
-
-func (d *GentooDependency) GetDepsList() []*GentooDependency {
-	ans := make([]*GentooDependency, 0)
-
-	if len(d.SubDeps) > 0 {
-		for _, d2 := range d.SubDeps {
-			list := d2.GetDepsList()
-			ans = append(ans, list...)
-		}
-	}
-
-	if d.Dep != nil {
-		ans = append(ans, d)
-	}
-
-	return ans
-}
-
-func (d *GentooDependency) AddSubDependency(pkg, use string) (*GentooDependency, error) {
-	ans, err := NewGentooDependency(pkg, use)
-	if err != nil {
-		return nil, err
-	}
-
-	d.SubDeps = append(d.SubDeps, ans)
-
-	return ans, nil
-}
-
-func (r *GentooRDEPEND) GetDependencies() []*GentooDependency {
-	ans := make([]*GentooDependency, 0)
-
-	for _, d := range r.Dependencies {
-		list := d.GetDepsList()
-		ans = append(ans, list...)
-	}
-
-	// the same dependency could be available in multiple use flags.
-	// It's needed avoid duplicate.
-	m := make(map[string]*GentooDependency, 0)
-
-	for _, p := range ans {
-		m[p.String()] = p
-	}
-
-	ans = make([]*GentooDependency, 0)
-	for _, p := range m {
-		ans = append(ans, p)
-	}
-
-	return ans
-}
-
-func ParseRDEPEND(rdepend string) (*GentooRDEPEND, error) {
-	var lastdep []*GentooDependency = make([]*GentooDependency, 0)
-	var pendingDep = false
-	var orDep = false
-	var dep *GentooDependency
-	var err error
-
-	ans := &GentooRDEPEND{
-		Dependencies: make([]*GentooDependency, 0),
-	}
-
-	if rdepend != "" {
-		rdepends := strings.Split(rdepend, "\n")
-		for _, rr := range rdepends {
-			rr = strings.TrimSpace(rr)
-			if rr == "" {
-				continue
-			}
-
-			if strings.HasPrefix(rr, "|| (") {
-				orDep = true
-				continue
-			}
-
-			if orDep {
-				rr = strings.TrimSpace(rr)
-				if rr == ")" {
-					orDep = false
-				}
-				continue
-			}
-
-			if strings.Index(rr, "?") > 0 {
-				// use flag present
-
-				if pendingDep {
-					dep, err = lastdep[len(lastdep)-1].AddSubDependency("", rr[:strings.Index(rr, "?")])
-					if err != nil {
-						Debug("Ignoring subdependency ", rr[:strings.Index(rr, "?")])
-					}
-				} else {
-					dep, err = NewGentooDependency("", rr[:strings.Index(rr, "?")])
-					if err != nil {
-						Debug("Ignoring dep", rr)
-					} else {
-						ans.Dependencies = append(ans.Dependencies, dep)
-					}
-				}
-
-				if strings.Index(rr, ")") < 0 {
-					pendingDep = true
-					lastdep = append(lastdep, dep)
-				}
-
-				if strings.Index(rr, "|| (") >= 0 {
-					// Ignore dep in or
-					continue
-				}
-
-				fields := strings.Split(rr[strings.Index(rr, "?")+1:], " ")
-				for _, f := range fields {
-					f = strings.TrimSpace(f)
-					if f == ")" || f == "(" || f == "" {
-						continue
-					}
-
-					_, err = dep.AddSubDependency(f, "")
-					if err != nil {
-						Debug("Ignoring subdependency ", f)
-					}
-				}
-
-			} else if pendingDep {
-				fields := strings.Split(rr, " ")
-				for _, f := range fields {
-					f = strings.TrimSpace(f)
-					if f == ")" || f == "(" || f == "" {
-						continue
-					}
-					_, err = lastdep[len(lastdep)-1].AddSubDependency(f, "")
-					if err != nil {
-						return nil, err
-					}
-				}
-
-				if strings.Index(rr, ")") >= 0 {
-					lastdep = lastdep[:len(lastdep)-1]
-					if len(lastdep) == 0 {
-						pendingDep = false
-					}
-				}
-
-			} else {
-				rr = strings.TrimSpace(rr)
-				// Check if there multiple deps in single row
-
-				fields := strings.Split(rr, " ")
-				if len(fields) > 1 {
-					for _, rrr := range fields {
-						rrr = strings.TrimSpace(rrr)
-						if rrr == "" {
-							continue
-						}
-						dep, err := NewGentooDependency(rrr, "")
-						if err != nil {
-							Debug("Ignoring dep", rr)
-						} else {
-							ans.Dependencies = append(ans.Dependencies, dep)
-						}
-					}
-				} else {
-					dep, err := NewGentooDependency(rr, "")
-					if err != nil {
-						Debug("Ignoring dep", rr)
-					} else {
-						ans.Dependencies = append(ans.Dependencies, dep)
-					}
-				}
-			}
-
-		}
-
-	}
-
-	return ans, nil
-}
-
-func SourceFile(ctx context.Context, path string, pkg *_gentoo.GentooPackage) (map[string]expand.Variable, error) {
-	content, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("could not open: %v", err)
-	}
-	scontent := string(content)
-
-	// Add default Genoo Variables
-	ebuild := fmt.Sprintf("P=%s\n", pkg.GetP()) +
-		fmt.Sprintf("PN=%s\n", pkg.GetPN()) +
-		fmt.Sprintf("PV=%s\n", pkg.GetPV()) +
-		fmt.Sprintf("PVR=%s\n", pkg.GetPVR())
-
-	// Disable inherit
-	scontent = strings.ReplaceAll(scontent, "inherit", "#inherit")
-	// Disable function from eclass (TODO: check how fix better this)
-	scontent = strings.ReplaceAll(scontent, "need_apache", "#need_apache")
-	scontent = strings.ReplaceAll(scontent, "want_apache", "#want_apache")
-
-	regexFuncs := regexp.MustCompile(
-		"[a-zA-Z]+.*[_][a-z]+[(][)][\\s]{",
-	)
-	matches := regexFuncs.FindAllIndex([]byte(scontent), -1)
-	// Drop section after functions (src_*, *() {)
-	if len(matches) > 0 {
-		ebuild = ebuild + scontent[:matches[0][0]]
-	} else {
-		ebuild = ebuild + scontent
-	}
-
-	// [[ ${PV} == "9999" ]] is not supported. Workaround but we need a better solution.
-	regexDoubleBrakets := regexp.MustCompile(
-		//"[[][[].*",
-		"^[[][[].*",
-		//"^.*\[\[.*\]\]",
-	)
-	matchDB := regexDoubleBrakets.FindAllIndex([]byte(ebuild), -1)
-	if len(matchDB) > 0 {
-		ebuild = ebuild[:matchDB[0][0]] + "#" + ebuild[matchDB[0][0]:]
-	}
-
-	//fmt.Println("EBUILD ", ebuild)
-
-	file, err := syntax.NewParser().Parse(strings.NewReader(ebuild), path)
-	if err != nil {
-		return nil, fmt.Errorf("could not parse: %v", err)
-	}
-	return shell.SourceNode(ctx, file)
-}
-
-// ScanEbuild returns a list of packages (always one with SimpleEbuildParser) decoded from an ebuild.
-func (ep *SimpleEbuildParser) ScanEbuild(path string) (pkg.Packages, error) {
-	Debug("Starting parsing of ebuild", path)
-
-	pkgstr := filepath.Base(path)
-	paths := strings.Split(filepath.Dir(path), "/")
-	pkgstr = paths[len(paths)-2] + "/" + strings.Replace(pkgstr, ".ebuild", "", -1)
-
-	gp, err := _gentoo.ParsePackageStr(pkgstr)
-	if err != nil {
-		return pkg.Packages{}, errors.New("Error on parsing package string")
-	}
-
-	pack := &pkg.DefaultPackage{
-		Name:     gp.Name,
-		Version:  fmt.Sprintf("%s%s", gp.Version, gp.VersionSuffix),
-		Category: gp.Category,
-		Uri:      make([]string, 0),
-	}
-
-	Debug("Prepare package ", pack.Category+"/"+pack.Name+"-"+pack.Version)
-
-	// Adding a timeout of 60secs, as with some bash files it can hang indefinetly
-	timeout, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-	defer cancel()
-	vars, err := SourceFile(timeout, path, gp)
-	if err != nil {
-		Error("Error on source file ", pack.Name, ": ", err)
-		return pkg.Packages{}, err
-	}
-
-	// TODO: Handle this a bit better
-	iuse, ok := vars["IUSE"]
-	if ok {
-		uses := strings.Split(strings.TrimSpace(iuse.String()), " ")
-		for _, u := range uses {
-			pack.AddUse(u)
-		}
-	}
-
-	// Retrieve package description
-	descr, ok := vars["DESCRIPTION"]
-	if ok {
-		pack.SetDescription(descr.String())
-	}
-	// Retrieve package license
-	license, ok := vars["LICENSE"]
-	if ok {
-		pack.SetLicense(license.String())
-	}
-	uri, ok := vars["SRC_URI"]
-	if ok {
-		// TODO: handle mirror:
-		uris := strings.Split(uri.String(), "\n")
-		for _, u := range uris {
-			u = strings.TrimSpace(u)
-
-			if u == "" {
-				continue
-			}
-			if match, _ := regexp.Match(uriRegex, []byte(u)); match {
-				if strings.Index(u, "(") >= 0 {
-					regexUri := regexp.MustCompile("(http|ftp|mirror).*[ ]")
-					matches := regexUri.FindAllIndex([]byte(u), -1)
-					if len(matches) > 0 {
-						u = u[matches[0][0]:matches[0][1]]
-					} else {
-						continue
-					}
-				}
-				pack.AddURI(u)
-				Debug("Add uri ", u)
-			} else {
-				Debug("Skip uri ", u)
-			}
-		}
-	}
-
-	rdepend, ok := vars["RDEPEND"]
-	if ok {
-		gRDEPEND, err := ParseRDEPEND(rdepend.String())
-		if err != nil {
-			Warning("Error on parsing RDEPEND for package ", pack.Category+"/"+pack.Name, err)
-			return pkg.Packages{pack}, nil
-			//	return pkg.Packages{}, err
-		}
-
-		pack.PackageConflicts = []*pkg.DefaultPackage{}
-		pack.PackageRequires = []*pkg.DefaultPackage{}
-
-		// TODO: See how handle use flags enabled.
-		// and if it's correct get list of deps directly.
-		for _, d := range gRDEPEND.GetDependencies() {
-
-			//TODO: Resolve to db or create a new one.
-			dep := &pkg.DefaultPackage{
-				Name:     d.Dep.Name,
-				Version:  d.Dep.Version + d.Dep.VersionSuffix,
-				Category: d.Dep.Category,
-			}
-			Debug(fmt.Sprintf("For package %s found dep: %s/%s %s",
-				gp, dep.Category, dep.Name, dep.Version))
-			if d.Dep.Condition == _gentoo.PkgCondNot {
-				pack.PackageConflicts = append(pack.PackageConflicts, dep)
-			} else {
-				pack.PackageRequires = append(pack.PackageRequires, dep)
-			}
-
-		}
-
-	}
-	Debug("Finished processing ebuild", path, "deps ", len(pack.PackageRequires))
-
-	//TODO: Deps and conflicts
-	return pkg.Packages{pack}, nil
-}

pkg/tree/builder/gentoo/simpleparser_test.go

@@ -1,669 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package gentoo_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
-	. "github.com/mudler/luet/pkg/tree/builder/gentoo"
-)
-
-var _ = Describe("GentooBuilder", func() {
-
-	Context("Parse RDEPEND1", func() {
-
-		rdepend := `
-	app-crypt/sbsigntools
-	x11-themes/sabayon-artwork-grub
-	sys-boot/os-prober
-	app-arch/xz-utils
-	>=sys-libs/ncurses-5.2-r5:0=
-`
-		gr, err := ParseRDEPEND(rdepend)
-		It("Check error", func() {
-			Expect(err).Should(BeNil())
-		})
-		It("Check gr", func() {
-			Expect(gr).ShouldNot(BeNil())
-		})
-
-		It("Check deps #", func() {
-			Expect(len(gr.Dependencies)).Should(Equal(5))
-		})
-
-		It("Check dep1", func() {
-			Expect(*gr.Dependencies[0]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sbsigntools",
-						Category: "app-crypt",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-
-		It("Check dep2", func() {
-			Expect(*gr.Dependencies[1]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sabayon-artwork-grub",
-						Category: "x11-themes",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-
-		It("Check dep5", func() {
-			Expect(*gr.Dependencies[4]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:          "ncurses",
-						Category:      "sys-libs",
-						Slot:          "0=",
-						Version:       "5.2",
-						VersionSuffix: "-r5",
-						Condition:     _gentoo.PkgCondGreaterEqual,
-					},
-				},
-			))
-		})
-
-	})
-
-	Context("Parse RDEPEND2", func() {
-
-		rdepend := `
-	app-crypt/sbsigntools
-	x11-themes/sabayon-artwork-grub
-	sys-boot/os-prober
-	app-arch/xz-utils
-	>=sys-libs/ncurses-5.2-r5:0=
-	mount? ( sys-fs/fuse )
-`
-		gr, err := ParseRDEPEND(rdepend)
-		It("Check error", func() {
-			Expect(err).Should(BeNil())
-		})
-		It("Check gr", func() {
-			Expect(gr).ShouldNot(BeNil())
-		})
-
-		It("Check deps #", func() {
-			Expect(len(gr.Dependencies)).Should(Equal(6))
-		})
-
-		It("Check dep1", func() {
-			Expect(*gr.Dependencies[0]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sbsigntools",
-						Category: "app-crypt",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-
-		It("Check dep2", func() {
-			Expect(*gr.Dependencies[1]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sabayon-artwork-grub",
-						Category: "x11-themes",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-
-		It("Check dep5", func() {
-			Expect(*gr.Dependencies[4]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:          "ncurses",
-						Category:      "sys-libs",
-						Slot:          "0=",
-						Version:       "5.2",
-						VersionSuffix: "-r5",
-						Condition:     _gentoo.PkgCondGreaterEqual,
-					},
-				},
-			))
-		})
-
-		It("Check dep6", func() {
-			Expect(*gr.Dependencies[5]).Should(Equal(
-				GentooDependency{
-					Use:          "mount",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps: []*GentooDependency{
-						&GentooDependency{
-							Use:          "",
-							UseCondition: _gentoo.PkgCondInvalid,
-							SubDeps:      make([]*GentooDependency, 0),
-							Dep: &_gentoo.GentooPackage{
-								Name:     "fuse",
-								Category: "sys-fs",
-								Slot:     "0",
-							},
-						},
-					},
-					Dep: nil,
-				},
-			))
-		})
-
-	})
-
-	Context("Parse RDEPEND3", func() {
-
-		rdepend := `
-	app-crypt/sbsigntools
-	x11-themes/sabayon-artwork-grub
-	sys-boot/os-prober
-	app-arch/xz-utils
-	>=sys-libs/ncurses-5.2-r5:0=
-	mount? ( sys-fs/fuse =sys-apps/pmount-0.9.99_alpha-r5:= )
-`
-		gr, err := ParseRDEPEND(rdepend)
-		It("Check error", func() {
-			Expect(err).Should(BeNil())
-		})
-		It("Check gr", func() {
-			Expect(gr).ShouldNot(BeNil())
-		})
-
-		It("Check deps #", func() {
-			Expect(len(gr.Dependencies)).Should(Equal(6))
-		})
-
-		It("Check dep1", func() {
-			Expect(*gr.Dependencies[0]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sbsigntools",
-						Category: "app-crypt",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-
-		It("Check dep2", func() {
-			Expect(*gr.Dependencies[1]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sabayon-artwork-grub",
-						Category: "x11-themes",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-
-		It("Check dep5", func() {
-			Expect(*gr.Dependencies[4]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:          "ncurses",
-						Category:      "sys-libs",
-						Slot:          "0=",
-						Version:       "5.2",
-						VersionSuffix: "-r5",
-						Condition:     _gentoo.PkgCondGreaterEqual,
-					},
-				},
-			))
-		})
-
-		It("Check dep6", func() {
-			Expect(*gr.Dependencies[5]).Should(Equal(
-				GentooDependency{
-					Use:          "mount",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps: []*GentooDependency{
-						&GentooDependency{
-							Use:          "",
-							UseCondition: _gentoo.PkgCondInvalid,
-							SubDeps:      make([]*GentooDependency, 0),
-							Dep: &_gentoo.GentooPackage{
-								Name:     "fuse",
-								Category: "sys-fs",
-								Slot:     "0",
-							},
-						},
-						&GentooDependency{
-							Use:          "",
-							UseCondition: _gentoo.PkgCondInvalid,
-							SubDeps:      make([]*GentooDependency, 0),
-							Dep: &_gentoo.GentooPackage{
-								Name:          "pmount",
-								Category:      "sys-apps",
-								Condition:     _gentoo.PkgCondEqual,
-								Version:       "0.9.99",
-								VersionSuffix: "_alpha-r5",
-								Slot:          "=",
-							},
-						},
-					},
-					Dep: nil,
-				},
-			))
-		})
-
-	})
-
-	Context("Parse RDEPEND4", func() {
-
-		rdepend := `
-	app-crypt/sbsigntools
-	x11-themes/sabayon-artwork-grub
-	sys-boot/os-prober
-	app-arch/xz-utils
-	>=sys-libs/ncurses-5.2-r5:0=
-	!mount? ( sys-fs/fuse =sys-apps/pmount-0.9.99_alpha-r5:= )
-`
-		gr, err := ParseRDEPEND(rdepend)
-		It("Check error", func() {
-			Expect(err).Should(BeNil())
-		})
-		It("Check gr", func() {
-			Expect(gr).ShouldNot(BeNil())
-		})
-
-		It("Check deps #", func() {
-			Expect(len(gr.Dependencies)).Should(Equal(6))
-		})
-
-		It("Check dep1", func() {
-			Expect(*gr.Dependencies[0]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sbsigntools",
-						Category: "app-crypt",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-
-		It("Check dep2", func() {
-			Expect(*gr.Dependencies[1]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sabayon-artwork-grub",
-						Category: "x11-themes",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-
-		It("Check dep5", func() {
-			Expect(*gr.Dependencies[4]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:          "ncurses",
-						Category:      "sys-libs",
-						Slot:          "0=",
-						Version:       "5.2",
-						VersionSuffix: "-r5",
-						Condition:     _gentoo.PkgCondGreaterEqual,
-					},
-				},
-			))
-		})
-
-		It("Check dep6", func() {
-			Expect(*gr.Dependencies[5]).Should(Equal(
-				GentooDependency{
-					Use:          "mount",
-					UseCondition: _gentoo.PkgCondNot,
-					SubDeps: []*GentooDependency{
-						&GentooDependency{
-							Use:          "",
-							UseCondition: _gentoo.PkgCondInvalid,
-							SubDeps:      make([]*GentooDependency, 0),
-							Dep: &_gentoo.GentooPackage{
-								Name:     "fuse",
-								Category: "sys-fs",
-								Slot:     "0",
-							},
-						},
-						&GentooDependency{
-							Use:          "",
-							UseCondition: _gentoo.PkgCondInvalid,
-							SubDeps:      make([]*GentooDependency, 0),
-							Dep: &_gentoo.GentooPackage{
-								Name:          "pmount",
-								Category:      "sys-apps",
-								Condition:     _gentoo.PkgCondEqual,
-								Version:       "0.9.99",
-								VersionSuffix: "_alpha-r5",
-								Slot:          "=",
-							},
-						},
-					},
-					Dep: nil,
-				},
-			))
-		})
-
-	})
-
-	Context("Parse RDEPEND5", func() {
-
-		rdepend := `
-	app-crypt/sbsigntools
-	>=sys-libs/ncurses-5.2-r5:0=
-	mount? (
-		sys-fs/fuse
-		=sys-apps/pmount-0.9.99_alpha-r5:=
-	)
-`
-		gr, err := ParseRDEPEND(rdepend)
-		It("Check error", func() {
-			Expect(err).Should(BeNil())
-		})
-		It("Check gr", func() {
-			Expect(gr).ShouldNot(BeNil())
-		})
-
-		It("Check deps #", func() {
-			Expect(len(gr.Dependencies)).Should(Equal(3))
-		})
-
-		It("Check dep1", func() {
-			Expect(*gr.Dependencies[0]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sbsigntools",
-						Category: "app-crypt",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-
-		It("Check dep2", func() {
-			Expect(*gr.Dependencies[1]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:          "ncurses",
-						Category:      "sys-libs",
-						Slot:          "0=",
-						Version:       "5.2",
-						VersionSuffix: "-r5",
-						Condition:     _gentoo.PkgCondGreaterEqual,
-					},
-				},
-			))
-		})
-
-		It("Check dep3", func() {
-			Expect(*gr.Dependencies[2]).Should(Equal(
-				GentooDependency{
-					Use:          "mount",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps: []*GentooDependency{
-						&GentooDependency{
-							Use:          "",
-							UseCondition: _gentoo.PkgCondInvalid,
-							SubDeps:      make([]*GentooDependency, 0),
-							Dep: &_gentoo.GentooPackage{
-								Name:     "fuse",
-								Category: "sys-fs",
-								Slot:     "0",
-							},
-						},
-						&GentooDependency{
-							Use:          "",
-							UseCondition: _gentoo.PkgCondInvalid,
-							SubDeps:      make([]*GentooDependency, 0),
-							Dep: &_gentoo.GentooPackage{
-								Name:          "pmount",
-								Category:      "sys-apps",
-								Condition:     _gentoo.PkgCondEqual,
-								Version:       "0.9.99",
-								VersionSuffix: "_alpha-r5",
-								Slot:          "=",
-							},
-						},
-					},
-					Dep: nil,
-				},
-			))
-		})
-
-	})
-
-	Context("Parse RDEPEND6", func() {
-
-		rdepend := `
-	app-crypt/sbsigntools
-	>=sys-libs/ncurses-5.2-r5:0=
-	mount? (
-		sys-fs/fuse
-		=sys-apps/pmount-0.9.99_alpha-r5:= )
-`
-		gr, err := ParseRDEPEND(rdepend)
-		It("Check error", func() {
-			Expect(err).Should(BeNil())
-		})
-		It("Check gr", func() {
-			Expect(gr).ShouldNot(BeNil())
-		})
-
-		It("Check deps #", func() {
-			Expect(len(gr.Dependencies)).Should(Equal(3))
-		})
-
-		It("Check dep1", func() {
-			Expect(*gr.Dependencies[0]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sbsigntools",
-						Category: "app-crypt",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-	})
-
-	Context("Parse RDEPEND7", func() {
-
-		rdepend := `
-	app-crypt/sbsigntools
-	>=sys-libs/ncurses-5.2-r5:0=
-	mount? (
-		sys-fs/fuse
-		=sys-apps/pmount-0.9.99_alpha-r5:=
-		ext2? (
-			sys-fs/genext2fs
-		)
-	)
-`
-		gr, err := ParseRDEPEND(rdepend)
-		It("Check error", func() {
-			Expect(err).Should(BeNil())
-		})
-		It("Check gr", func() {
-			Expect(gr).ShouldNot(BeNil())
-		})
-
-		It("Check deps #", func() {
-			Expect(len(gr.Dependencies)).Should(Equal(3))
-		})
-
-		It("Check dep1", func() {
-			Expect(*gr.Dependencies[0]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:     "sbsigntools",
-						Category: "app-crypt",
-						Slot:     "0",
-					},
-				},
-			))
-		})
-
-		It("Check dep2", func() {
-			Expect(*gr.Dependencies[1]).Should(Equal(
-				GentooDependency{
-					Use:          "",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps:      make([]*GentooDependency, 0),
-					Dep: &_gentoo.GentooPackage{
-						Name:          "ncurses",
-						Category:      "sys-libs",
-						Slot:          "0=",
-						Version:       "5.2",
-						VersionSuffix: "-r5",
-						Condition:     _gentoo.PkgCondGreaterEqual,
-					},
-				},
-			))
-		})
-
-		It("Check dep3", func() {
-			Expect(*gr.Dependencies[2]).Should(Equal(
-				GentooDependency{
-					Use:          "mount",
-					UseCondition: _gentoo.PkgCondInvalid,
-					SubDeps: []*GentooDependency{
-						&GentooDependency{
-							Use:          "",
-							UseCondition: _gentoo.PkgCondInvalid,
-							SubDeps:      make([]*GentooDependency, 0),
-							Dep: &_gentoo.GentooPackage{
-								Name:     "fuse",
-								Category: "sys-fs",
-								Slot:     "0",
-							},
-						},
-						&GentooDependency{
-							Use:          "",
-							UseCondition: _gentoo.PkgCondInvalid,
-							SubDeps:      make([]*GentooDependency, 0),
-							Dep: &_gentoo.GentooPackage{
-								Name:          "pmount",
-								Category:      "sys-apps",
-								Condition:     _gentoo.PkgCondEqual,
-								Version:       "0.9.99",
-								VersionSuffix: "_alpha-r5",
-								Slot:          "=",
-							},
-						},
-						&GentooDependency{
-							Use:          "ext2",
-							UseCondition: _gentoo.PkgCondInvalid,
-							SubDeps: []*GentooDependency{
-								&GentooDependency{
-									Use:          "",
-									UseCondition: _gentoo.PkgCondInvalid,
-									SubDeps:      make([]*GentooDependency, 0),
-									Dep: &_gentoo.GentooPackage{
-										Name:     "genext2fs",
-										Category: "sys-fs",
-										Slot:     "0",
-									},
-								},
-							},
-							Dep: nil,
-						},
-					},
-				},
-			))
-		})
-
-	})
-
-	Context("Simple test", func() {
-		for _, dbType := range []MemoryDB{InMemory, BoltDB} {
-			It("parses correctly deps", func() {
-				gb := NewGentooBuilder(&SimpleEbuildParser{}, 20, dbType)
-				tree, err := gb.Generate("../../../../tests/fixtures/overlay")
-				Expect(err).ToNot(HaveOccurred())
-				defer func() {
-					Expect(tree.Clean()).ToNot(HaveOccurred())
-				}()
-
-				Expect(len(tree.GetPackages())).To(Equal(10))
-
-				for _, p := range tree.World() {
-
-					Expect(p.GetName()).To(ContainSubstring("pinentry"))
-					Expect(p.GetVersion()).To(ContainSubstring("1."))
-				}
-
-			})
-		}
-	})
-
-})

pkg/tree/compiler_recipe.go

@@ -74,39 +74,92 @@ func (r *CompilerRecipe) Load(path string) error {
 			return errors.Wrap(err, "Error on walk path "+currentpath)
 		}
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
-		pack, err := ReadDefinitionFile(currentpath)
-		if err != nil {
-			return err
-		}
-		// Path is set only internally when tree is loaded from disk
-		pack.SetPath(filepath.Dir(currentpath))
+		switch info.Name() {
+		case DefinitionFile:
 
-		// Instead of rdeps, have a different tree for build deps.
-		compileDefPath := pack.Rel(CompilerDefinitionFile)
-		if helpers.Exists(compileDefPath) {
-			dat, err := ioutil.ReadFile(compileDefPath)
+			pack, err := ReadDefinitionFile(currentpath)
 			if err != nil {
-				return errors.Wrap(err,
-					"Error reading file "+CompilerDefinitionFile+" from "+
-						filepath.Dir(currentpath))
+				return err
+			}
+			// Path is set only internally when tree is loaded from disk
+			pack.SetPath(filepath.Dir(currentpath))
+
+			// Instead of rdeps, have a different tree for build deps.
+			compileDefPath := pack.Rel(CompilerDefinitionFile)
+			if helpers.Exists(compileDefPath) {
+
+				dat, err := helpers.RenderFiles(compileDefPath, currentpath, "")
+				if err != nil {
+					return errors.Wrap(err,
+						"Error templating file "+CompilerDefinitionFile+" from "+
+							filepath.Dir(currentpath))
+				}
+
+				packbuild, err := pkg.DefaultPackageFromYaml([]byte(dat))
+				if err != nil {
+					return errors.Wrap(err,
+						"Error reading yaml "+CompilerDefinitionFile+" from "+
+							filepath.Dir(currentpath))
+				}
+				pack.Requires(packbuild.GetRequires())
+				pack.Conflicts(packbuild.GetConflicts())
+			}
+
+			_, err = r.Database.CreatePackage(&pack)
+			if err != nil {
+				return errors.Wrap(err, "Error creating package "+pack.GetName())
+			}
+
+		case CollectionFile:
+			dat, err := ioutil.ReadFile(currentpath)
+			if err != nil {
+				return errors.Wrap(err, "Error reading file "+currentpath)
+			}
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			packsRaw, err := pkg.GetRawPackages(dat)
+
+			for _, pack := range packs {
+				pack.SetPath(filepath.Dir(currentpath))
+
+				// Instead of rdeps, have a different tree for build deps.
+				compileDefPath := pack.Rel(CompilerDefinitionFile)
+				if helpers.Exists(compileDefPath) {
+
+					raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
+					buildyaml, err := ioutil.ReadFile(compileDefPath)
+					if err != nil {
+						return errors.Wrap(err, "Error reading file "+currentpath)
+					}
+					dat, err := helpers.RenderHelm(string(buildyaml), raw, map[string]interface{}{})
+					if err != nil {
+						return errors.Wrap(err,
+							"Error templating file "+CompilerDefinitionFile+" from "+
+								filepath.Dir(currentpath))
+					}
+
+					packbuild, err := pkg.DefaultPackageFromYaml([]byte(dat))
+					if err != nil {
+						return errors.Wrap(err,
+							"Error reading yaml "+CompilerDefinitionFile+" from "+
+								filepath.Dir(currentpath))
+					}
+					pack.Requires(packbuild.GetRequires())
+					pack.Conflicts(packbuild.GetConflicts())
+				}
+
+				_, err = r.Database.CreatePackage(&pack)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+pack.GetName())
+				}
 			}
-			packbuild, err := pkg.DefaultPackageFromYaml(dat)
-			if err != nil {
-				return errors.Wrap(err,
-					"Error reading yaml "+CompilerDefinitionFile+" from "+
-						filepath.Dir(currentpath))
-			}
-			pack.Requires(packbuild.GetRequires())
-			pack.Conflicts(packbuild.GetConflicts())
-		}
 
-		_, err = r.Database.CreatePackage(&pack)
-		if err != nil {
-			return errors.Wrap(err, "Error creating package "+pack.GetName())
 		}
 
 		return nil

pkg/tree/installer_recipe.go

@@ -85,7 +85,7 @@ func (r *InstallerRecipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
@@ -93,16 +93,35 @@ func (r *InstallerRecipe) Load(path string) error {
 		if err != nil {
 			return errors.Wrap(err, "Error reading file "+currentpath)
 		}
-		pack, err := pkg.DefaultPackageFromYaml(dat)
-		if err != nil {
-			return errors.Wrap(err, "Error reading yaml "+currentpath)
-		}
 
-		// Path is set only internally when tree is loaded from disk
-		pack.SetPath(filepath.Dir(currentpath))
-		_, err = r.Database.CreatePackage(&pack)
-		if err != nil {
-			return errors.Wrap(err, "Error creating package "+pack.GetName())
+		switch info.Name() {
+		case DefinitionFile:
+			pack, err := pkg.DefaultPackageFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+
+			// Path is set only internally when tree is loaded from disk
+			pack.SetPath(filepath.Dir(currentpath))
+			_, err = r.Database.CreatePackage(&pack)
+			if err != nil {
+				return errors.Wrap(err, "Error creating package "+pack.GetName())
+			}
+
+		case CollectionFile:
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			for _, p := range packs {
+				// Path is set only internally when tree is loaded from disk
+				p.SetPath(filepath.Dir(currentpath))
+				_, err = r.Database.CreatePackage(&p)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+p.GetName())
+				}
+			}
+
 		}
 
 		return nil

pkg/tree/recipes.go

@@ -34,6 +34,7 @@ import (
 
 const (
 	DefinitionFile = "definition.yaml"
+	CollectionFile = "collection.yaml"
 )
 
 func NewGeneralRecipe(db pkg.PackageDatabase) Builder { return &Recipe{Database: db} }
@@ -94,7 +95,7 @@ func (r *Recipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
@@ -102,16 +103,34 @@ func (r *Recipe) Load(path string) error {
 		if err != nil {
 			return errors.Wrap(err, "Error reading file "+currentpath)
 		}
-		pack, err := pkg.DefaultPackageFromYaml(dat)
-		if err != nil {
-			return errors.Wrap(err, "Error reading yaml "+currentpath)
-		}
 
-		// Path is set only internally when tree is loaded from disk
-		pack.SetPath(filepath.Dir(currentpath))
-		_, err = r.Database.CreatePackage(&pack)
-		if err != nil {
-			return errors.Wrap(err, "Error creating package "+pack.GetName())
+		switch info.Name() {
+		case DefinitionFile:
+			pack, err := pkg.DefaultPackageFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+
+			// Path is set only internally when tree is loaded from disk
+			pack.SetPath(filepath.Dir(currentpath))
+			_, err = r.Database.CreatePackage(&pack)
+			if err != nil {
+				return errors.Wrap(err, "Error creating package "+pack.GetName())
+			}
+		case CollectionFile:
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			for _, p := range packs {
+				// Path is set only internally when tree is loaded from disk
+				p.SetPath(filepath.Dir(currentpath))
+				_, err = r.Database.CreatePackage(&p)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+p.GetName())
+				}
+			}
+
 		}
 
 		return nil

pkg/tree/recipes_test.go

@@ -1,136 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-// Recipe is a builder imeplementation.
-
-// It reads a Tree and spit it in human readable form (YAML), called recipe,
-// It also loads a tree (recipe) from a YAML (to a db, e.g. BoltDB), allowing to query it
-// with the solver, using the package object.
-package tree_test
-
-import (
-	"io/ioutil"
-	"os"
-
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
-	gentoo "github.com/mudler/luet/pkg/tree/builder/gentoo"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	. "github.com/mudler/luet/pkg/tree"
-)
-
-type FakeParser struct {
-}
-
-var _ = Describe("Recipe", func() {
-	for _, dbType := range []gentoo.MemoryDB{gentoo.InMemory, gentoo.BoltDB} {
-		Context("Tree generation and storing", func() {
-			It("parses and writes a tree", func() {
-				tmpdir, err := ioutil.TempDir("", "tree")
-				Expect(err).ToNot(HaveOccurred())
-				defer os.RemoveAll(tmpdir) // clean up
-
-				gb := gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, 20, dbType)
-				tree, err := gb.Generate("../../tests/fixtures/overlay")
-				Expect(err).ToNot(HaveOccurred())
-				defer func() {
-					Expect(tree.Clean()).ToNot(HaveOccurred())
-				}()
-
-				Expect(len(tree.GetPackages())).To(Equal(10))
-
-				generalRecipe := NewGeneralRecipe(tree)
-				err = generalRecipe.Save(tmpdir)
-				Expect(err).ToNot(HaveOccurred())
-			})
-		})
-
-		Context("Reloading trees", func() {
-			It("writes and reads back the same tree", func() {
-				tmpdir, err := ioutil.TempDir("", "tree")
-				Expect(err).ToNot(HaveOccurred())
-				defer os.RemoveAll(tmpdir) // clean up
-
-				gb := gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, 20, dbType)
-				tree, err := gb.Generate("../../tests/fixtures/overlay")
-				Expect(err).ToNot(HaveOccurred())
-				defer func() {
-					Expect(tree.Clean()).ToNot(HaveOccurred())
-				}()
-
-				Expect(len(tree.GetPackages())).To(Equal(10))
-
-				generalRecipe := NewGeneralRecipe(tree)
-				err = generalRecipe.Save(tmpdir)
-				Expect(err).ToNot(HaveOccurred())
-
-				db := pkg.NewInMemoryDatabase(false)
-				generalRecipe = NewGeneralRecipe(db)
-
-				generalRecipe.WithDatabase(nil)
-				Expect(generalRecipe.GetDatabase()).To(BeNil())
-
-				err = generalRecipe.Load(tmpdir)
-				Expect(err).ToNot(HaveOccurred())
-
-				Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
-
-				for _, p := range tree.World() {
-					Expect(p.GetName()).To(ContainSubstring("pinentry"))
-				}
-			})
-		})
-
-		Context("Simple solving with the fixture tree", func() {
-			It("writes and reads back the same tree", func() {
-				tmpdir, err := ioutil.TempDir("", "tree")
-				Expect(err).ToNot(HaveOccurred())
-				defer os.RemoveAll(tmpdir) // clean up
-
-				gb := gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, 20, dbType)
-				tree, err := gb.Generate("../../tests/fixtures/overlay")
-				Expect(err).ToNot(HaveOccurred())
-				defer func() {
-					Expect(tree.Clean()).ToNot(HaveOccurred())
-				}()
-
-				Expect(len(tree.GetPackages())).To(Equal(10))
-
-				pack, err := tree.FindPackage(&pkg.DefaultPackage{
-					Name:     "pinentry",
-					Version:  "1.0.0-r2",
-					Category: "app-crypt",
-				}) // Note: the definition depends on pinentry-base without an explicit version
-				Expect(err).ToNot(HaveOccurred())
-
-				s := solver.NewSolver(pkg.NewInMemoryDatabase(false), tree, tree)
-				solution, err := s.Install([]pkg.Package{pack})
-				Expect(err).ToNot(HaveOccurred())
-				Expect(len(solution)).To(Equal(33))
-
-				var allSol string
-				for _, sol := range solution {
-					allSol = allSol + "\n" + sol.ToString()
-				}
-
-				Expect(allSol).To(ContainSubstring("app-crypt/pinentry-base 1.0.0 installed"))
-				Expect(allSol).To(ContainSubstring("app-crypt/pinentry 1.1.0-r2 not installed"))
-				Expect(allSol).To(ContainSubstring("app-crypt/pinentry 1.0.0-r2 installed"))
-			})
-		})
-	}
-})

pkg/tree/tree_test.go

@@ -59,7 +59,7 @@ var _ = Describe("Tree", func() {
 				Expect(len(CfromD.GetRequires()) != 0).To(BeTrue())
 				Expect(CfromD.GetRequires()[0].GetName()).To(Equal("b"))
 
-				s := solver.NewSolver(pkg.NewInMemoryDatabase(false), generalRecipe.GetDatabase(), db)
+				s := solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false), generalRecipe.GetDatabase(), db)
 				pack, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 				Expect(err).ToNot(HaveOccurred())
 
@@ -69,31 +69,25 @@ var _ = Describe("Tree", func() {
 				solution, err = solution.Order(generalRecipe.GetDatabase(), pack.GetFingerPrint())
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution[0].Package.GetName()).To(Equal("a"))
-				Expect(solution[0].Value).To(BeFalse())
+				Expect(solution[0].Package.GetName()).To(Equal("b"))
+				Expect(solution[0].Value).To(BeTrue())
 
-				Expect(solution[1].Package.GetName()).To(Equal("b"))
+				Expect(solution[1].Package.GetName()).To(Equal("c"))
 				Expect(solution[1].Value).To(BeTrue())
 
-				Expect(solution[2].Package.GetName()).To(Equal("c"))
+				Expect(solution[2].Package.GetName()).To(Equal("d"))
 				Expect(solution[2].Value).To(BeTrue())
-
-				Expect(solution[3].Package.GetName()).To(Equal("d"))
-				Expect(solution[3].Value).To(BeTrue())
-				Expect(len(solution)).To(Equal(4))
+				Expect(len(solution)).To(Equal(3))
 
 				newsolution := solution.Drop(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
-				Expect(len(newsolution)).To(Equal(3))
+				Expect(len(newsolution)).To(Equal(2))
 
-				Expect(newsolution[0].Package.GetName()).To(Equal("a"))
-				Expect(newsolution[0].Value).To(BeFalse())
+				Expect(newsolution[0].Package.GetName()).To(Equal("b"))
+				Expect(newsolution[0].Value).To(BeTrue())
 
-				Expect(newsolution[1].Package.GetName()).To(Equal("b"))
+				Expect(newsolution[1].Package.GetName()).To(Equal("c"))
 				Expect(newsolution[1].Value).To(BeTrue())
 
-				Expect(newsolution[2].Package.GetName()).To(Equal("c"))
-				Expect(newsolution[2].Value).To(BeTrue())
-
 			}
 		})
 	})
@@ -131,7 +125,7 @@ var _ = Describe("Tree", func() {
 				Expect(len(CfromD.GetRequires()) != 0).To(BeTrue())
 				Expect(CfromD.GetRequires()[0].GetName()).To(Equal("b"))
 
-				s := solver.NewSolver(pkg.NewInMemoryDatabase(false), generalRecipe.GetDatabase(), db)
+				s := solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false), generalRecipe.GetDatabase(), db)
 
 				Dd, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 				Expect(err).ToNot(HaveOccurred())
@@ -146,11 +140,11 @@ var _ = Describe("Tree", func() {
 
 				base, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "base", Category: "layer", Version: "0.2"})
 				Expect(err).ToNot(HaveOccurred())
-				Expect(solution).To(ContainElement(solver.PackageAssert{Package: pack.(*pkg.DefaultPackage), Value: false}))
+				Expect(solution).ToNot(ContainElement(solver.PackageAssert{Package: pack.(*pkg.DefaultPackage), Value: true}))
 				Expect(solution).To(ContainElement(solver.PackageAssert{Package: D.(*pkg.DefaultPackage), Value: true}))
-				Expect(solution).To(ContainElement(solver.PackageAssert{Package: extra.(*pkg.DefaultPackage), Value: false}))
-				Expect(solution).To(ContainElement(solver.PackageAssert{Package: base.(*pkg.DefaultPackage), Value: false}))
-				Expect(len(solution)).To(Equal(6))
+				Expect(solution).ToNot(ContainElement(solver.PackageAssert{Package: extra.(*pkg.DefaultPackage), Value: true}))
+				Expect(solution).ToNot(ContainElement(solver.PackageAssert{Package: base.(*pkg.DefaultPackage), Value: true}))
+				Expect(len(solution)).To(Equal(3))
 			}
 		})
 	})