Tag 0.11.2

Allow to add arguments to the backend build arguments

Fixes #146

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: mudler
+
+---
+
+<!-- Thanks for helping us to improve Luet! We welcome all bug reports. Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
+
+**Luet version:**
+<!-- Provide the output from "luet --version" -->
+
+**CPU architecture, OS, and Version:**
+<!-- Provide the output from "uname -a"  -->
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+<!-- Steps to reproduce the behavior, including the luet command used -->
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Logs**
+<!-- If applicable, add logs with the "--debug" flag enabled to help explain your problem. -->
+
+**Additional context**
+<!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: mudler
+
+---
+
+<!-- Thanks for helping us to improve Luet! We welcome all feature requests. Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
+
+**Is your feature request related to a problem? Please describe.**
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]  -->
+
+**Describe the solution you'd like**
+<!-- A clear and concise description of what you want to happen.  -->
+
+**Describe alternatives you've considered**
+<!-- A clear and concise description of any alternative solutions or features you've considered.  -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/workflows/release.yml

@@ -0,0 +1,47 @@
+on: push
+name: Build and release on push
+jobs:
+  release:
+    name: Test and Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+      - name: Build test
+        run: sudo -E env "PATH=$PATH" make multiarch-build-small
+      - name: Login to quay with img
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Tests with Img backend
+        run: |
+            sudo -E env "PATH=$PATH" \
+                    env "LUET_BACKEND=img" \
+                    make test-integration
+      - name: Tests
+        run: |
+              sudo -E \
+                    env "PATH=$PATH" \
+                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                    make test-integration test-coverage
+      - name: Build
+        run: sudo -E env "PATH=$PATH" make multiarch-build-small && sudo chmod -R 777 release/
+      - name: Release
+        uses: fnkr/github-action-ghr@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GHR_PATH: release/
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,30 @@
+
+on: pull_request
+name: Build and Test
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+    - name: Build
+      run: sudo -E env "PATH=$PATH" make multiarch-build-small
+    - name: Tests with Img backend
+      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make test-integration test-coverage

.travis.yml

@@ -1,19 +1,35 @@
+dist: bionic
 language: go
-services:
-  - docker
 go:
   - "1.14"
+
 env:
-  - "GO15VENDOREXPERIMENT=1"
+  global:
+    - "GO15VENDOREXPERIMENT=1"
+  jobs:
+    - "DOCKER_BUILDKIT=0"
+    - "DOCKER_BUILDKIT=1"
 before_install:
-  - make deps
-  - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && mkdir -p $HOME/bin && export PATH=$PATH:$HOME/bin && mv container-diff-linux-amd64 $HOME/bin/container-diff
+  - sudo rm -rf /var/lib/apt/lists/*
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) edge"
+  - sudo apt-get update
+  - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
+  - export DOCKER_CLI_EXPERIMENTAL=enabled
+  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  - mkdir -vp ~/.docker/cli-plugins/
+  - curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+  - chmod a+x ~/.docker/cli-plugins/docker-buildx
+  - docker buildx version
+  - sudo -E env "PATH=$PATH" apt-get install -y libcap2-bin
+  - sudo -E env "PATH=$PATH" make deps
 script:
-  - make multiarch-build test-integration test-coverage
-after_success:
-  - |
-      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
-        git config --global user.name "Deployer" && git config --global user.email foo@bar.com
-        go get github.com/tcnksm/ghr
-        ghr -u mudler -r luet --replace $TRAVIS_TAG release/
-      fi
+  - sudo -E env "PATH=$PATH" make multiarch-build test-integration test-coverage
+
+#after_success:
+#  - |
+#      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+#        sudo -E env "PATH=$PATH" git config --global user.name "Deployer" && git config --global user.email foo@bar.com
+#        sudo -E env "PATH=$PATH" go get github.com/tcnksm/ghr
+#        sudo -E env "PATH=$PATH" ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+#      fi

Dockerfile

@@ -1,6 +1,7 @@
 FROM golang as builder
+RUN apt-get update && apt-get install -y upx
 ADD . /luet
-RUN cd /luet && make build
+RUN cd /luet && make build-small
 
 FROM scratch
 ENV LUET_NOLOCK=true

Makefile

@@ -1,7 +1,7 @@
 
 # go tool nm ./luet | grep Commit
-LDFLAGS += -X "github.com/mudler/luet/cmd.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S %Z')"
-LDFLAGS += -X "github.com/mudler/luet/cmd.BuildCommit=$(shell git rev-parse HEAD)"
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S %Z')"
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildCommit=$(shell git rev-parse HEAD)"
 
 NAME ?= luet
 PACKAGE_NAME ?= $(NAME)
@@ -9,7 +9,6 @@ PACKAGE_CONFLICT ?= $(PACKAGE_NAME)-beta
 REVISION := $(shell git rev-parse --short HEAD || echo dev)
 VERSION := $(shell git describe --tags || echo $(REVISION))
 VERSION := $(shell echo $(VERSION) | sed -e 's/^v//g')
-ITTERATION := $(shell date +%s)
 BUILD_PLATFORMS ?= -osarch="linux/amd64" -osarch="linux/386" -osarch="linux/arm"
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
@@ -66,14 +65,15 @@ deps:
 build:
 	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)'
 
+.PHONY: build-small
+build-small:
+	@$(MAKE) LDFLAGS+="-s -w" build
+	upx --brute -1 $(NAME)
+
 .PHONY: image
 image:
 	docker build --rm -t luet/base .
 
-.PHONY: gox-build
-gox-build:
-	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
-
 .PHONY: lint
 lint:
 	golint ./... | grep -v "be unexported"
@@ -88,6 +88,11 @@ test-docker:
 				--workdir /go/src/github.com/mudler/luet -ti golang:latest \
 				bash -c "make test"
 
-.PHONY: multiarch-build
 multiarch-build:
-	gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+
+multiarch-build-small:
+	@$(MAKE) LDFLAGS+="-s -w" multiarch-build
+	for file in $(ROOT_DIR)/release/* ; do \
+		upx --brute -1 $${file} ; \
+	done

README.md

@@ -1,13 +1,16 @@
 # luet - Container-based Package manager
+
 [![Docker Repository on Quay](https://quay.io/repository/luet/base/status "Docker Repository on Quay")](https://quay.io/repository/luet/base)
 [![Go Report Card](https://goreportcard.com/badge/github.com/mudler/luet)](https://goreportcard.com/report/github.com/mudler/luet)
 [![Build Status](https://travis-ci.org/mudler/luet.svg?branch=master)](https://travis-ci.org/mudler/luet)
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
 [![codecov](https://codecov.io/gh/mudler/luet/branch/master/graph/badge.svg)](https://codecov.io/gh/mudler/luet)
 
-Luet is a multi-platform Package Manager based off from containers - it uses Docker (and other tech) to sandbox your builds and generate packages from them. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
+[![asciicast](https://asciinema.org/a/388348.svg)](https://asciinema.org/a/388348)
 
-It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/specfile/) in YAML notation to define both packages and rootfs. As it is based on containers, it can be used to build seed stages for Linux From Scratch installations and it can build and track updates for those systems.
+Luet is a multi-platform Package Manager based off from containers - it uses Docker (and others) to build packages. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
+
+It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/) in YAML notation to define both [packages](https://luet-lab.github.io/docs/docs/concepts/packages/) and [rootfs](https://luet-lab.github.io/docs/docs/concepts/packages/#package-layers). As it is based on containers, it can be also used to build stages for Linux From Scratch installations and it can build and track updates for those systems.
 
 It is written entirely in Golang and where used as package manager, it can run in from scratch environment, with zero dependencies.
 
@@ -16,25 +19,41 @@ It is written entirely in Golang and where used as package manager, it can run i
 - Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.
 - It builds, installs, uninstalls and perform upgrades on machines
 - Installer doesn't depend on anything ( 0 dep installer !), statically built
-- Support for packages as "layers"
-- It uses SAT solving techniques to solve the deptree ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
+- You can install it aside also with your current distro package manager, and start building and distributing your packages
+- [Support for packages as "layers"](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/#building-strategies)
+- [It uses SAT solving techniques to solve the deptree](https://luet-lab.github.io/docs/docs/concepts/overview/constraints/) ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
+- Support for [collections](https://luet-lab.github.io/docs/docs/concepts/packages/collections/) and [templated package definitions](https://luet-lab.github.io/docs/docs/concepts/packages/templates/)
+- [Can be extended with Plugins and Extensions](https://luet-lab.github.io/docs/docs/concepts/plugins-and-extensions/)
+- [Can build packages in Kubernetes (experimental)](https://github.com/mudler/luet-k8s)
 
 ## Install
 
-To install luet, you can grab a release on the [Release page](https://github.com/mudler/luet/releases) or compile it in your machine (requires Golang installed):
+To install luet, you can grab a release on the [Release page](https://github.com/mudler/luet/releases) or to install it in your system:
 
-    $ git clone https://github.com/mudler/luet.git
-    $ cd luet
-    $ make build
+```bash
+$ curl https://get.mocaccino.org/luet/get_luet_root.sh | sudo sh
+$ luet search ...
+$ luet install ..
+$ luet --help
+```
 
-## Status
+## Build from source
 
-Luet is not feature-complete yet, it can build, install/uninstall/upgrade packages - but it doesn't support yet all the features you would normally expect from a Package Manager nowadays.
+```bash
+$ git clone https://github.com/mudler/luet.git
+$ cd luet
+$ make build
+```
+
+## Documentation
+
+[Documentation](https://luet-lab.github.io/docs) is available, or
+run `luet --help`,  any subcommand is documented as well, try e.g.: `luet build --help`.
 
 # Dependency solving
 
 Luet uses SAT and Reinforcement learning engine for dependency solving.
-It encodes the package requirements into a SAT problem, using gophersat to solve the dependency tree and give a concrete model as result.
+It encodes the package requirements into a SAT problem, using [gophersat](https://github.com/crillab/gophersat) to solve the dependency tree and give a concrete model as result.
 
 ## SAT encoding
 
@@ -48,10 +67,6 @@ when they arises while trying to validate your queries against the system model.
 
 To leverage it, simply pass ```--solver-type qlearning``` to the subcommands that supports it ( you can check out by invoking ```--help``` ).
 
-## Documentation
-
-[Documentation](https://luet-lab.github.io/docs) is available, or
-run `luet --help`,  any subcommand is documented as well, try e.g.: `luet build --help`.
 
 ## Authors
 

cmd/build.go

@@ -15,15 +15,19 @@
 package cmd
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/spf13/cobra"
@@ -33,9 +37,30 @@ import (
 var buildCmd = &cobra.Command{
 	Use:   "build <package name> <package name> <package name> ...",
 	Short: "build a package or a tree",
-	Long:  `build packages or trees from luet tree definitions. Packages are in [category]/[name]-[version] form`,
-	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("clean", cmd.Flags().Lookup("clean"))
+	Long: `Builds one or more packages from a tree (current directory is implied):
+
+	$ luet build utils/busybox utils/yq ...
+	
+Builds all packages
+	
+	$ luet build --all
+	
+Builds only the leaf packages:
+	
+	$ luet build --full
+
+Build package revdeps:
+	
+	$ luet build --revdeps utils/yq
+
+Build package without dependencies (needs the images already in the host, or either need to be available online):
+
+	$ luet build --nodeps utils/yq ...
+
+Build packages specifying multiple definition trees:
+
+	$ luet build --tree overlay/path --tree overlay/path2 utils/yq ...
+`, PreRun: func(cmd *cobra.Command, args []string) {
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
 		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
@@ -46,23 +71,26 @@ var buildCmd = &cobra.Command{
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
 		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("values", cmd.Flags().Lookup("values"))
+		viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
 
 		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
 		viper.BindPFlag("push", cmd.Flags().Lookup("push"))
 		viper.BindPFlag("pull", cmd.Flags().Lookup("pull"))
+		viper.BindPFlag("wait", cmd.Flags().Lookup("wait"))
 		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
 
-		LuetCfg.Viper.BindPFlag("keep-exported-images", cmd.Flags().Lookup("keep-exported-images"))
-
 		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
 		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
 		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("general.show_build_output", cmd.Flags().Lookup("live-output"))
+		LuetCfg.Viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
+
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
-		clean := viper.GetBool("clean")
-		src := viper.GetString("tree")
+		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("destination")
 		concurrency := LuetCfg.GetGeneral().Concurrency
 		backendType := viper.GetString("backend")
@@ -72,22 +100,28 @@ var buildCmd = &cobra.Command{
 		databaseType := viper.GetString("database")
 		compressionType := viper.GetString("compression")
 		imageRepository := viper.GetString("image-repository")
+		values := viper.GetString("values")
+		wait := viper.GetBool("wait")
 		push := viper.GetBool("push")
 		pull := viper.GetBool("pull")
 		keepImages := viper.GetBool("keep-images")
 		nodeps := viper.GetBool("nodeps")
 		onlydeps := viper.GetBool("onlydeps")
-		keepExportedImages := viper.GetBool("keep-exported-images")
+		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
+		full, _ := cmd.Flags().GetBool("full")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		var results Results
+		backendArgs := viper.GetStringSlice("backend-args")
 
-		compilerSpecs := compiler.NewLuetCompilationspecs()
-		var compilerBackend compiler.CompilerBackend
-		var db pkg.PackageDatabase
-		switch backendType {
-		case "img":
-			compilerBackend = backend.NewSimpleImgBackend()
-		case "docker":
-			compilerBackend = backend.NewSimpleDockerBackend()
+		out, _ := cmd.Flags().GetString("output")
+		if out != "terminal" {
+			LuetCfg.GetLogging().SetLogLevel("error")
 		}
+		pretend, _ := cmd.Flags().GetBool("pretend")
+		compilerSpecs := compiler.NewLuetCompilationspecs()
+		var db pkg.PackageDatabase
+
+		compilerBackend := backend.NewBackend(backendType)
 
 		switch databaseType {
 		case "memory":
@@ -105,14 +139,17 @@ var buildCmd = &cobra.Command{
 
 		generalRecipe := tree.NewCompilerRecipe(db)
 
-		Info("Loading", src)
-		Info("Building in", dst)
+		for _, src := range treePaths {
+			Info("Loading tree", src)
 
-		err := generalRecipe.Load(src)
-		if err != nil {
-			Fatal("Error: " + err.Error())
+			err := generalRecipe.Load(src)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
 		}
 
+		Info("Building in", dst)
+
 		stype := LuetCfg.Viper.GetString("solver.type")
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
@@ -123,23 +160,43 @@ var buildCmd = &cobra.Command{
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
 		LuetCfg.GetSolverOptions().MaxAttempts = attempts
 
+		LuetCfg.GetGeneral().ShowBuildOutput = LuetCfg.Viper.GetBool("general.show_build_output")
+
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
 		opts := compiler.NewDefaultCompilerOptions()
 		opts.SolverOptions = *LuetCfg.GetSolverOptions()
 		opts.ImageRepository = imageRepository
-		opts.Clean = clean
 		opts.PullFirst = pull
 		opts.KeepImg = keepImages
 		opts.Push = push
 		opts.OnlyDeps = onlydeps
 		opts.NoDeps = nodeps
-		opts.KeepImageExport = keepExportedImages
+		opts.Wait = wait
+		opts.PackageTargetOnly = onlyTarget
+		opts.BuildValuesFile = values
+		var solverOpts solver.Options
+		if concurrent {
+			solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
+		} else {
+			solverOpts = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+		}
 
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts)
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts, solverOpts)
+		luetCompiler.SetBackendArgs(backendArgs)
 		luetCompiler.SetConcurrency(concurrency)
 		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
-		if !all {
+		if full {
+			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
+			if err != nil {
+				Fatal(err.Error())
+			}
+			for _, spec := range specs {
+				Info(":package: Selecting ", spec.GetPackage().GetName(), spec.GetPackage().GetVersion())
+
+				compilerSpecs.Add(spec)
+			}
+		} else if !all {
 			for _, a := range args {
 
 				pack, err := helpers.ParsePackageStr(a)
@@ -174,9 +231,58 @@ var buildCmd = &cobra.Command{
 		if revdeps {
 			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
+		} else if pretend {
+			toCalculate := []compiler.CompilationSpec{}
+			if full {
+				var err error
+				toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...)
+				if err != nil {
+					errs = append(errs, err)
+				}
+			} else {
+				toCalculate = compilerSpecs.All()
+			}
+
+			for _, sp := range toCalculate {
+				packs, err := luetCompiler.ComputeDepTree(sp)
+				if err != nil {
+					errs = append(errs, err)
+				}
+				for _, p := range packs {
+					results.Packages = append(results.Packages,
+						PackageResult{
+							Name:       p.Package.GetName(),
+							Version:    p.Package.GetVersion(),
+							Category:   p.Package.GetCategory(),
+							Repository: "",
+							Hidden:     p.Package.IsHidden(),
+							Target:     sp.GetPackage().HumanReadableString(),
+						})
+				}
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			case "terminal":
+				for _, p := range results.Packages {
+					Info(p.String())
+				}
+			}
 		} else {
 			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
-
 		}
 		if len(errs) != 0 {
 			for _, e := range errs {
@@ -195,27 +301,37 @@ func init() {
 	if err != nil {
 		Fatal(err)
 	}
-	buildCmd.Flags().Bool("clean", true, "Build all packages without considering the packages present in the build directory")
-	buildCmd.Flags().String("tree", path, "Source luet tree")
+
+	buildCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the tree to use.")
 	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
 	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
 	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
-	buildCmd.Flags().Bool("all", false, "Build all packages in the tree")
-	buildCmd.Flags().String("destination", path, "Destination folder")
-	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
+	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
+	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
+	buildCmd.Flags().String("values", "", "Build values file to interpolate with each package")
+	buildCmd.Flags().StringSliceP("backend-args", "a", []string{}, "Backend args")
+
+	buildCmd.Flags().String("destination", filepath.Join(path, "build"), "Destination folder")
+	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip, zstd")
 	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
 	buildCmd.Flags().Bool("push", false, "Push images to a hub")
 	buildCmd.Flags().Bool("pull", false, "Pull images from a hub")
+	buildCmd.Flags().Bool("wait", false, "Don't build all intermediate images, but wait for them until they are available")
 	buildCmd.Flags().Bool("keep-images", true, "Keep built docker images in the host")
 	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
 	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
-	buildCmd.Flags().Bool("keep-exported-images", false, "Keep exported images used during building")
-
+	buildCmd.Flags().Bool("only-target-package", false, "Build packages of only the required target. Otherwise builds all the necessary ones not present in the destination")
 	buildCmd.Flags().String("solver-type", "", "Solver strategy")
 	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	buildCmd.Flags().Bool("live-output", LuetCfg.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
+
+	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
+
+	buildCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 
 	RootCmd.AddCommand(buildCmd)
 }

cmd/config.go

@@ -19,14 +19,16 @@ import (
 	"fmt"
 
 	config "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
 
 	"github.com/spf13/cobra"
 )
 
 var configCmd = &cobra.Command{
-	Use:   "config",
-	Short: "Print config",
-	Long:  `Show luet configuration`,
+	Use:     "config",
+	Short:   "Print config",
+	Long:    `Show luet configuration`,
+	Aliases: []string{"c"},
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println(config.LuetCfg.GetLogging())
 		fmt.Println(config.LuetCfg.GetGeneral())
@@ -51,6 +53,23 @@ var configCmd = &cobra.Command{
 			}
 		}
 
+		if len(config.LuetCfg.ConfigProtectConfDir) > 0 {
+
+			// Load config protect configs
+			installer.LoadConfigProtectConfs(config.LuetCfg)
+
+			fmt.Println("config_protect_confdir:")
+			for _, dir := range config.LuetCfg.ConfigProtectConfDir {
+				fmt.Println("  - ", dir)
+			}
+
+			if len(config.LuetCfg.GetConfigProtectConfFiles()) > 0 {
+				fmt.Println("protect_conf_files:")
+				for _, file := range config.LuetCfg.GetConfigProtectConfFiles() {
+					fmt.Println("  - ", file.String())
+				}
+			}
+		}
 	},
 }
 

cmd/convert.go

@@ -1,101 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-package cmd
-
-import (
-	"io/ioutil"
-
-	. "github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
-	pkg "github.com/mudler/luet/pkg/package"
-	tree "github.com/mudler/luet/pkg/tree"
-
-	"github.com/mudler/luet/pkg/tree/builder/gentoo"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-)
-
-var convertCmd = &cobra.Command{
-	Use:   "convert",
-	Short: "convert other package manager tree into luet",
-	Long:  `Parses external PM and produces a luet parsable tree`,
-	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
-		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
-	},
-	Run: func(cmd *cobra.Command, args []string) {
-
-		t := viper.GetString("type")
-		databaseType := viper.GetString("database")
-		var db pkg.PackageDatabase
-
-		if len(args) != 2 {
-			Fatal("Incorrect number of arguments")
-		}
-
-		input := args[0]
-		output := args[1]
-		Info("Converting trees from " + input + " [" + t + "]")
-
-		var builder tree.Parser
-		switch t {
-		case "gentoo":
-			builder = gentoo.NewGentooBuilder(
-				&gentoo.SimpleEbuildParser{},
-				LuetCfg.GetGeneral().Concurrency,
-				gentoo.InMemory)
-		default: // dup
-			builder = gentoo.NewGentooBuilder(
-				&gentoo.SimpleEbuildParser{},
-				LuetCfg.GetGeneral().Concurrency,
-				gentoo.InMemory)
-		}
-
-		switch databaseType {
-		case "memory":
-			db = pkg.NewInMemoryDatabase(false)
-		case "boltdb":
-			tmpdir, err := ioutil.TempDir("", "package")
-			if err != nil {
-				Fatal(err)
-			}
-			db = pkg.NewBoltDatabase(tmpdir)
-		}
-		defer db.Clean()
-
-		packageTree, err := builder.Generate(input)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-
-		defer packageTree.Clean()
-		Info("Tree generated")
-
-		generalRecipe := tree.NewGeneralRecipe(packageTree)
-		Info("Saving generated tree to " + output)
-
-		err = generalRecipe.Save(output)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-	},
-}
-
-func init() {
-	convertCmd.Flags().String("type", "gentoo", "source type")
-	convertCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
-
-	RootCmd.AddCommand(convertCmd)
-}

cmd/create-repo.go

@@ -16,8 +16,10 @@ package cmd
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -30,11 +32,31 @@ import (
 var createrepoCmd = &cobra.Command{
 	Use:   "create-repo",
 	Short: "Create a luet repository from a build",
-	Long:  `Generate and renew repository metadata`,
+	Long: `Builds tree metadata from a set of packages and a tree definition:
+
+	$ luet create-repo
+
+Provide specific paths for packages, tree, and metadata output which is generated:
+
+	$ luet create-repo --packages my/packages/path --tree my/tree/path --output my/packages/path ...
+
+Provide name and description of the repository:
+
+	$ luet create-repo --name "foo" --description "bar" ...
+
+Change compression method:
+	
+	$ luet create-repo --tree-compression gzip --meta-compression gzip
+
+Create a repository from the metadata description defined in the luet.yaml config file:
+
+	$ luet create-repo --repo repository1
+`,
 	PreRun: func(cmd *cobra.Command, args []string) {
 		viper.BindPFlag("packages", cmd.Flags().Lookup("packages"))
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("output", cmd.Flags().Lookup("output"))
+		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
 		viper.BindPFlag("name", cmd.Flags().Lookup("name"))
 		viper.BindPFlag("descr", cmd.Flags().Lookup("descr"))
 		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
@@ -45,12 +67,16 @@ var createrepoCmd = &cobra.Command{
 		viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename"))
 		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
 		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
+
+		viper.BindPFlag("force-push", cmd.Flags().Lookup("force-push"))
+		viper.BindPFlag("push-images", cmd.Flags().Lookup("push-images"))
+
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var err error
 		var repo installer.Repository
 
-		tree := viper.GetString("tree")
+		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("output")
 		packages := viper.GetString("packages")
 		name := viper.GetString("name")
@@ -63,9 +89,13 @@ var createrepoCmd = &cobra.Command{
 		metatype := viper.GetString("meta-compression")
 		metaName := viper.GetString("meta-filename")
 		source_repo := viper.GetString("repo")
+		backendType := viper.GetString("backend")
 
 		treeFile := installer.NewDefaultTreeRepositoryFile()
 		metaFile := installer.NewDefaultMetaRepositoryFile()
+		compilerBackend := backend.NewBackend(backendType)
+		force := viper.GetBool("force-push")
+		imagePush := viper.GetBool("push-images")
 
 		if source_repo != "" {
 			// Search for system repository
@@ -74,8 +104,8 @@ var createrepoCmd = &cobra.Command{
 				Fatal("Error: " + err.Error())
 			}
 
-			if tree == "" {
-				tree = lrepo.TreePath
+			if len(treePaths) <= 0 {
+				treePaths = []string{lrepo.TreePath}
 			}
 
 			if t == "" {
@@ -87,12 +117,12 @@ var createrepoCmd = &cobra.Command{
 				lrepo.Urls,
 				lrepo.Priority,
 				packages,
-				tree,
-				pkg.NewInMemoryDatabase(false))
+				treePaths,
+				pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force)
 
 		} else {
 			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
-				tree, pkg.NewInMemoryDatabase(false))
+				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force)
 		}
 
 		if err != nil {
@@ -118,7 +148,7 @@ var createrepoCmd = &cobra.Command{
 		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
-		err = repo.Write(dst, reset)
+		err = repo.Write(dst, reset, true)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
@@ -130,19 +160,23 @@ func init() {
 	if err != nil {
 		Fatal(err)
 	}
-	createrepoCmd.Flags().String("packages", path, "Packages folder (output from build)")
-	createrepoCmd.Flags().String("tree", path, "Source luet tree")
-	createrepoCmd.Flags().String("output", path, "Destination folder")
+	createrepoCmd.Flags().String("packages", filepath.Join(path, "build"), "Packages folder (output from build)")
+	createrepoCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the source trees to use.")
+	createrepoCmd.Flags().String("output", filepath.Join(path, "build"), "Destination for generated archives. With 'docker' repository type, it should be an image reference (e.g 'foo/bar')")
 	createrepoCmd.Flags().String("name", "luet", "Repository name")
 	createrepoCmd.Flags().String("descr", "luet", "Repository description")
 	createrepoCmd.Flags().StringSlice("urls", []string{}, "Repository URLs")
-	createrepoCmd.Flags().String("type", "disk", "Repository type (disk)")
+	createrepoCmd.Flags().String("type", "disk", "Repository type (disk, http, docker)")
 	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
 	createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
+	createrepoCmd.Flags().String("backend", "docker", "backend used (docker,img)")
 
-	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip")
+	createrepoCmd.Flags().Bool("force-push", false, "Force overwrite of docker images if already present online")
+	createrepoCmd.Flags().Bool("push-images", false, "Enable/Disable docker image push for docker repositories")
+
+	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
-	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
 
 	RootCmd.AddCommand(createrepoCmd)

cmd/database.go

@@ -0,0 +1,41 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/database"
+
+	"github.com/spf13/cobra"
+)
+
+var databaseGroupCmd = &cobra.Command{
+	Use:   "database [command] [OPTIONS]",
+	Short: "Manage system database (dangerous commands ahead!)",
+	Long: `Allows to manipulate Luet internal database of installed packages. Use with caution!
+
+Removing packages by hand from the database can result in a broken system, and thus it's not reccomended.
+`,
+}
+
+func init() {
+	RootCmd.AddCommand(databaseGroupCmd)
+
+	databaseGroupCmd.AddCommand(
+		NewDatabaseCreateCommand(),
+		NewDatabaseRemoveCommand(),
+	)
+}

cmd/database/create.go

@@ -0,0 +1,81 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"io/ioutil"
+
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseCreateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "create <artifact_metadata1.yaml> <artifact_metadata1.yaml>",
+		Short: "Insert a package in the system DB",
+		Long: `Inserts a package in the system database:
+
+		$ luet database create foo.yaml
+
+"luet database create" injects a package in the system database without actually installing it, use it with caution.
+
+This commands takes multiple yaml input file representing package artifacts, that are usually generated while building packages.
+
+The yaml must contain the package definition, and the file list at least.
+
+For reference, inspect a "metadata.yaml" file generated while running "luet build"`,
+		Args: cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			systemDB := LuetCfg.GetSystemDB()
+
+			for _, a := range args {
+				dat, err := ioutil.ReadFile(a)
+				if err != nil {
+					Fatal("Failed reading ", a, ": ", err.Error())
+				}
+				art, err := compiler.NewPackageArtifactFromYaml(dat)
+				if err != nil {
+					Fatal("Failed reading yaml ", a, ": ", err.Error())
+				}
+
+				files := art.GetFiles()
+
+				if _, err := systemDB.CreatePackage(art.GetCompileSpec().GetPackage()); err != nil {
+					Fatal("Failed to create ", a, ": ", err.Error())
+				}
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.GetCompileSpec().GetPackage().GetFingerPrint(), Files: files}); err != nil {
+					Fatal("Failed setting package files for ", a, ": ", err.Error())
+				}
+
+				Info(art.GetCompileSpec().GetPackage().HumanReadableString(), " created")
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/database/remove.go

@@ -0,0 +1,65 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	. "github.com/mudler/luet/pkg/logger"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseRemoveCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "remove [package1] [package2] ...",
+		Short: "Remove a package from the system DB (forcefully - you normally don't want to do that)",
+		Long: `Removes a package in the system database without actually uninstalling it:
+
+		$ luet database remove foo/bar
+
+This commands takes multiple packages as arguments and prunes their entries from the system database.
+`,
+		Args: cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			systemDB := LuetCfg.GetSystemDB()
+
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					Fatal("Invalid package string ", a, ": ", err.Error())
+				}
+
+				if err := systemDB.RemovePackage(pack); err != nil {
+					Fatal("Failed removing ", a, ": ", err.Error())
+				}
+
+				if err := systemDB.RemovePackageFiles(pack); err != nil {
+					Fatal("Failed removing files for ", a, ": ", err.Error())
+				}
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/helpers/cli.go

@@ -14,12 +14,13 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package cmd_helpers
 
 import (
 	"errors"
 	"fmt"
 	"regexp"
+	"strings"
 
 	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -41,7 +42,41 @@ func CreateRegexArray(rgx []string) ([]*regexp.Regexp, error) {
 	return ans, nil
 }
 
+func packageData(p string) (string, string) {
+	cat := ""
+	name := ""
+	if strings.Contains(p, "/") {
+		packagedata := strings.Split(p, "/")
+		cat = packagedata[0]
+		name = packagedata[1]
+	} else {
+		name = p
+	}
+	return cat, name
+}
 func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
+
+	if !strings.HasPrefix(p, "=") {
+		ver := ">=0"
+		cat := ""
+		name := ""
+
+		if strings.Contains(p, "@") {
+			packageinfo := strings.Split(p, "@")
+			ver = packageinfo[1]
+			cat, name = packageData(packageinfo[0])
+		} else {
+			cat, name = packageData(p)
+		}
+
+		return &pkg.DefaultPackage{
+			Name:     name,
+			Category: cat,
+			Version:  ver,
+			Uri:      make([]string, 0),
+		}, nil
+	}
+
 	gp, err := _gentoo.ParsePackageStr(p)
 	if err != nil {
 		return nil, err

cmd/helpers/cli_suite_test.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package gentoo_test
+package cmd_helpers_test
 
 import (
 	"testing"
@@ -25,8 +25,8 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func TestGentooBuilder(t *testing.T) {
+func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
 	LoadConfig(config.LuetCfg)
-	RunSpecs(t, "Gentoo Suite")
+	RunSpecs(t, "CLI helpers test Suite")
 }

cmd/helpers/cli_test.go

@@ -0,0 +1,70 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_helpers_test
+
+import (
+	. "github.com/mudler/luet/cmd/helpers"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("CLI Helpers", func() {
+	Context("Can parse package strings correctly", func() {
+		It("accept single package names", func() {
+			pack, err := ParsePackageStr("foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal(""))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept unversioned packages with category", func() {
+			pack, err := ParsePackageStr("cat/foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept versioned packages with category", func() {
+			pack, err := ParsePackageStr("cat/foo@1.1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("1.1"))
+		})
+		It("accept versioned ranges with category", func() {
+			pack, err := ParsePackageStr("cat/foo@>=1.1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=1.1"))
+		})
+		It("accept gentoo regex parsing without versions", func() {
+			pack, err := ParsePackageStr("=cat/foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept gentoo regex parsing with versions", func() {
+			pack, err := ParsePackageStr("=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("1.2"))
+		})
+	})
+})

cmd/install.go

@@ -16,12 +16,12 @@ package cmd
 
 import (
 	"os"
-	"path/filepath"
 
 	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -31,6 +31,23 @@ import (
 var installCmd = &cobra.Command{
 	Use:   "install <pkg1> <pkg2> ...",
 	Short: "Install a package",
+	Long: `Installs one or more packages without asking questions:
+
+	$ luet install -y utils/busybox utils/yq ...
+	
+To install only deps of a package:
+	
+	$ luet install --onlydeps utils/busybox ...
+	
+To not install deps of a package:
+	
+	$ luet install --nodeps utils/busybox ...
+
+To force install a package:
+	
+	$ luet install --force utils/busybox ...
+`,
+	Aliases: []string{"i"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
 		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
@@ -41,11 +58,10 @@ var installCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
-	Long: `Install packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
 		var toInstall pkg.Packages
-		var systemDB pkg.PackageDatabase
 
 		for _, a := range args {
 			pack, err := helpers.ParsePackageStr(a)
@@ -72,13 +88,25 @@ var installCmd = &cobra.Command{
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps := LuetCfg.Viper.GetBool("nodeps")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
+
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
 		LuetCfg.GetSolverOptions().MaxAttempts = attempts
 
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
 			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
 			SolverOptions:               *LuetCfg.GetSolverOptions(),
@@ -86,16 +114,11 @@ var installCmd = &cobra.Command{
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
 			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
 		})
 		inst.Repositories(repos)
 
-		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-		} else {
-			systemDB = pkg.NewInMemoryDatabase(true)
-		}
-		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
 		err := inst.Install(toInstall, system)
 		if err != nil {
 			Fatal("Error: " + err.Error())
@@ -117,6 +140,8 @@ func init() {
 	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
 	installCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
 	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	installCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	installCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/pack.go

@@ -0,0 +1,98 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var packCmd = &cobra.Command{
+	Use:   "pack <package name>",
+	Short: "pack a custom package",
+	Long: `Pack creates a package from a directory, generating the metadata required from a tree to generate a repository.
+
+Pack can be used to manually replace what "luet build" does automatically by reading the packages build.yaml files.
+
+	$ mkdir -p output/etc/foo
+	$ echo "my config" > output/etc/foo
+	$ luet pack foo/bar@1.1 --source output
+
+Afterwards, you can use the content generated and associate it with a tree and a corresponding definition.yaml file with "luet create-repo".
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
+		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+		viper.BindPFlag("source", cmd.Flags().Lookup("source"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		sourcePath := viper.GetString("source")
+
+		dst := viper.GetString("destination")
+		compressionType := viper.GetString("compression")
+		concurrency := LuetCfg.GetGeneral().Concurrency
+
+		if len(args) != 1 {
+			Fatal("You must specify a package name")
+		}
+
+		packageName := args[0]
+
+		p, err := helpers.ParsePackageStr(packageName)
+		if err != nil {
+			Fatal("Invalid package string ", packageName, ": ", err.Error())
+		}
+
+		spec := &compiler.LuetCompilationSpec{Package: p}
+		artifact := compiler.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
+		artifact.SetCompressionType(compiler.CompressionImplementation(compressionType))
+		err = artifact.Compress(sourcePath, concurrency)
+		if err != nil {
+			Fatal("failed compressing ", packageName, ": ", err.Error())
+		}
+		artifact.SetCompileSpec(spec)
+		filelist, err := artifact.FileList()
+		if err != nil {
+			Fatal("failed generating file list for ", packageName, ": ", err.Error())
+		}
+		artifact.SetFiles(filelist)
+		artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
+		err = artifact.WriteYaml(dst)
+		if err != nil {
+			Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	packCmd.Flags().String("source", path, "Source folder")
+	packCmd.Flags().String("destination", path, "Destination folder")
+	packCmd.Flags().String("compression", "gzip", "Compression alg: none, gzip")
+
+	RootCmd.AddCommand(packCmd)
+}

cmd/reclaim.go

@@ -16,13 +16,11 @@ package cmd
 
 import (
 	"os"
-	"path/filepath"
 
 	installer "github.com/mudler/luet/pkg/installer"
 
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
-	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
 )
@@ -35,10 +33,13 @@ var reclaimCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
-	Long: `Add packages to the systemdb if files belonging to packages 
-in available repositories exists in the target root.`,
+	Long: `Reclaim tries to find association between packages in the online repositories and the system one.
+
+	$ luet reclaim
+
+It scans the target file system, and if finds a match with a package available in the repositories, it marks as installed in the system database.
+`,
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
 
 		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
 		repos := installer.Repositories{}
@@ -61,13 +62,7 @@ in available repositories exists in the target root.`,
 		})
 		inst.Repositories(repos)
 
-		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-		} else {
-			systemDB = pkg.NewInMemoryDatabase(true)
-		}
-		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
 		err := inst.Reclaim(system)
 		if err != nil {
 			Fatal("Error: " + err.Error())

cmd/replace.go

@@ -0,0 +1,148 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+var replaceCmd = &cobra.Command{
+	Use:     "replace <pkg1> <pkg2> --for <pkg3> --for <pkg4> ...",
+	Short:   "replace a set of packages",
+	Aliases: []string{"r"},
+	Long: `Replaces one or a group of packages without asking questions:
+
+	$ luet replace -y system/busybox ... --for shells/bash --for system/coreutils ...
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("for", cmd.Flags().Lookup("for"))
+
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var toUninstall pkg.Packages
+		var toAdd pkg.Packages
+
+		f := LuetCfg.Viper.GetStringSlice("for")
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps := LuetCfg.Viper.GetBool("nodeps")
+		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
+
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toUninstall = append(toUninstall, pack)
+		}
+
+		for _, a := range f {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toAdd = append(toAdd, pack)
+		}
+
+		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+		})
+		inst.Repositories(repos)
+
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Swap(toUninstall, toAdd, system)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	replaceCmd.Flags().String("system-dbpath", path, "System db path")
+	replaceCmd.Flags().String("system-target", path, "System rootpath")
+	replaceCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	replaceCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	replaceCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	replaceCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	replaceCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	replaceCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	replaceCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	replaceCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	replaceCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	replaceCmd.Flags().StringSlice("for", []string{}, "Packages that has to be installed in place of others")
+
+	RootCmd.AddCommand(replaceCmd)
+}

cmd/repo/update.go

@@ -35,6 +35,7 @@ $> luet repo update
 # Update only repo1 and repo2
 $> luet repo update repo1 repo2
 `,
+		Aliases: []string{"up"},
 		PreRun: func(cmd *cobra.Command, args []string) {
 		},
 		Run: func(cmd *cobra.Command, args []string) {

cmd/root.go

@@ -24,11 +24,13 @@ import (
 	"strings"
 
 	"github.com/marcsauter/single"
+	bus "github.com/mudler/luet/pkg/bus"
+
+	extensions "github.com/mudler/cobra-extensions"
 	config "github.com/mudler/luet/pkg/config"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	repo "github.com/mudler/luet/pkg/repository"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -38,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.7.6"
+	LuetCLIVersion = "0.11.2"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -52,9 +54,31 @@ var (
 
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
-	Use:     "luet",
-	Short:   "Package manager for the XXth century!",
-	Long:    `Package manager which uses containers to build packages`,
+	Use:   "luet",
+	Short: "Container based package manager",
+	Long: `Luet is a single-binary package manager based on containers to build packages.
+	
+To install a package:
+
+	$ luet install package
+
+To search for a package in the repositories:
+
+$ luet search package
+
+To list all packages installed in the system:
+
+	$ luet search --installed .
+
+To show hidden packages:
+
+	$ luet search --hidden package
+
+To build a package, from a tree definition:
+
+	$ luet build --tree tree/path package
+	
+`,
 	Version: fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime),
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		err := LoadConfig(config.LuetCfg)
@@ -68,6 +92,18 @@ var RootCmd = &cobra.Command{
 		if err != nil {
 			Fatal("failed on init tmp basedir:", err.Error())
 		}
+
+		viper.BindPFlag("plugin", cmd.Flags().Lookup("plugin"))
+
+		plugin := viper.GetStringSlice("plugin")
+
+		bus.Manager.Load(plugin...).Register()
+		if len(bus.Manager.Plugins) != 0 {
+			Info(":lollipop:Enabled plugins:")
+			for _, p := range bus.Manager.Plugins {
+				Info("\t:arrow_right:", p.Name)
+			}
+		}
 	},
 	PersistentPostRun: func(cmd *cobra.Command, args []string) {
 		// Cleanup all tmp directories used by luet
@@ -88,11 +124,16 @@ func LoadConfig(c *config.LuetConfig) error {
 		return err
 	}
 
+	noSpinner := c.Viper.GetBool("no_spinner")
+
+	InitAurora()
+	if !noSpinner {
+		NewSpinner()
+	}
+
 	Debug("Using config file:", c.Viper.ConfigFileUsed())
 
-	NewSpinner()
-
-	if c.GetLogging().Path != "" {
+	if c.GetLogging().EnableLogFile && c.GetLogging().Path != "" {
 		// Init zap logger
 		err = ZapLogger()
 		if err != nil {
@@ -132,15 +173,6 @@ func Execute() {
 	}
 
 	if err := RootCmd.Execute(); err != nil {
-		if len(os.Args) > 0 {
-			for _, c := range RootCmd.Commands() {
-				if c.Name() == os.Args[1] {
-					os.Exit(-1) // Something failed
-				}
-			}
-			// Try to load a bin from path.
-			helpers.Exec("luet-"+os.Args[1], os.Args[1:], os.Environ())
-		}
 		fmt.Println(err)
 		os.Exit(-1)
 	}
@@ -152,44 +184,84 @@ func init() {
 	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
 	pflags.BoolP("debug", "d", false, "verbose output")
 	pflags.Bool("fatal", false, "Enables Warnings to exit")
+	pflags.Bool("enable-logfile", false, "Enable log to file")
+	pflags.Bool("no-spinner", false, "Disable spinner.")
+	pflags.Bool("color", config.LuetCfg.GetLogging().Color, "Enable/Disable color.")
+	pflags.Bool("emoji", config.LuetCfg.GetLogging().EnableEmoji, "Enable/Disable emoji.")
+	pflags.Bool("skip-config-protect", config.LuetCfg.ConfigProtectSkip,
+		"Disable config protect analysis.")
+	pflags.StringP("logfile", "l", config.LuetCfg.GetLogging().Path,
+		"Logfile path. Empty value disable log to file.")
+	pflags.StringSlice("plugin", []string{}, "A list of runtime plugins to load")
 
-	sameOwner := false
-	u, err := user.Current()
-	// os/user doesn't work in from scratch environments
+	// os/user doesn't work in from scratch environments.
+	// Check if i can retrieve user informations.
+	_, err := user.Current()
 	if err != nil {
 		Warning("failed to retrieve user identity:", err.Error())
-		sameOwner = true
 	}
-	if u != nil && u.Uid == "0" {
-		sameOwner = true
-	}
-	pflags.Bool("same-owner", sameOwner, "Maintain same owner on uncompress.")
+	pflags.Bool("same-owner", config.LuetCfg.GetGeneral().SameOwner, "Maintain same owner on uncompress.")
 	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
 
-	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
-	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("debug"))
+	config.LuetCfg.Viper.BindPFlag("logging.color", pflags.Lookup("color"))
+	config.LuetCfg.Viper.BindPFlag("logging.enable_emoji", pflags.Lookup("emoji"))
+	config.LuetCfg.Viper.BindPFlag("logging.enable_logfile", pflags.Lookup("enable-logfile"))
+	config.LuetCfg.Viper.BindPFlag("logging.path", pflags.Lookup("logfile"))
+
 	config.LuetCfg.Viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
+	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("debug"))
 	config.LuetCfg.Viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
+	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
+	config.LuetCfg.Viper.BindPFlag("plugin", pflags.Lookup("plugin"))
+
+	// Currently I maintain this only from cli.
+	config.LuetCfg.Viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))
+	config.LuetCfg.Viper.BindPFlag("config_protect_skip", pflags.Lookup("skip-config-protect"))
+
+	// Extensions must be binary with the "luet-" prefix to be able to be shown in the help.
+	// we also accept extensions in the relative path where luet is being started, "extensions/"
+	exts := extensions.Discover("luet", "extensions")
+	for _, ex := range exts {
+		cobraCmd := ex.CobraCommand()
+		RootCmd.AddCommand(cobraCmd)
+	}
+
 }
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
+	// Luet support these priorities on read configuration file:
+	// - command line option (if available)
+	// - $PWD/.luet.yaml
+	// - $HOME/.luet.yaml
+	// - /etc/luet/luet.yaml
+	//
+	// Note: currently a single viper instance support only one config name.
 
-	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
-	if err != nil {
-		Error(err)
-		os.Exit(1)
-	}
 	viper.SetEnvPrefix(LuetEnvPrefix)
 	viper.SetConfigType("yaml")
-	viper.SetConfigName(".luet") // name of config file (without extension)
-	if cfgFile != "" {           // enable ability to specify config file via flag
+
+	if cfgFile != "" { // enable ability to specify config file via flag
 		viper.SetConfigFile(cfgFile)
 	} else {
-		viper.AddConfigPath(dir)
-		viper.AddConfigPath(".")
-		viper.AddConfigPath("$HOME")
-		viper.AddConfigPath("/etc/luet")
+		// Retrieve pwd directory
+		pwdDir, err := os.Getwd()
+		if err != nil {
+			Error(err)
+			os.Exit(1)
+		}
+		homeDir := helpers.GetHomeDir()
+
+		if helpers.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && helpers.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
+			viper.AddConfigPath(".")
+			if homeDir != "" {
+				viper.AddConfigPath(homeDir)
+			}
+			viper.SetConfigName(".luet")
+		} else {
+			viper.SetConfigName("luet")
+			viper.AddConfigPath("/etc/luet")
+		}
 	}
 
 	viper.AutomaticEnv() // read in environment variables that match

cmd/search.go

@@ -17,9 +17,11 @@ package cmd
 import (
 	"fmt"
 	"os"
-	"path/filepath"
+	"strings"
 
 	"github.com/ghodss/yaml"
+	"github.com/jedib0t/go-pretty/table"
+	"github.com/jedib0t/go-pretty/v6/list"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -32,16 +34,73 @@ type PackageResult struct {
 	Category   string `json:"category"`
 	Version    string `json:"version"`
 	Repository string `json:"repository"`
+	Target     string `json:"target"`
+	Hidden     bool   `json:"hidden"`
 }
 
 type Results struct {
 	Packages []PackageResult `json:"packages"`
 }
 
+func (r PackageResult) String() string {
+	return fmt.Sprintf("%s/%s-%s required for %s", r.Category, r.Name, r.Version, r.Target)
+}
+
+var rows table.Row = table.Row{"Package", "Category", "Name", "Version", "Repository", "Description", "License", "URI"}
+
+func packageToRow(repo string, p pkg.Package) table.Row {
+	return table.Row{p.HumanReadableString(), p.GetCategory(), p.GetName(), p.GetVersion(), repo, p.GetDescription(), p.GetLicense(), strings.Join(p.GetURI(), "\n")}
+}
+
+func packageToList(l list.Writer, repo string, p pkg.Package) {
+	l.AppendItem(p.HumanReadableString())
+	l.Indent()
+	l.AppendItem(fmt.Sprintf("Category: %s", p.GetCategory()))
+	l.AppendItem(fmt.Sprintf("Name: %s", p.GetName()))
+	l.AppendItem(fmt.Sprintf("Version: %s", p.GetVersion()))
+	l.AppendItem(fmt.Sprintf("Description: %s", p.GetDescription()))
+	l.AppendItem(fmt.Sprintf("Repository: %s ", repo))
+	l.AppendItem(fmt.Sprintf("Uri: %s ", strings.Join(p.GetURI(), "\n")))
+	l.UnIndent()
+}
+
 var searchCmd = &cobra.Command{
 	Use:   "search <term>",
 	Short: "Search packages",
-	Long:  `Search for installed and available packages`,
+	Long: `Search for installed and available packages
+	
+To search a package in the repositories:
+
+	$ luet search <regex>
+
+To search a package and display results in a table (wide screens):
+
+	$ luet search --table <regex>
+
+To look into the installed packages:
+
+	$ luet search --installed <regex>
+
+Note: the regex argument is optional, if omitted implies "all"
+
+To search a package by label:
+
+	$ luet search --by-label <label>
+
+or by regex against the label:
+
+	$ luet search --by-label-regex <label>
+
+It can also show a package revdeps by:
+
+	$ luet search --revdeps <regex>
+
+Search can also return results in the terminal in different ways: as terminal output, as json or as yaml.
+
+	$ luet search --json <regex> # JSON output
+	$ luet search --yaml <regex> # YAML output
+`,
+	Aliases: []string{"s"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
 		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
@@ -52,11 +111,14 @@ var searchCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
 		var results Results
-		if len(args) != 1 {
+		if len(args) > 1 {
 			Fatal("Wrong number of arguments (expected 1)")
+		} else if len(args) == 0 {
+			args = []string{"."}
 		}
+		hidden, _ := cmd.Flags().GetBool("hidden")
+
 		installed := LuetCfg.Viper.GetBool("installed")
 		stype := LuetCfg.Viper.GetString("solver.type")
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
@@ -65,6 +127,7 @@ var searchCmd = &cobra.Command{
 		searchWithLabel, _ := cmd.Flags().GetBool("by-label")
 		searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex")
 		revdeps, _ := cmd.Flags().GetBool("revdeps")
+		tableMode, _ := cmd.Flags().GetBool("table")
 
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
@@ -76,6 +139,9 @@ var searchCmd = &cobra.Command{
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
 		LuetCfg.GetSolverOptions().MaxAttempts = attempts
 
+		l := list.NewWriter()
+		t := table.NewWriter()
+		t.AppendHeader(rows)
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
 		if !installed {
@@ -113,37 +179,39 @@ var searchCmd = &cobra.Command{
 			}
 			for _, m := range matches {
 				if !revdeps {
-					Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
-					results.Packages = append(results.Packages,
-						PackageResult{
-							Name:       m.Package.GetName(),
-							Version:    m.Package.GetVersion(),
-							Category:   m.Package.GetCategory(),
-							Repository: m.Repo.GetName(),
-						})
-				} else {
-					visited := make(map[string]interface{})
-					for _, revdep := range m.Package.ExpandedRevdeps(m.Repo.GetTree().GetDatabase(), visited) {
-						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
+					if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
+						t.AppendRow(packageToRow(m.Repo.GetName(), m.Package))
+						packageToList(l, m.Repo.GetName(), m.Package)
 						results.Packages = append(results.Packages,
 							PackageResult{
-								Name:       revdep.GetName(),
-								Version:    revdep.GetVersion(),
-								Category:   revdep.GetCategory(),
+								Name:       m.Package.GetName(),
+								Version:    m.Package.GetVersion(),
+								Category:   m.Package.GetCategory(),
 								Repository: m.Repo.GetName(),
+								Hidden:     m.Package.IsHidden(),
 							})
 					}
+				} else {
+					packs, _ := m.Repo.GetTree().GetDatabase().GetRevdeps(m.Package)
+					for _, revdep := range packs {
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							t.AppendRow(packageToRow(m.Repo.GetName(), revdep))
+							packageToList(l, m.Repo.GetName(), revdep)
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: m.Repo.GetName(),
+									Hidden:     revdep.IsHidden(),
+								})
+						}
+					}
 				}
 			}
 		} else {
 
-			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-				systemDB = pkg.NewBoltDatabase(
-					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-			} else {
-				systemDB = pkg.NewInMemoryDatabase(true)
-			}
-			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+			system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
 
 			var err error
 			iMatches := pkg.Packages{}
@@ -161,31 +229,48 @@ var searchCmd = &cobra.Command{
 
 			for _, pack := range iMatches {
 				if !revdeps {
-					Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
-					results.Packages = append(results.Packages,
-						PackageResult{
-							Name:       pack.GetName(),
-							Version:    pack.GetVersion(),
-							Category:   pack.GetCategory(),
-							Repository: "system",
-						})
-				} else {
-					visited := make(map[string]interface{})
-
-					for _, revdep := range pack.ExpandedRevdeps(system.Database, visited) {
-						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+					if !pack.IsHidden() || pack.IsHidden() && hidden {
+						t.AppendRow(packageToRow("system", pack))
+						packageToList(l, "system", pack)
 						results.Packages = append(results.Packages,
 							PackageResult{
-								Name:       revdep.GetName(),
-								Version:    revdep.GetVersion(),
-								Category:   revdep.GetCategory(),
+								Name:       pack.GetName(),
+								Version:    pack.GetVersion(),
+								Category:   pack.GetCategory(),
 								Repository: "system",
+								Hidden:     pack.IsHidden(),
 							})
 					}
+				} else {
+					packs, _ := system.Database.GetRevdeps(pack)
+					for _, revdep := range packs {
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							t.AppendRow(packageToRow("system", pack))
+							packageToList(l, "system", pack)
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: "system",
+									Hidden:     revdep.IsHidden(),
+								})
+						}
+					}
 				}
 			}
 		}
 
+		t.AppendFooter(rows)
+		t.SetStyle(table.StyleColoredBright)
+
+		l.SetStyle(list.StyleConnectedRounded)
+		if tableMode {
+			Info(t.Render())
+		} else {
+			Info(l.Render())
+		}
+
 		y, err := yaml.Marshal(results)
 		if err != nil {
 			fmt.Printf("err: %v\n", err)
@@ -222,5 +307,8 @@ func init() {
 	searchCmd.Flags().Bool("by-label", false, "Search packages through label")
 	searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex")
 	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
+	searchCmd.Flags().Bool("table", false, "show output in a table (wider screens)")
+
 	RootCmd.AddCommand(searchCmd)
 }

cmd/tree.go

@@ -34,5 +34,6 @@ func init() {
 		NewTreePkglistCommand(),
 		NewTreeValidateCommand(),
 		NewTreeBumpCommand(),
+		NewTreeImageCommand(),
 	)
 }

cmd/tree/bump.go

@@ -18,11 +18,11 @@ package cmd_tree
 
 import (
 	"fmt"
-	//"os"
-	//"sort"
 
 	. "github.com/mudler/luet/pkg/logger"
+	spectooling "github.com/mudler/luet/pkg/spectooling"
 	tree "github.com/mudler/luet/pkg/tree"
+	version "github.com/mudler/luet/pkg/versioner"
 
 	"github.com/spf13/cobra"
 )
@@ -42,19 +42,28 @@ func NewTreeBumpCommand() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			spec, _ := cmd.Flags().GetString("definition-file")
 			toStdout, _ := cmd.Flags().GetBool("to-stdout")
+			pkgVersion, _ := cmd.Flags().GetString("pkg-version")
 			pack, err := tree.ReadDefinitionFile(spec)
 			if err != nil {
 				Fatal(err.Error())
 			}
 
-			// Retrieve version build section with Gentoo parser
-			err = pack.BumpBuildVersion()
-			if err != nil {
-				Fatal("Error on increment build version: " + err.Error())
+			if pkgVersion != "" {
+				validator := &version.WrappedVersioner{}
+				err := validator.Validate(pkgVersion)
+				if err != nil {
+					Fatal("Invalid version string: " + err.Error())
+				}
+				pack.SetVersion(pkgVersion)
+			} else {
+				// Retrieve version build section with Gentoo parser
+				err = pack.BumpBuildVersion()
+				if err != nil {
+					Fatal("Error on increment build version: " + err.Error())
+				}
 			}
-
 			if toStdout {
-				data, err := pack.Yaml()
+				data, err := spectooling.NewDefaultPackageSanitized(&pack).Yaml()
 				if err != nil {
 					Fatal("Error on yaml conversion: " + err.Error())
 				}
@@ -71,6 +80,7 @@ func NewTreeBumpCommand() *cobra.Command {
 		},
 	}
 
+	ans.Flags().StringP("pkg-version", "p", "", "Set a specific package version")
 	ans.Flags().StringP("definition-file", "f", "", "Path of the definition to bump.")
 	ans.Flags().BoolP("to-stdout", "o", false, "Bump package to output.")
 

cmd/tree/images.go

@@ -0,0 +1,140 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"fmt"
+	"os"
+
+	//. "github.com/mudler/luet/pkg/config"
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func NewTreeImageCommand() *cobra.Command {
+
+	var ans = &cobra.Command{
+		Use:   "images [OPTIONS]",
+		Short: "List of the images of a package",
+		PreRun: func(cmd *cobra.Command, args []string) {
+			t, _ := cmd.Flags().GetStringArray("tree")
+			if len(t) == 0 {
+				Fatal("Mandatory tree param missing.")
+			}
+
+			if len(args) != 1 {
+				Fatal("Expects one package as parameter")
+			}
+			viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var results TreeResults
+
+			treePath, _ := cmd.Flags().GetStringArray("tree")
+			imageRepository := viper.GetString("image-repository")
+
+			out, _ := cmd.Flags().GetString("output")
+			if out != "terminal" {
+				LuetCfg.GetLogging().SetLogLevel("error")
+			}
+
+			reciper := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			for _, t := range treePath {
+				err := reciper.Load(t)
+				if err != nil {
+					Fatal("Error on load tree ", err)
+				}
+			}
+			compilerBackend := backend.NewSimpleDockerBackend()
+
+			opts := compiler.NewDefaultCompilerOptions()
+			opts.SolverOptions = *LuetCfg.GetSolverOptions()
+			opts.ImageRepository = imageRepository
+
+			solverOpts := solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
+			luetCompiler := compiler.NewLuetCompiler(compilerBackend, reciper.GetDatabase(), opts, solverOpts)
+
+			a := args[0]
+
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+
+			spec, err := luetCompiler.FromPackage(pack)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+			asserts, err := luetCompiler.ComputeDepTree(spec)
+
+			for _, assertion := range asserts { //highly dependent on the order
+
+				//buildImageHash := imageRepository + ":" + assertion.Hash.BuildHash
+				currentPackageImageHash := imageRepository + ":" + assertion.Hash.PackageHash
+
+				results.Packages = append(results.Packages, TreePackageResult{
+					Name:     assertion.Package.GetName(),
+					Version:  assertion.Package.GetVersion(),
+					Category: assertion.Package.GetCategory(),
+					Image:    currentPackageImageHash,
+				})
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			default:
+				for _, p := range results.Packages {
+					fmt.Println(fmt.Sprintf("%s/%s-%s: %s", p.Category, p.Name, p.Version, p.Image))
+				}
+			}
+		},
+	}
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
+	ans.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+
+	return ans
+}

cmd/tree/pkglist.go

@@ -18,14 +18,16 @@ package cmd_tree
 
 import (
 	"fmt"
+	"os"
 	"sort"
 
 	//. "github.com/mudler/luet/pkg/config"
 	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/spf13/cobra"
@@ -36,6 +38,7 @@ type TreePackageResult struct {
 	Category string `json:"category"`
 	Version  string `json:"version"`
 	Path     string `json:"path"`
+	Image    string `json:"image"`
 }
 
 type TreeResults struct {
@@ -74,15 +77,24 @@ func NewTreePkglistCommand() *cobra.Command {
 			if len(t) == 0 {
 				Fatal("Mandatory tree param missing.")
 			}
+
+			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
+			if revdeps && deps {
+				Fatal("Both revdeps and deps option used. Choice only one.")
+			}
+
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			var results TreeResults
+			var depSolver solver.PackageSolver
 
 			treePath, _ := cmd.Flags().GetStringArray("tree")
 			verbose, _ := cmd.Flags().GetBool("verbose")
 			buildtime, _ := cmd.Flags().GetBool("buildtime")
 			full, _ := cmd.Flags().GetBool("full")
 			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
 
 			out, _ := cmd.Flags().GetString("output")
 			if out != "terminal" {
@@ -95,6 +107,7 @@ func NewTreePkglistCommand() *cobra.Command {
 			} else {
 				reciper = tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
 			}
+
 			for _, t := range treePath {
 				err := reciper.Load(t)
 				if err != nil {
@@ -102,6 +115,15 @@ func NewTreePkglistCommand() *cobra.Command {
 				}
 			}
 
+			if deps {
+				emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+
+				depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
+					reciper.GetDatabase(),
+					emptyInstallationDb)
+
+			}
+
 			regExcludes, err := helpers.CreateRegexArray(excludes)
 			if err != nil {
 				Fatal(err.Error())
@@ -146,17 +168,9 @@ func NewTreePkglistCommand() *cobra.Command {
 				}
 
 				if addPkg {
-					if !revdeps {
-						plist = append(plist, pkgstr)
-						results.Packages = append(results.Packages, TreePackageResult{
-							Name:     p.GetName(),
-							Version:  p.GetVersion(),
-							Category: p.GetCategory(),
-							Path:     p.GetPath(),
-						})
-					} else {
-						visited := make(map[string]interface{})
-						for _, revdep := range p.ExpandedRevdeps(reciper.GetDatabase(), visited) {
+					if revdeps {
+						packs, _ := reciper.GetDatabase().GetRevdeps(p)
+						for _, revdep := range packs {
 							if full {
 								pkgstr = pkgDetail(revdep)
 							} else if verbose {
@@ -172,7 +186,58 @@ func NewTreePkglistCommand() *cobra.Command {
 								Path:     revdep.GetPath(),
 							})
 						}
+					} else if deps {
+
+						Spinner(32)
+						solution, err := depSolver.Install(pkg.Packages{p})
+						if err != nil {
+							Fatal(err.Error())
+						}
+						ass := solution.SearchByName(p.GetPackageName())
+						solution, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+						if err != nil {
+							Fatal(err.Error())
+						}
+						SpinnerStop()
+
+						for _, pa := range solution {
+
+							if pa.Value {
+								// Exclude itself
+								if pa.Package.GetName() == p.GetName() && pa.Package.GetCategory() == p.GetCategory() {
+									continue
+								}
+
+								if full {
+									pkgstr = pkgDetail(pa.Package)
+								} else if verbose {
+									pkgstr = pa.Package.HumanReadableString()
+								} else {
+									pkgstr = fmt.Sprintf("%s/%s", pa.Package.GetCategory(), pa.Package.GetName())
+								}
+								plist = append(plist, pkgstr)
+								results.Packages = append(results.Packages, TreePackageResult{
+									Name:     pa.Package.GetName(),
+									Version:  pa.Package.GetVersion(),
+									Category: pa.Package.GetCategory(),
+									Path:     pa.Package.GetPath(),
+								})
+							}
+
+						}
+
+					} else {
+
+						plist = append(plist, pkgstr)
+						results.Packages = append(results.Packages, TreePackageResult{
+							Name:     p.GetName(),
+							Version:  p.GetVersion(),
+							Category: p.GetCategory(),
+							Path:     p.GetPath(),
+						})
+
 					}
+
 				}
 			}
 
@@ -192,7 +257,9 @@ func NewTreePkglistCommand() *cobra.Command {
 				}
 				fmt.Println(string(j2))
 			default:
-				sort.Strings(plist)
+				if !deps {
+					sort.Strings(plist)
+				}
 				for _, p := range plist {
 					fmt.Println(p)
 				}
@@ -200,14 +267,18 @@ func NewTreePkglistCommand() *cobra.Command {
 
 		},
 	}
-
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
 	ans.Flags().BoolP("buildtime", "b", false, "Build time match")
 	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 	ans.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	ans.Flags().Bool("deps", false, "Search package dependencies")
 
 	ans.Flags().BoolP("verbose", "v", false, "Add package version")
 	ans.Flags().BoolP("full", "f", false, "Show package detail")
-	ans.Flags().StringArrayP("tree", "t", []string{}, "Path of the tree to use.")
+	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
 	ans.Flags().StringSliceVarP(&matches, "matches", "m", []string{},
 		"Include only matched packages from list. (Use string as regex).")
 	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},

cmd/tree/validate.go

@@ -25,8 +25,8 @@ import (
 	"strconv"
 	"sync"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
@@ -35,220 +35,421 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func validateWorker(i int,
-	wg *sync.WaitGroup,
-	c <-chan pkg.Package,
-	reciper tree.Builder,
-	withSolver bool,
-	regExcludes, regMatches []*regexp.Regexp,
-	excludes, matches []string,
-	errs chan error) {
+type ValidateOpts struct {
+	WithSolver    bool
+	OnlyRuntime   bool
+	OnlyBuildtime bool
+	RegExcludes   []*regexp.Regexp
+	RegMatches    []*regexp.Regexp
+	Excludes      []string
+	Matches       []string
 
-	defer wg.Done()
+	// Runtime validate stuff
+	RuntimeCacheDeps *pkg.InMemoryDatabase
+	RuntimeReciper   *tree.InstallerRecipe
+
+	// Buildtime validate stuff
+	BuildtimeCacheDeps *pkg.InMemoryDatabase
+	BuildtimeReciper   *tree.CompilerRecipe
+
+	Mutex      sync.Mutex
+	BrokenPkgs int
+	BrokenDeps int
+
+	Errors []error
+}
+
+func (o *ValidateOpts) IncrBrokenPkgs() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenPkgs++
+}
+
+func (o *ValidateOpts) IncrBrokenDeps() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenDeps++
+}
+
+func (o *ValidateOpts) AddError(err error) {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.Errors = append(o.Errors, err)
+}
+
+func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, reciper tree.Builder, cacheDeps *pkg.InMemoryDatabase) error {
+	var errstr string
+	var ans error
 
 	var depSolver solver.PackageSolver
-	brokenPkgs := 0
-	brokenDeps := 0
-	var errstr string
 
-	emptyInstallationDb := pkg.NewInMemoryDatabase(false)
-	if withSolver {
-		depSolver = solver.NewSolver(pkg.NewInMemoryDatabase(false),
+	if opts.WithSolver {
+		emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+		depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
 			reciper.GetDatabase(),
 			emptyInstallationDb)
 	}
 
-	for p := range c {
-		found, err := reciper.GetDatabase().FindPackages(
-			&pkg.DefaultPackage{
-				Name:     p.GetName(),
-				Category: p.GetCategory(),
-				Version:  ">=0",
-			},
-		)
+	found, err := reciper.GetDatabase().FindPackages(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  ">=0",
+		},
+	)
 
-		if err != nil || len(found) < 1 {
+	if err != nil || len(found) < 1 {
+		if err != nil {
+			errstr = err.Error()
+		} else {
+			errstr = "No packages"
+		}
+		Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			errstr,
+		))
+
+		opts.IncrBrokenDeps()
+
+		return errors.New(
+			fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+				checkType,
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				errstr,
+			))
+	}
+
+	// Ensure that we use the right package from right recipier for deps
+	pReciper, err := reciper.GetDatabase().FindPackage(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  p.GetVersion(),
+		},
+	)
+	if err != nil {
+		errstr = fmt.Sprintf("[%9s] %s/%s-%s: Error on retrieve package - %s.",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			err.Error(),
+		)
+		Error(errstr)
+
+		return errors.New(errstr)
+	}
+	p = pReciper
+
+	pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
+		p.GetVersion())
+
+	validpkg := true
+
+	if len(opts.Matches) > 0 {
+		matched := false
+		for _, rgx := range opts.RegMatches {
+			if rgx.MatchString(pkgstr) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return nil
+		}
+	}
+
+	if len(opts.Excludes) > 0 {
+		excluded := false
+		for _, rgx := range opts.RegExcludes {
+			if rgx.MatchString(pkgstr) {
+				excluded = true
+				break
+			}
+		}
+
+		if excluded {
+			return nil
+		}
+	}
+
+	Info(fmt.Sprintf("[%9s] Checking package ", checkType)+
+		fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
+		"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
+
+	all := p.GetRequires()
+	all = append(all, p.GetConflicts()...)
+	for idx, r := range all {
+
+		var deps pkg.Packages
+		var err error
+		if r.IsSelector() {
+			deps, err = reciper.GetDatabase().FindPackages(
+				&pkg.DefaultPackage{
+					Name:     r.GetName(),
+					Category: r.GetCategory(),
+					Version:  r.GetVersion(),
+				},
+			)
+		} else {
+			deps = append(deps, r)
+		}
+
+		if err != nil || len(deps) < 1 {
 			if err != nil {
 				errstr = err.Error()
 			} else {
 				errstr = "No packages"
 			}
-			Error(fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
+			Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+				checkType,
 				p.GetCategory(), p.GetName(), p.GetVersion(),
+				r.GetCategory(), r.GetName(), r.GetVersion(),
 				errstr,
 			))
 
-			errs <- errors.New(
-				fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
-					p.GetCategory(), p.GetName(), p.GetVersion(),
-					errstr,
-				))
+			opts.IncrBrokenDeps()
 
-			brokenPkgs++
-		}
-
-		pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
-			p.GetVersion())
-
-		validpkg := true
-
-		if len(matches) > 0 {
-			matched := false
-			for _, rgx := range regMatches {
-				if rgx.MatchString(pkgstr) {
-					matched = true
-					break
-				}
-			}
-
-			if !matched {
-				continue
-			}
-		}
-
-		if len(excludes) > 0 {
-			excluded := false
-			for _, rgx := range regExcludes {
-				if rgx.MatchString(pkgstr) {
-					excluded = true
-					break
-				}
-			}
-
-			if excluded {
-				continue
-			}
-		}
-		Info("Checking package "+fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()), "with", len(p.GetRequires()), "dependencies.")
-
-		all := p.GetRequires()
-		all = append(all, p.GetConflicts()...)
-		for _, r := range all {
-			var deps pkg.Packages
-			var err error
-			if r.IsSelector() {
-				deps, err = reciper.GetDatabase().FindPackages(
-					&pkg.DefaultPackage{
-						Name:     r.GetName(),
-						Category: r.GetCategory(),
-						Version:  r.GetVersion(),
-					},
-				)
-			} else {
-				deps = append(deps, r)
-			}
-
-			if err != nil || len(deps) < 1 {
-				if err != nil {
-					errstr = err.Error()
-				} else {
-					errstr = "No packages"
-				}
-				Error(fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
+			ans = errors.New(
+				fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+					checkType,
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					r.GetCategory(), r.GetName(), r.GetVersion(),
+					errstr))
+
+			validpkg = false
+
+		} else {
+
+			Debug(fmt.Sprintf("[%9s] Find packages for dep", checkType),
+				fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+
+			if opts.WithSolver {
+
+				Info(fmt.Sprintf("[%9s]  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
+					checkType,
+					idx+1, len(all),
 					p.GetCategory(), p.GetName(), p.GetVersion(),
 					r.GetCategory(), r.GetName(), r.GetVersion(),
-					errstr,
 				))
 
-				errs <- errors.New(
-					fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
-						p.GetCategory(), p.GetName(), p.GetVersion(),
-						r.GetCategory(), r.GetName(), r.GetVersion(),
-						errstr))
+				// Check if the solver is already been done for the deep
+				_, err := cacheDeps.Get(r.HashFingerprint(""))
+				if err == nil {
+					Debug(fmt.Sprintf("[%9s]  :direct_hit: Cache Hit for dep", checkType),
+						fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+					continue
+				}
 
-				brokenDeps++
+				Spinner(32)
+				solution, err := depSolver.Install(pkg.Packages{r})
+				ass := solution.SearchByName(r.GetPackageName())
+				SpinnerStop()
+				if err == nil {
+					if ass == nil {
 
-				validpkg = false
-
-			} else {
-
-				Debug("Find packages for dep",
-					fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
-
-				if withSolver {
-					Spinner(32)
-					solution, err := depSolver.Install(pkg.Packages{r})
-					ass := solution.SearchByName(r.GetPackageName())
-					if err == nil {
-						_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
-					}
-					SpinnerStop()
-
-					if err != nil {
-
-						Error(fmt.Sprintf("%s/%s-%s: solver broken for dep %s/%s-%s - %s",
-							p.GetCategory(), p.GetName(), p.GetVersion(),
-							r.GetCategory(), r.GetName(), r.GetVersion(),
-							err.Error(),
-						))
-
-						errs <- errors.New(
-							fmt.Sprintf("%s/%s-%s: solver broken for Dep %s/%s-%s - %s",
+						ans = errors.New(
+							fmt.Sprintf("[%9s] %s/%s-%s: solution doesn't retrieve package %s/%s-%s.",
+								checkType,
 								p.GetCategory(), p.GetName(), p.GetVersion(),
 								r.GetCategory(), r.GetName(), r.GetVersion(),
-								err.Error()))
+							))
 
-						brokenDeps++
+						if LuetCfg.GetGeneral().Debug {
+							for idx, pa := range solution {
+								fmt.Println(fmt.Sprintf("[%9s] %s/%s-%s: solution %d: %s",
+									checkType,
+									p.GetCategory(), p.GetName(), p.GetVersion(), idx,
+									pa.Package.GetPackageName()))
+							}
+						}
+
+						Error(ans.Error())
+						opts.IncrBrokenDeps()
 						validpkg = false
+					} else {
+						_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
 					}
-
 				}
+
+				if err != nil {
+
+					Error(fmt.Sprintf("[%9s] %s/%s-%s: solver broken for dep %s/%s-%s - %s",
+						checkType,
+						p.GetCategory(), p.GetName(), p.GetVersion(),
+						r.GetCategory(), r.GetName(), r.GetVersion(),
+						err.Error(),
+					))
+
+					ans = errors.New(
+						fmt.Sprintf("[%9s] %s/%s-%s: solver broken for Dep %s/%s-%s - %s",
+							checkType,
+							p.GetCategory(), p.GetName(), p.GetVersion(),
+							r.GetCategory(), r.GetName(), r.GetVersion(),
+							err.Error()))
+
+					opts.IncrBrokenDeps()
+					validpkg = false
+				}
+
+				// Register the key
+				cacheDeps.Set(r.HashFingerprint(""), "1")
+
+			}
+		}
+
+	}
+
+	if !validpkg {
+		opts.IncrBrokenPkgs()
+	}
+
+	return ans
+}
+
+func validateWorker(i int,
+	wg *sync.WaitGroup,
+	c <-chan pkg.Package,
+	opts *ValidateOpts) {
+
+	defer wg.Done()
+
+	for p := range c {
+
+		if opts.OnlyBuildtime {
+			// Check buildtime compiler/deps
+			err := validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+		} else if opts.OnlyRuntime {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+		} else {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+			// Check buildtime compiler/deps
+			err = validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
 			}
 
 		}
 
-		if !validpkg {
-			brokenPkgs++
+	}
+}
+
+func initOpts(opts *ValidateOpts, onlyRuntime, onlyBuildtime, withSolver bool, treePaths []string) {
+	var err error
+
+	opts.OnlyBuildtime = onlyBuildtime
+	opts.OnlyRuntime = onlyRuntime
+	opts.WithSolver = withSolver
+	opts.RuntimeReciper = nil
+	opts.BuildtimeReciper = nil
+	opts.BrokenPkgs = 0
+	opts.BrokenDeps = 0
+
+	if onlyBuildtime {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+	} else if onlyRuntime {
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	} else {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	}
+
+	opts.RuntimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+	opts.BuildtimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+
+	for _, treePath := range treePaths {
+		Info(fmt.Sprintf("Loading :deciduous_tree: %s...", treePath))
+		if opts.BuildtimeReciper != nil {
+			err = opts.BuildtimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
+		}
+		if opts.RuntimeReciper != nil {
+			err = opts.RuntimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
 		}
 	}
+
+	opts.RegExcludes, err = helpers.CreateRegexArray(opts.Excludes)
+	if err != nil {
+		Fatal(err.Error())
+	}
+	opts.RegMatches, err = helpers.CreateRegexArray(opts.Matches)
+	if err != nil {
+		Fatal(err.Error())
+	}
+
 }
 
 func NewTreeValidateCommand() *cobra.Command {
 	var excludes []string
 	var matches []string
 	var treePaths []string
+	var opts ValidateOpts
 
 	var ans = &cobra.Command{
 		Use:   "validate [OPTIONS]",
 		Short: "Validate a tree or a list of packages",
 		Args:  cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
 			if len(treePaths) < 1 {
 				Fatal("Mandatory tree param missing.")
 			}
+			if onlyRuntime && onlyBuildtime {
+				Fatal("Both --only-runtime and --only-buildtime options are not possibile.")
+			}
 		},
 		Run: func(cmd *cobra.Command, args []string) {
+			var reciper tree.Builder
+
 			concurrency := LuetCfg.GetGeneral().Concurrency
 
 			withSolver, _ := cmd.Flags().GetBool("with-solver")
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
 
-			reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
-			for _, treePath := range treePaths {
-				err := reciper.Load(treePath)
-				if err != nil {
-					Fatal("Error on load tree ", err)
-				}
-			}
+			opts.Excludes = excludes
+			opts.Matches = matches
+			initOpts(&opts, onlyRuntime, onlyBuildtime, withSolver, treePaths)
 
-			regExcludes, err := helpers.CreateRegexArray(excludes)
-			if err != nil {
-				Fatal(err.Error())
-			}
-			regMatches, err := helpers.CreateRegexArray(matches)
-			if err != nil {
-				Fatal(err.Error())
+			// We need at least one valid reciper for get list of the packages.
+			if onlyBuildtime {
+				reciper = opts.BuildtimeReciper
+			} else {
+				reciper = opts.RuntimeReciper
 			}
 
 			all := make(chan pkg.Package)
-			errs := make(chan error)
 
 			var wg = new(sync.WaitGroup)
 
 			for i := 0; i < concurrency; i++ {
 				wg.Add(1)
-				go validateWorker(i, wg, all,
-					reciper, withSolver, regExcludes, regMatches, excludes, matches,
-					errs)
+				go validateWorker(i, wg, all, &opts)
 			}
 			for _, p := range reciper.GetDatabase().World() {
 				all <- p
@@ -258,11 +459,10 @@ func NewTreeValidateCommand() *cobra.Command {
 			// Wait separately and once done close the channel
 			go func() {
 				wg.Wait()
-				close(errs)
 			}()
 
 			stringerrs := []string{}
-			for e := range errs {
+			for _, e := range opts.Errors {
 				stringerrs = append(stringerrs, e.Error())
 			}
 			sort.Strings(stringerrs)
@@ -272,19 +472,24 @@ func NewTreeValidateCommand() *cobra.Command {
 
 			// fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
 			if len(stringerrs) != 0 {
+				Error(fmt.Sprintf("Found %d broken packages and %d broken deps.",
+					opts.BrokenPkgs, opts.BrokenDeps))
 				Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
-				//	if brokenPkgs > 0 {
-				//os.Exit(1)
 			} else {
 				Info("All good! :white_check_mark:")
 				os.Exit(0)
 			}
 		},
 	}
-
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
+	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")
 	ans.Flags().BoolP("with-solver", "s", false,
 		"Enable check of requires also with solver.")
-	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{},
+	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{path},
 		"Path of the tree to use.")
 	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},
 		"Exclude matched packages from analysis. (Use string as regex).")

cmd/uninstall.go

@@ -16,21 +16,22 @@ package cmd
 
 import (
 	"os"
-	"path/filepath"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/spf13/cobra"
 )
 
 var uninstallCmd = &cobra.Command{
-	Use:   "uninstall <pkg> <pkg2> ...",
-	Short: "Uninstall a package or a list of packages",
-	Long:  `Uninstall packages`,
+	Use:     "uninstall <pkg> <pkg2> ...",
+	Short:   "Uninstall a package or a list of packages",
+	Long:    `Uninstall packages`,
+	Aliases: []string{"rm", "un"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
 		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
@@ -40,49 +41,61 @@ var uninstallCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
-
+		toRemove := []pkg.Package{}
 		for _, a := range args {
 
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
 				Fatal("Invalid package string ", a, ": ", err.Error())
 			}
+			toRemove = append(toRemove, pack)
+		}
 
-			stype := LuetCfg.Viper.GetString("solver.type")
-			discount := LuetCfg.Viper.GetFloat64("solver.discount")
-			rate := LuetCfg.Viper.GetFloat64("solver.rate")
-			attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
-			force := LuetCfg.Viper.GetBool("force")
-			nodeps := LuetCfg.Viper.GetBool("nodeps")
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps, _ := cmd.Flags().GetBool("nodeps")
+		full, _ := cmd.Flags().GetBool("full")
+		checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
+		fullClean, _ := cmd.Flags().GetBool("full-clean")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
 
-			LuetCfg.GetSolverOptions().Type = stype
-			LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-			LuetCfg.GetSolverOptions().Discount = float32(discount)
-			LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
-			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
 
-			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-				Concurrency:   LuetCfg.GetGeneral().Concurrency,
-				SolverOptions: *LuetCfg.GetSolverOptions(),
-				NoDeps:        nodeps,
-				Force:         force,
-			})
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			FullUninstall:               full,
+			FullCleanUninstall:          fullClean,
+			CheckConflicts:              checkconflicts,
+			Ask:                         !yes,
+			PreserveSystemEssentialData: true,
+		})
 
-			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-				systemDB = pkg.NewBoltDatabase(
-					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-			} else {
-				systemDB = pkg.NewInMemoryDatabase(true)
-			}
-			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
-			err = inst.Uninstall(pack, system)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+
+		if err := inst.Uninstall(system, toRemove...); err != nil {
+			Fatal("Error: " + err.Error())
 		}
 	},
 }
@@ -98,8 +111,13 @@ func init() {
 	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	uninstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
-	uninstallCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	uninstallCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
 	uninstallCmd.Flags().Bool("force", false, "Force uninstall")
+	uninstallCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
+	uninstallCmd.Flags().Bool("conflictscheck", true, "Check if the package marked for deletion is required by other packages")
+	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
+	uninstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	uninstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -16,19 +16,19 @@ package cmd
 
 import (
 	"os"
-	"path/filepath"
 
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
-	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/spf13/cobra"
 )
 
 var upgradeCmd = &cobra.Command{
-	Use:   "upgrade",
-	Short: "Upgrades the system",
+	Use:     "upgrade",
+	Short:   "Upgrades the system",
+	Aliases: []string{"u"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", installCmd.Flags().Lookup("system-dbpath"))
 		LuetCfg.Viper.BindPFlag("system.rootfs", installCmd.Flags().Lookup("system-target"))
@@ -37,10 +37,10 @@ var upgradeCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
 	Long: `Upgrades packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
 
 		repos := installer.Repositories{}
 		for _, repo := range LuetCfg.SystemRepositories {
@@ -57,34 +57,45 @@ var upgradeCmd = &cobra.Command{
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
 		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
+		nodeps, _ := cmd.Flags().GetBool("nodeps")
+		full, _ := cmd.Flags().GetBool("full")
+		universe, _ := cmd.Flags().GetBool("universe")
+		clean, _ := cmd.Flags().GetBool("clean")
+		sync, _ := cmd.Flags().GetBool("sync")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
 
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
 		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
 
 		Debug("Solver", LuetCfg.GetSolverOptions().String())
 
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:   LuetCfg.GetGeneral().Concurrency,
-			SolverOptions: *LuetCfg.GetSolverOptions(),
-			Force:         force,
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			Force:                       force,
+			FullUninstall:               full,
+			NoDeps:                      nodeps,
+			SolverUpgrade:               universe,
+			RemoveUnavailableOnUpgrade:  clean,
+			UpgradeNewRevisions:         sync,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
 		})
 		inst.Repositories(repos)
-		_, err := inst.SyncRepositories(false)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
 
-		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-		} else {
-			systemDB = pkg.NewInMemoryDatabase(true)
-		}
-		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
-		err = inst.Upgrade(system)
-		if err != nil {
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		if err := inst.Upgrade(system); err != nil {
 			Fatal("Error: " + err.Error())
 		}
 	},
@@ -102,6 +113,13 @@ func init() {
 	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
+	upgradeCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
+	upgradeCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
+	upgradeCmd.Flags().Bool("universe", false, "Use ONLY the SAT solver to compute upgrades (experimental)")
+	upgradeCmd.Flags().Bool("clean", false, "Try to drop removed packages (experimental, only when --universe is enabled)")
+	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
+	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	upgradeCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

contrib/config/config.protect.d/01_etc.yml.example

@@ -0,0 +1,3 @@
+name: "etc_conf"
+dirs:
+  - "/etc/"

contrib/config/get_luet_root.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+if [ $(id -u) -ne 0 ]
+  then echo "Please run the installer with sudo/as root"
+  exit
+fi
+
+set -ex
+export LUET_NOLOCK=true
+
+LUET_VERSION=$(curl -s https://api.github.com/repos/mudler/luet/releases/latest | ( grep -oP '"tag_name": "\K(.*)(?=")' || echo "0.9.24" ))
+LUET_ROOTFS=${LUET_ROOTFS:-/}
+LUET_DATABASE_PATH=${LUET_DATABASE_PATH:-/var/luet/db}
+LUET_DATABASE_ENGINE=${LUET_DATABASE_ENGINE:-boltdb}
+LUET_CONFIG_PROTECT=${LUET_CONFIG_PROTECT:-1}
+
+curl -L https://github.com/mudler/luet/releases/download/${LUET_VERSION}/luet-${LUET_VERSION}-linux-amd64 --output luet
+chmod +x luet
+
+mkdir -p /etc/luet/repos.conf.d || true
+mkdir -p $LUET_DATABASE_PATH || true
+mkdir -p /var/tmp/luet || true
+
+if [ "${LUET_CONFIG_PROTECT}" = "1" ] ; then
+  mkdir -p /etc/luet/config.protect.d || true
+  curl -L https://raw.githubusercontent.com/mudler/luet/master/contrib/config/config.protect.d/01_etc.yml.example --output /etc/luet/config.protect.d/01_etc.yml
+fi
+curl -L https://raw.githubusercontent.com/mocaccinoOS/repository-index/master/packages/mocaccino-repository-index.yml --output /etc/luet/repos.conf.d/mocaccino-repository-index.yml
+
+cat > /etc/luet/luet.yaml <<EOF
+general:
+  debug: false
+system:
+  rootfs: ${LUET_ROOTFS}
+  database_path: "${LUET_DATABASE_PATH}"
+  database_engine: "${LUET_DATABASE_ENGINE}"
+  tmpdir_base: "/var/tmp/luet"
+EOF
+
+./luet install -y repository/luet repository/mocaccino-repository-index
+./luet install -y system/luet system/luet-extensions
+
+rm -rf luet
+

contrib/config/luet.yaml

@@ -4,8 +4,11 @@
 # Logging configuration section:
 # ---------------------------------------------
 # logging:
+#   # Enable loggging to file (if path is not empty)
+#   enable_logfile: false
+#
 #   Leave empty to skip logging to file.
-#   path: ""
+#   path: "/var/log/luet.log"
 #
 #   Set logging level: error|warning|info|debug
 #   level: "info"
@@ -13,6 +16,12 @@
 #   Enable JSON log format instead of console mode.
 #   json_format: false.
 #
+#   Disable/Enable color
+#   color: true
+#
+#   Enable/Disable emoji
+#   enable_emoji: true
+#
 # ---------------------------------------------
 # General configuration section:
 # ---------------------------------------------
@@ -60,6 +69,7 @@
 #   Default $TMPDIR/tmpluet
 #   tmpdir_base: "/tmp/tmpluet"
 #
+#
 # ---------------------------------------------
 # Repositories configurations directories.
 # ---------------------------------------------
@@ -70,7 +80,25 @@
 #   - /etc/luet/repos.conf.d
 #
 #
-# ---------------------------------------------
+# ------------------------------------------------
+# Config protect configuration files directories.
+# -----------------------------------------------
+# Define the list of directories where load
+# configuration files with the list of config
+# protect paths.
+# config_protect_confdir:
+#   - /etc/luet/config.protect.d
+#
+# Permit to ignore rules defined on
+# config protect confdir and packages
+# annotation.
+# config_protect_skip: false
+#
+# The paths used for load repositories and config
+# protects are based on host rootfs.
+# If set to false rootfs path is used as prefix.
+# config_from_host: true
+#
 # System repositories
 # ---------------------------------------------
 # In alternative to define repositories files

go.mod

@@ -1,49 +1,66 @@
 module github.com/mudler/luet
 
-go 1.12
+go 1.14
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/Sabayon/pkgs-checker v0.6.2-0.20200404093625-076438c31739
+	github.com/Sabayon/pkgs-checker v0.7.2
+	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
-	github.com/briandowns/spinner v1.7.0
-	github.com/cavaliercoder/grab v2.0.0+incompatible
-	github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3
-	github.com/docker/docker v17.12.0-ce-rc1.0.20200417035958-130b0bc6032c+incompatible
+	github.com/briandowns/spinner v1.12.1-0.20201220203425-e201aaea0a31
+	github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec
+	github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc
+	github.com/crillab/gophersat v1.3.2-0.20201023142334-3fc2ac466765
+	github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
+	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
+	github.com/fsouza/go-dockerclient v1.6.4
+	github.com/genuinetools/img v0.5.11
 	github.com/ghodss/yaml v1.0.0
+	github.com/google/go-containerregistry v0.2.1
+	github.com/hashicorp/go-multierror v1.0.0
 	github.com/hashicorp/go-version v1.2.0
+	github.com/jedib0t/go-pretty v4.3.0+incompatible
+	github.com/jedib0t/go-pretty/v6 v6.0.5
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
+	github.com/klauspost/compress v1.8.3
 	github.com/klauspost/pgzip v1.2.1
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
-	github.com/kr/pretty v0.2.0 // indirect
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
-	github.com/mattn/go-isatty v0.0.10 // indirect
-	github.com/moby/sys/mount v0.1.1-0.20200320164225-6154f11e6840 // indirect
+	github.com/moby/buildkit v0.7.2
+	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.0
-	github.com/otiai10/copy v1.0.2
-	github.com/pelletier/go-toml v1.6.0 // indirect
+	github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
+	github.com/onsi/ginkgo v1.14.2
+	github.com/onsi/gomega v1.10.3
+	github.com/opencontainers/image-spec v1.0.1
+	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
-	github.com/pkg/errors v0.8.1
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.5.0
-	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
-	go.etcd.io/bbolt v1.3.4
+	github.com/pkg/errors v0.9.1
+	github.com/schollz/progressbar/v3 v3.7.1
+	github.com/sirupsen/logrus v1.6.0
+	github.com/spf13/cobra v1.0.0
+	github.com/spf13/viper v1.7.0
+	go.etcd.io/bbolt v1.3.5
 	go.uber.org/atomic v1.5.1 // indirect
 	go.uber.org/multierr v1.4.0 // indirect
 	go.uber.org/zap v1.13.0
-	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
-	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
-	golang.org/x/tools v0.0.0-20200102200121-6de373a2766c // indirect
-	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
-	gopkg.in/yaml.v2 v2.2.7
-	mvdan.cc/sh/v3 v3.0.0-beta1
+	gopkg.in/yaml.v2 v2.3.0
+	helm.sh/helm/v3 v3.3.4
+
 )
 
-replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200418093736-b2b0766ef22c+incompatible
+replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200605210607-749178b8f80d+incompatible
+
+replace github.com/containerd/containerd => github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55
+
+replace github.com/hashicorp/go-immutable-radix => github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe
+
+replace github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305
+
+replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc9.0.20200221051241-688cf6d43cc4

pkg/bus/events.go

@@ -0,0 +1,67 @@
+package bus
+
+import (
+	"github.com/mudler/go-pluggable"
+)
+
+var (
+	// Package events
+
+	// EventPackageInstall is the event fired when a new package is being installed
+	EventPackageInstall pluggable.EventType = "package.install"
+	// EventPackageUnInstall is the event fired when a new package is being uninstalled
+	EventPackageUnInstall pluggable.EventType = "package.uninstall"
+
+	// Package build
+
+	// EventPackagePreBuild is the event fired before a package is being built
+	EventPackagePreBuild pluggable.EventType = "package.pre.build"
+	// EventPackagePreBuildArtifact is the event fired before a package tarball is being generated
+	EventPackagePreBuildArtifact pluggable.EventType = "package.pre.build_artifact"
+	// EventPackagePostBuildArtifact is the event fired after a package tarball is generated
+	EventPackagePostBuildArtifact pluggable.EventType = "package.post.build_artifact"
+	// EventPackagePostBuild is the event fired after a package was built
+	EventPackagePostBuild pluggable.EventType = "package.post.build"
+
+	// Image build
+
+	// EventImagePreBuild is the event fired before a image is being built
+	EventImagePreBuild pluggable.EventType = "image.pre.build"
+	// EventImagePrePull is the event fired before a image is being pulled
+	EventImagePrePull pluggable.EventType = "image.pre.pull"
+	// EventImagePrePush is the event fired before a image is being pushed
+	EventImagePrePush pluggable.EventType = "image.pre.push"
+	// EventImagePostBuild is the event fired after an image is being built
+	EventImagePostBuild pluggable.EventType = "image.post.build"
+	// EventImagePostPull is the event fired after an image is being pulled
+	EventImagePostPull pluggable.EventType = "image.post.pull"
+	// EventImagePostPush is the event fired after an image is being pushed
+	EventImagePostPush pluggable.EventType = "image.post.push"
+
+	// Repository events
+
+	// EventRepositoryPreBuild is the event fired before a repository is being built
+	EventRepositoryPreBuild pluggable.EventType = "repository.pre.build"
+	// EventRepositoryPostBuild is the event fired after a repository was built
+	EventRepositoryPostBuild pluggable.EventType = "repository.post.build"
+)
+
+// Manager is the bus instance manager, which subscribes plugins to events emitted by Luet
+var Manager *pluggable.Manager = pluggable.NewManager(
+	[]pluggable.EventType{
+		EventPackageInstall,
+		EventPackageUnInstall,
+		EventPackagePreBuild,
+		EventPackagePreBuildArtifact,
+		EventPackagePostBuildArtifact,
+		EventPackagePostBuild,
+		EventRepositoryPreBuild,
+		EventRepositoryPostBuild,
+		EventImagePreBuild,
+		EventImagePrePull,
+		EventImagePrePush,
+		EventImagePostBuild,
+		EventImagePostPull,
+		EventImagePostPush,
+	},
+)

pkg/compiler/artifact.go

@@ -18,6 +18,8 @@ package compiler
 import (
 	"archive/tar"
 	"bufio"
+	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -25,15 +27,19 @@ import (
 	"path/filepath"
 	"regexp"
 
+	system "github.com/docker/docker/pkg/system"
+	zstd "github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
 	"strings"
 	"sync"
 
+	bus "github.com/mudler/luet/pkg/bus"
 	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v2"
@@ -42,8 +48,9 @@ import (
 type CompressionImplementation string
 
 const (
-	None CompressionImplementation = "none" // e.g. tar for standard packages
-	GZip CompressionImplementation = "gzip"
+	None      CompressionImplementation = "none" // e.g. tar for standard packages
+	GZip      CompressionImplementation = "gzip"
+	Zstandard CompressionImplementation = "zstd"
 )
 
 type ArtifactIndex []Artifact
@@ -137,14 +144,14 @@ func (a *PackageArtifact) Hash() error {
 
 func (a *PackageArtifact) Verify() error {
 	sum := Checksums{}
-	err := sum.Generate(a)
-	if err != nil {
+	if err := sum.Generate(a); err != nil {
 		return err
 	}
-	err = sum.Compare(a.Checksums)
-	if err != nil {
+
+	if err := sum.Compare(a.Checksums); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -166,6 +173,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
 	}
 
+	bus.Manager.Publish(bus.EventPackagePreBuildArtifact, a)
+
 	mangle, err := NewPackageArtifactFromYaml(data)
 	if err != nil {
 		return errors.Wrap(err, "Generated invalid artifact")
@@ -187,6 +196,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 		return errors.Wrap(err, "While writing PackageArtifact YAML")
 	}
 	//a.CompileSpec.GetPackage().SetPath(p)
+	bus.Manager.Publish(bus.EventPackagePostBuildArtifact, a)
 
 	return nil
 }
@@ -227,13 +237,133 @@ func (a *PackageArtifact) GetPath() string {
 	return a.Path
 }
 
+func (a *PackageArtifact) GetFileName() string {
+	return path.Base(a.GetPath())
+}
+
 func (a *PackageArtifact) SetPath(p string) {
 	a.Path = p
 }
 
-// Compress Archives and compress (TODO) to the artifact path
+func (a *PackageArtifact) genDockerfile() string {
+	return `
+FROM scratch
+COPY . /`
+}
+
+// CreateArtifactForFile creates a new artifact from the given file
+func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
+
+	fileName := path.Base(s)
+	archive, err := LuetCfg.GetSystem().TempDir("archive")
+	if err != nil {
+		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
+	}
+	defer os.RemoveAll(archive) // clean up
+	helpers.CopyFile(s, filepath.Join(archive, fileName))
+	artifact, err := LuetCfg.GetSystem().TempDir("artifact")
+	if err != nil {
+		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
+	}
+	a := &PackageArtifact{Path: filepath.Join(artifact, fileName)}
+
+	for _, o := range opts {
+		o(a)
+	}
+
+	return a, a.Compress(archive, 1)
+}
+
+// GenerateFinalImage takes an artifact and builds a Docker image with its content
+func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend, keepPerms bool) (CompilerBackendOptions, error) {
+	builderOpts := CompilerBackendOptions{}
+	archive, err := LuetCfg.GetSystem().TempDir("archive")
+	if err != nil {
+		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	}
+	defer os.RemoveAll(archive) // clean up
+
+	uncompressedFiles := filepath.Join(archive, "files")
+	dockerFile := filepath.Join(archive, "Dockerfile")
+
+	if err := os.MkdirAll(uncompressedFiles, os.ModePerm); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	}
+
+	if err := a.Unpack(uncompressedFiles, keepPerms); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
+	}
+
+	empty, err := helpers.DirectoryIsEmpty(uncompressedFiles)
+	if err != nil {
+		return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles)
+	}
+
+	// See https://github.com/moby/moby/issues/38039.
+	// We can't generate FROM scratch empty images. Docker will refuse to export them
+	// workaround: Inject a .virtual empty file
+	if empty {
+		helpers.Touch(filepath.Join(uncompressedFiles, ".virtual"))
+	}
+
+	data := a.genDockerfile()
+	if err := ioutil.WriteFile(dockerFile, []byte(data), 0644); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path)
+	}
+
+	builderOpts = CompilerBackendOptions{
+		ImageName:      imageName,
+		SourcePath:     archive,
+		DockerFileName: dockerFile,
+		Context:        uncompressedFiles,
+	}
+	return builderOpts, b.BuildImage(builderOpts)
+}
+
+// Compress is responsible to archive and compress to the artifact Path.
+// It accepts a source path, which is the content to be archived/compressed
+// and a concurrency parameter.
 func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	switch a.CompressionType {
+
+	case Zstandard:
+		err := helpers.Tar(src, a.Path)
+		if err != nil {
+			return err
+		}
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return err
+		}
+		defer original.Close()
+
+		zstdFile := a.getCompressedName()
+		bufferedReader := bufio.NewReader(original)
+
+		// Open a file for writing.
+		dst, err := os.Create(zstdFile)
+		if err != nil {
+			return err
+		}
+
+		enc, err := zstd.NewWriter(dst)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(enc, bufferedReader)
+		if err != nil {
+			enc.Close()
+			return err
+		}
+		if err := enc.Close(); err != nil {
+			return err
+		}
+
+		os.RemoveAll(a.Path) // Remove original
+		Debug("Removed artifact", a.Path)
+
+		a.Path = zstdFile
+		return nil
 	case GZip:
 		err := helpers.Tar(src, a.Path)
 		if err != nil {
@@ -245,7 +375,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 		}
 		defer original.Close()
 
-		gzipfile := a.Path + ".gz"
+		gzipfile := a.getCompressedName()
 		bufferedReader := bufio.NewReader(original)
 
 		// Open a file for writing.
@@ -255,7 +385,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 		}
 		// Create gzip writer.
 		w := gzip.NewWriter(dst)
-		w.SetConcurrency(concurrency, 10)
+		w.SetConcurrency(1<<20, concurrency)
 		defer w.Close()
 		defer dst.Close()
 		_, err = io.Copy(w, bufferedReader)
@@ -264,6 +394,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 		}
 		w.Close()
 		os.RemoveAll(a.Path) // Remove original
+		Debug("Removed artifact", a.Path)
 		//	a.CompressedPath = gzipfile
 		a.Path = gzipfile
 		return nil
@@ -271,15 +402,147 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.Tar(src, a.Path)
-
+		return helpers.Tar(src, a.getCompressedName())
 	}
 	return errors.New("Compression type must be supplied")
 }
 
+func (a *PackageArtifact) getCompressedName() string {
+	switch a.CompressionType {
+	case Zstandard:
+		return a.Path + ".zst"
+
+	case GZip:
+		return a.Path + ".gz"
+	}
+	return a.Path
+}
+
+// GetUncompressedName returns the artifact path without the extension suffix
+func (a *PackageArtifact) GetUncompressedName() string {
+	switch a.CompressionType {
+	case Zstandard, GZip:
+		return strings.TrimSuffix(a.Path, filepath.Ext(a.Path))
+	}
+	return a.Path
+}
+
+func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	// If the destination path already exists I rename target file name with postfix.
+	var destPath string
+
+	// Read data. TODO: We need change archive callback to permit to return a Reader
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	// If file is not present on archive but is defined on mods
+	// I receive the callback. Prevent nil exception.
+	if header != nil {
+		switch header.Typeflag {
+		case tar.TypeReg:
+			destPath = filepath.Join(dst, path)
+		default:
+			// Nothing to do. I return original reader
+			return header, buffer.Bytes(), nil
+		}
+
+		// Check if exists
+		if helpers.Exists(destPath) {
+			for i := 1; i < 1000; i++ {
+				name := filepath.Join(filepath.Join(filepath.Dir(path),
+					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
+
+				if helpers.Exists(name) {
+					continue
+				}
+				Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
+					filepath.Join(dst, name)))
+				return &tar.Header{
+					Mode:       header.Mode,
+					Typeflag:   header.Typeflag,
+					PAXRecords: header.PAXRecords,
+					Name:       name,
+				}, buffer.Bytes(), nil
+			}
+		}
+	}
+
+	return header, buffer.Bytes(), nil
+}
+
+func (a *PackageArtifact) GetProtectFiles() []string {
+	ans := []string{}
+	annotationDir := ""
+
+	if !LuetCfg.ConfigProtectSkip {
+
+		// a.CompileSpec could be nil when artifact.Unpack is used for tree tarball
+		if a.CompileSpec != nil &&
+			a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
+			}
+		}
+		// TODO: check if skip this if we have a.CompileSpec nil
+
+		cp := NewConfigProtect(annotationDir)
+		cp.Map(a.Files)
+
+		// NOTE: for unpack we need files path without initial /
+		ans = cp.GetProtectFiles(false)
+	}
+
+	return ans
+}
+
 // Unpack Untar and decompress (TODO) to the given path
 func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+
+	// Create
+	protectedFiles := a.GetProtectFiles()
+
+	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
+
 	switch a.CompressionType {
+	case Zstandard:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+
+		d, err := zstd.NewReader(bufferedReader)
+		if err != nil {
+			return err
+		}
+		defer d.Close()
+
+		_, err = io.Copy(archive, d)
+		if err != nil {
+			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
+		}
+
+		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
+		if err != nil {
+			return err
+		}
+		return nil
 	case GZip:
 		// Create the uncompressed archive
 		archive, err := os.Create(a.GetPath() + ".uncompressed")
@@ -307,15 +570,16 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
 		}
 
-		err = helpers.Untar(a.GetPath()+".uncompressed", dst,
-			LuetCfg.GetGeneral().SameOwner)
+		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
 		}
 		return nil
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.Untar(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner)
+		return helpers.UntarProtect(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner,
+			protectedFiles, tarModifier)
 	}
 	return errors.New("Compression type must be supplied")
 }
@@ -324,6 +588,27 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 func (a *PackageArtifact) FileList() ([]string, error) {
 	var tr *tar.Reader
 	switch a.CompressionType {
+	case Zstandard:
+		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		if err != nil {
+			return []string{}, err
+		}
+		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := zstd.NewReader(bufferedReader)
+		if err != nil {
+			return []string{}, err
+		}
+		defer r.Close()
+		tr = tar.NewReader(r)
 	case GZip:
 		// Create the uncompressed archive
 		archive, err := os.Create(a.GetPath() + ".uncompressed")
@@ -386,6 +671,27 @@ type CopyJob struct {
 	Artifact string
 }
 
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
@@ -399,16 +705,32 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 		// 	continue
 		// }
 
-		if !helpers.Exists(job.Dst) {
+		_, err := os.Lstat(job.Dst)
+		if err != nil {
+			Debug("Copying ", job.Src)
 			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
+			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
 
+func compileRegexes(regexes []string) []*regexp.Regexp {
+	var result []*regexp.Regexp
+	for _, i := range regexes {
+		r, e := regexp.Compile(i)
+		if e != nil {
+			Warning("Failed compiling regex:", e)
+			continue
+		}
+		result = append(result, r)
+	}
+	return result
+}
+
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, t CompressionImplementation) (Artifact, error) {
+func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t CompressionImplementation) (Artifact, error) {
 
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
@@ -438,16 +760,9 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 	}
 
 	// Handle includes in spec. If specified they filter what gets in the package
-	if len(includes) > 0 {
-		var includeRegexp []*regexp.Regexp
-		for _, i := range includes {
-			r, e := regexp.Compile(i)
-			if e != nil {
-				Warning("Failed compiling regex:", e)
-				continue
-			}
-			includeRegexp = append(includeRegexp, r)
-		}
+
+	if len(includes) > 0 && len(excludes) == 0 {
+		includeRegexp := compileRegexes(includes)
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 		ADDS:
@@ -459,19 +774,83 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 					}
 				}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
+
+	} else if len(includes) == 0 && len(excludes) != 0 {
+		excludeRegexp := compileRegexes(excludes)
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		ADD:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range excludeRegexp {
+					if i.MatchString(a.Name) {
+						continue ADD
+					}
+				}
+				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
+		}
+
+	} else if len(includes) != 0 && len(excludes) != 0 {
+		includeRegexp := compileRegexes(includes)
+		excludeRegexp := compileRegexes(excludes)
+
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		EXCLUDES:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) {
+						for _, e := range excludeRegexp {
+							if e.MatchString(a.Name) {
+								continue EXCLUDES
+							}
+						}
+						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+						continue EXCLUDES
+					}
+				}
+			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
+		}
+
 	} else {
 		// Otherwise just grab all
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 			for _, a := range l.Diffs.Additions {
+				Debug("File ", a.Name, " added")
 				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	}
 
 	close(toCopy)
 	wg.Wait()
+
 	a := NewPackageArtifact(dst)
 	a.SetCompressionType(t)
 	err = a.Compress(archive, concurrency)

pkg/compiler/artifact_test.go

@@ -20,7 +20,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/solver"
 
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
@@ -41,7 +43,7 @@ var _ = Describe("Artifact", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -85,7 +87,8 @@ ENV PACKAGE_CATEGORY=app-admin`))
 				DockerFileName: "Dockerfile",
 				Destination:    filepath.Join(tmpdir2, "output1.tar"),
 			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
+			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
 
@@ -95,37 +98,47 @@ ENV PACKAGE_CATEGORY=app-admin`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
-			opts = CompilerBackendOptions{
+			opts2 := CompilerBackendOptions{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
 				Destination:    filepath.Join(tmpdir, "output2.tar"),
 			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
+			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
-			diffs, err := b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))
+			diffs, err := b.Changes(opts, opts2)
 			Expect(err).ToNot(HaveOccurred())
 
+			artifacts := []ArtifactNode{{
+				Name: "/luetbuild/LuetDockerfile",
+				Size: 175,
+			}}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
 			Expect(diffs).To(Equal(
 				[]ArtifactLayer{{
-					FromImage: filepath.Join(tmpdir2, "output1.tar"),
-					ToImage:   filepath.Join(tmpdir, "output2.tar"),
+					FromImage: "luet/base",
+					ToImage:   "test",
 					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+						Additions: artifacts,
 					},
 				}}))
-			err = b.ExtractRootfs(CompilerBackendOptions{SourcePath: filepath.Join(tmpdir, "output2.tar"), Destination: rootfs}, false)
+			err = b.ExtractRootfs(CompilerBackendOptions{ImageName: "test", Destination: rootfs}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, None)
+			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, None)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
 			err = helpers.Untar(artifact.GetPath(), unpacked, false)
@@ -149,5 +162,107 @@ RUN echo bar > /test2`))
 			Expect(err).To(HaveOccurred())
 		})
 
+		It("Generates packages images", func() {
+			b := NewSimpleDockerBackend()
+			imageprefix := "foo/"
+			testString := []byte(`funky test data`)
+
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "artifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpWork, err := ioutil.TempDir(os.TempDir(), "artifact2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpWork) // clean up
+
+			Expect(os.MkdirAll(filepath.Join(tmpdir, "foo", "bar"), os.ModePerm)).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "test"), testString, 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "foo", "bar", "test"), testString, 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			artifact := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			artifact.SetCompileSpec(&LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}})
+
+			err = artifact.Compress(tmpdir, 1)
+			Expect(err).ToNot(HaveOccurred())
+			resultingImage := imageprefix + "foo--1.0"
+			opts, err := artifact.GenerateFinalImage(resultingImage, b, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(opts.ImageName).To(Equal(resultingImage))
+
+			Expect(b.ImageExists(resultingImage)).To(BeTrue())
+
+			result, err := ioutil.TempDir(os.TempDir(), "result")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(result) // clean up
+
+			err = b.ExtractRootfs(CompilerBackendOptions{ImageName: resultingImage, Destination: result}, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			content, err := ioutil.ReadFile(filepath.Join(result, "test"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(content).To(Equal(testString))
+
+			content, err = ioutil.ReadFile(filepath.Join(result, "foo", "bar", "test"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(content).To(Equal(testString))
+		})
+
+		It("Generates empty packages images", func() {
+			b := NewSimpleDockerBackend()
+			imageprefix := "foo/"
+
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "artifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpWork, err := ioutil.TempDir(os.TempDir(), "artifact2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpWork) // clean up
+
+			artifact := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			artifact.SetCompileSpec(&LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}})
+
+			err = artifact.Compress(tmpdir, 1)
+			Expect(err).ToNot(HaveOccurred())
+			resultingImage := imageprefix + "foo--1.0"
+			opts, err := artifact.GenerateFinalImage(resultingImage, b, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(opts.ImageName).To(Equal(resultingImage))
+
+			Expect(b.ImageExists(resultingImage)).To(BeTrue())
+
+			result, err := ioutil.TempDir(os.TempDir(), "result")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(result) // clean up
+
+			err = b.ExtractRootfs(CompilerBackendOptions{ImageName: resultingImage, Destination: result}, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.DirectoryIsEmpty(result)).To(BeFalse())
+			content, err := ioutil.ReadFile(filepath.Join(result, ".virtual"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(content)).To(Equal(""))
+		})
+
+		It("Retrieves uncompressed name", func() {
+			a := NewPackageArtifact("foo.tar.gz")
+			a.SetCompressionType(compiler.GZip)
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+
+			a = NewPackageArtifact("foo.tar.zst")
+			a.SetCompressionType(compiler.Zstandard)
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+
+			a = NewPackageArtifact("foo.tar")
+			a.SetCompressionType(compiler.None)
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+		})
 	})
 })

pkg/compiler/backend/common.go

@@ -0,0 +1,86 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"os/exec"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/pkg/errors"
+)
+
+const (
+	ImgBackend    = "img"
+	DockerBackend = "docker"
+)
+
+func imageAvailable(image string) bool {
+	_, err := crane.Digest(image)
+	return err == nil
+}
+
+func NewBackend(s string) compiler.CompilerBackend {
+	var compilerBackend compiler.CompilerBackend
+
+	switch s {
+	case ImgBackend:
+		compilerBackend = NewSimpleImgBackend()
+	case DockerBackend:
+		compilerBackend = NewSimpleDockerBackend()
+	}
+	return compilerBackend
+}
+
+func runCommand(cmd *exec.Cmd) error {
+	output := ""
+	buffered := !config.LuetCfg.GetGeneral().ShowBuildOutput
+	writer := NewBackendWriter(buffered)
+
+	cmd.Stdout = writer
+	cmd.Stderr = writer
+
+	if buffered {
+		Spinner(22)
+		defer SpinnerStop()
+	}
+
+	err := cmd.Start()
+	if err != nil {
+		return errors.Wrap(err, "Failed starting command")
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		output = writer.GetCombinedOutput()
+		return errors.Wrapf(err, "Failed running command: %s", output)
+	}
+
+	return nil
+}
+
+func genBuildCommand(opts compiler.CompilerBackendOptions) []string {
+	context := opts.Context
+
+	if context == "" {
+		context = "."
+	}
+	buildarg := append(opts.BackendArgs, "-f", opts.DockerFileName, "-t", opts.ImageName, context)
+	return append([]string{"build"}, buildarg...)
+}

pkg/compiler/backend/diff.go

@@ -0,0 +1,165 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/pkg/errors"
+)
+
+// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
+// example of json return: [
+//   {
+//     "Image1": "luet/base",
+//     "Image2": "alpine",
+//     "DiffType": "File",
+//     "Diff": {
+//       "Adds": null,
+//       "Dels": [
+//         {
+//           "Name": "/luetbuild",
+//           "Size": 5830706
+//         },
+//         {
+//           "Name": "/luetbuild/Dockerfile",
+//           "Size": 50
+//         },
+//         {
+//           "Name": "/luetbuild/output1",
+//           "Size": 5830656
+//         }
+//       ],
+//       "Mods": null
+//     }
+//   }
+// ]
+func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
+
+	res := compiler.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
+
+	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(srcRootFS) // clean up
+
+	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(dstRootFS) // clean up
+
+	srcImageExtract := compiler.CompilerBackendOptions{
+		ImageName:   fromImage.ImageName,
+		Destination: srcRootFS,
+	}
+	Debug("Extracting source image", fromImage.ImageName)
+	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
+	}
+
+	dstImageExtract := compiler.CompilerBackendOptions{
+		ImageName:   toImage.ImageName,
+		Destination: dstRootFS,
+	}
+	Debug("Extracting destination image", toImage.ImageName)
+	err = b.ExtractRootfs(dstImageExtract, false)
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
+	}
+
+	// Get Additions/Changes. dst -> src
+	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, dstRootFS, "", -1)
+		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
+		if err == nil {
+			var sizeA, sizeB int64
+			sizeA = fileInfo.Size()
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+
+			if sizeA != sizeB {
+				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
+				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
+					Name: filepath.Join("/", realpath),
+					Size: int(sizeB),
+				})
+			} else {
+				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
+			}
+		} else {
+			var sizeB int64
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+				Size: int(sizeB),
+			})
+
+			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+	}
+
+	// Get deletions. src -> dst
+	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, srcRootFS, "", -1)
+		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
+			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
+			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+			})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+	}
+
+	return []compiler.ArtifactLayer{res}, nil
+}

pkg/compiler/backend/diff_test.go

@@ -0,0 +1,78 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend_test
+
+import (
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Docker image diffs", func() {
+	var b compiler.CompilerBackend
+
+	BeforeEach(func() {
+		b = NewSimpleDockerBackend()
+	})
+
+	Context("Generate diffs from docker images", func() {
+		It("Detect no changes", func() {
+			opts := compiler.CompilerBackendOptions{
+				ImageName: "alpine:latest",
+			}
+			err := b.DownloadImage(opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, opts, opts)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+
+		It("Detects additions and changed files", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/micro",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			err = b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/extra",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/micro",
+			}, compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/extra",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+
+			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+	})
+})

pkg/compiler/backend/simpledocker.go

@@ -16,7 +16,6 @@
 package backend
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -25,6 +24,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	docker "github.com/fsouza/go-dockerclient"
 	capi "github.com/mudler/docker-companion/api"
 
 	"github.com/mudler/luet/pkg/compiler"
@@ -44,23 +44,34 @@ func NewSimpleDockerBackend() compiler.CompilerBackend {
 // TODO: Missing still: labels, and build args expansion
 func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
-	path := opts.SourcePath
-	dockerfileName := opts.DockerFileName
-	buildarg := []string{"build", "-f", dockerfileName, "-t", name, "."}
 
-	Debug(":whale2: Building image " + name)
+	buildarg := genBuildCommand(opts)
+	Info(":whale2: Building image " + name)
 	cmd := exec.Command("docker", buildarg...)
-	cmd.Dir = path
-	out, err := cmd.CombinedOutput()
+	cmd.Dir = opts.SourcePath
+	err := runCommand(cmd)
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return err
 	}
+
 	Info(":whale: Building image " + name + " done")
 
-	if config.LuetCfg.GetGeneral().ShowBuildOutput {
-		Info(string(out))
-	} else {
-		Debug(string(out))
+	if os.Getenv("DOCKER_SQUASH") == "true" {
+		Info(":whale: Squashing image " + name)
+		var client *docker.Client
+
+		Spinner(22)
+		defer SpinnerStop()
+		client, err = docker.NewClientFromEnv()
+		if err != nil {
+			return errors.Wrap(err, "could not connect to the Docker daemon")
+		}
+		err = capi.Squash(client, name, name)
+		if err != nil {
+			return errors.Wrap(err, "Failed squashing image")
+		}
+
+		Info(":whale: Squashing image " + name + " done")
 	}
 
 	return nil
@@ -81,15 +92,37 @@ func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	buildarg := []string{"pull", name}
 	Debug(":whale: Downloading image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	cmd := exec.Command("docker", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed pulling image: "+string(out))
 	}
+
 	Info(":whale: Downloaded image:", name)
 	return nil
 }
 
+func (*SimpleDocker) ImageExists(imagename string) bool {
+	buildarg := []string{"inspect", "--type=image", imagename}
+	Debug(":whale: Checking existance of docker image: " + imagename)
+	cmd := exec.Command("docker", buildarg...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		Debug("Image not present")
+		Debug(string(out))
+		return false
+	}
+	return true
+}
+
+func (*SimpleDocker) ImageAvailable(imagename string) bool {
+	return imageAvailable(imagename)
+}
+
 func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
@@ -105,6 +138,10 @@ func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	pusharg := []string{"push", name}
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("docker", pusharg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed pushing image: "+string(out))
@@ -133,12 +170,16 @@ func (*SimpleDocker) ExportImage(opts compiler.CompilerBackendOptions) error {
 
 	buildarg := []string{"save", name, "-o", path}
 	Debug(":whale: Saving image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("docker", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed exporting image: "+string(out))
 	}
 
-	Info(":whale: Exported image:", name)
+	Debug(":whale: Exported image:", name)
 	return nil
 }
 
@@ -146,10 +187,42 @@ type ManifestEntry struct {
 	Layers []string `json:"Layers"`
 }
 
-func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
-	src := opts.SourcePath
+func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
+	name := opts.ImageName
 	dst := opts.Destination
 
+	tempexport, err := ioutil.TempDir(dst, "tmprootfs")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tempexport) // clean up
+
+	imageExport := filepath.Join(tempexport, "image.tar")
+
+	Spinner(22)
+	defer SpinnerStop()
+
+	if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: name, Destination: imageExport}); err != nil {
+		return errors.Wrap(err, "failed while extracting rootfs for "+name)
+	}
+
+	SpinnerStop()
+
+	src := imageExport
+
+	if src == "" && opts.ImageName != "" {
+		tempUnpack, err := ioutil.TempDir(dst, "tempUnpack")
+		if err != nil {
+			return errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+		defer os.RemoveAll(tempUnpack) // clean up
+		imageExport := filepath.Join(tempUnpack, "image.tar")
+		if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
+			return errors.Wrap(err, "while exporting image before extraction")
+		}
+		src = imageExport
+	}
+
 	rootfs, err := ioutil.TempDir(dst, "tmprootfs")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
@@ -190,7 +263,7 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 			}
 		}
 	}
-
+	// TODO: Drop capi in favor of the img approach already used in pkg/installer/repository
 	export, err := capi.CreateExport(rootfs)
 	if err != nil {
 		return err
@@ -209,64 +282,9 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 	return nil
 }
 
-// 	container-diff diff daemon://luet/base alpine --type=file -j
-// [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("tmpdiffs")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-	var errorBuffer bytes.Buffer
-
-	diffargs := []string{"diff", fromImage, toImage, "-v", "error", "-q", "--type=file", "-j", "-n", "-c", tmpdiffs}
-	cmd := exec.Command("container-diff", diffargs...)
-	cmd.Stderr = &errorBuffer
-	out, err := cmd.Output()
-
-	if string(errorBuffer.Bytes()) != "" {
-		Warning("container-diff errored with: " + string(errorBuffer.Bytes()))
-	}
-
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
-	}
-
-	if config.LuetCfg.GetGeneral().ShowBuildOutput {
-		Info(string(out))
-	}
-
-	var diffs []compiler.ArtifactLayer
-
-	err = json.Unmarshal(out, &diffs)
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
-	}
+// Changes retrieves changes between image layers
+func (d *SimpleDocker) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
+	diffs, err := GenerateChanges(d, fromImage, toImage)
 
 	if config.LuetCfg.GetGeneral().Debug {
 		summary := compiler.ComputeArtifactLayerSummary(diffs)
@@ -279,5 +297,5 @@ func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLaye
 		}
 	}
 
-	return diffs, nil
+	return diffs, err
 }

pkg/compiler/backend/simpledocker_test.go

@@ -18,6 +18,7 @@ package backend_test
 import (
 	. "github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/solver"
 
 	"io/ioutil"
 	"os"
@@ -40,7 +41,7 @@ var _ = Describe("Docker backend", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -76,9 +77,10 @@ ENV PACKAGE_CATEGORY=app-admin`))
 				DockerFileName: "Dockerfile",
 				Destination:    filepath.Join(tmpdir2, "output1.tar"),
 			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
@@ -86,33 +88,60 @@ ENV PACKAGE_CATEGORY=app-admin`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
-			opts = CompilerBackendOptions{
+			opts2 := CompilerBackendOptions{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
 				Destination:    filepath.Join(tmpdir, "output2.tar"),
 			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
+
+			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
-			Expect(b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))).To(Equal(
+			artifacts := []ArtifactNode{{
+				Name: "/luetbuild/LuetDockerfile",
+				Size: 175,
+			}}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
+			Expect(b.Changes(opts, opts2)).To(Equal(
 				[]ArtifactLayer{{
-					FromImage: filepath.Join(tmpdir2, "output1.tar"),
-					ToImage:   filepath.Join(tmpdir, "output2.tar"),
+					FromImage: "luet/base",
+					ToImage:   "test",
 					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+						Additions: artifacts,
 					},
 				}}))
 
+			opts2 = CompilerBackendOptions{
+				ImageName:      "test",
+				SourcePath:     tmpdir,
+				DockerFileName: "LuetDockerfile",
+				Destination:    filepath.Join(tmpdir, "output3.tar"),
+			}
+
+			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
+			Expect(b.ImageExists(opts2.ImageName)).To(BeFalse())
 		})
 
+		It("Detects available images", func() {
+			b := NewSimpleDockerBackend()
+			Expect(b.ImageAvailable("quay.io/mocaccino/extra")).To(BeTrue())
+			Expect(b.ImageAvailable("ubuntu:20.10")).To(BeTrue())
+			Expect(b.ImageAvailable("igjo5ijgo25nho52")).To(BeFalse())
+		})
 	})
 })

pkg/compiler/backend/simpleimg.go

@@ -18,6 +18,7 @@ package backend
 import (
 	"os"
 	"os/exec"
+	"strings"
 
 	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/logger"
@@ -34,21 +35,20 @@ func NewSimpleImgBackend() compiler.CompilerBackend {
 // TODO: Missing still: labels, and build args expansion
 func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
-	path := opts.SourcePath
-	dockerfileName := opts.DockerFileName
 
-	buildarg := []string{"build", "-f", dockerfileName, "-t", name, "."}
-	Spinner(22)
-	defer SpinnerStop()
-	Debug(":tea: Building image " + name)
+	buildarg := genBuildCommand(opts)
+
+	Info(":tea: Building image " + name)
+
 	cmd := exec.Command("img", buildarg...)
-	cmd.Dir = path
-	out, err := cmd.CombinedOutput()
-
+	cmd.Dir = opts.SourcePath
+	err := runCommand(cmd)
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return err
 	}
+
 	Info(":tea: Building image " + name + " done")
+
 	return nil
 }
 
@@ -59,7 +59,7 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	defer SpinnerStop()
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed removing image: "+string(out))
 	}
 
 	Info(":tea: Image " + name + " removed")
@@ -67,15 +67,17 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
 }
 
 func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
-
 	name := opts.ImageName
 	buildarg := []string{"pull", name}
-
 	Debug(":tea: Downloading image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	cmd := exec.Command("img", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed downloading image: "+string(out))
 	}
 
 	Info(":tea: Image " + name + " downloaded")
@@ -97,6 +99,23 @@ func (*SimpleImg) CopyImage(src, dst string) error {
 	return nil
 }
 
+func (*SimpleImg) ImageAvailable(imagename string) bool {
+	return imageAvailable(imagename)
+}
+
+// ImageExists check if the given image is available locally
+func (*SimpleImg) ImageExists(imagename string) bool {
+	cmd := exec.Command("img", "ls")
+	out, err := cmd.Output()
+	if err != nil {
+		return false
+	}
+	if strings.Contains(string(out), imagename) {
+		return true
+	}
+	return false
+}
+
 func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
@@ -115,35 +134,49 @@ func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
 	path := opts.Destination
 	buildarg := []string{"save", "-o", path, name}
 	Debug(":tea: Saving image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed exporting image: "+string(out))
 	}
 	Info(":tea: Image " + name + " saved")
 	return nil
 }
 
-// TODO: Dup in docker, refactor common code in helpers for shared parts
-func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
+// ExtractRootfs extracts the docker image content inside the destination
+func (s *SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
 	name := opts.ImageName
 	path := opts.Destination
 
+	if !s.ImageExists(name) {
+		if err := s.DownloadImage(opts); err != nil {
+			return errors.Wrap(err, "failed pulling image "+name+" during extraction")
+		}
+	}
+
 	os.RemoveAll(path)
+
 	buildarg := []string{"unpack", "-o", path, name}
 	Debug(":tea: Extracting image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed extracting image: "+string(out))
 	}
-	Info(":tea: Image " + name + " extracted")
+	Debug(":tea: Image " + name + " extracted")
 	return nil
-	//return NewSimpleDockerBackend().ExtractRootfs(opts, keepPerms)
 }
 
 // TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
 // Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	return NewSimpleDockerBackend().Changes(fromImage, toImage)
+func (i *SimpleImg) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
+	return GenerateChanges(i, fromImage, toImage)
 }
 
 func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {

pkg/compiler/backend/writer.go

@@ -0,0 +1,48 @@
+// Copyright © 2021 Daniele Rondina <geaaru@sabayonlinux.org>
+//                  Ettore Di Giacinto <mudler@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"bytes"
+
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+type BackendWriter struct {
+	BufferedOutput bool
+	Buffer         *bytes.Buffer
+}
+
+func NewBackendWriter(buffered bool) *BackendWriter {
+	return &BackendWriter{
+		BufferedOutput: buffered,
+		Buffer:         &bytes.Buffer{},
+	}
+}
+
+func (b *BackendWriter) Write(p []byte) (int, error) {
+	if b.BufferedOutput {
+		return b.Buffer.Write(p)
+	}
+
+	Msg("info", false, false, (string(p)))
+
+	return len(p), nil
+}
+
+func (b *BackendWriter) Close() error              { return nil }
+func (b *BackendWriter) GetCombinedOutput() string { return b.Buffer.String() }

pkg/compiler/compiler.go

@@ -20,9 +20,14 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+
 	"regexp"
 	"strings"
 	"sync"
+	"time"
+
+	bus "github.com/mudler/luet/pkg/bus"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -33,6 +38,8 @@ import (
 )
 
 const BuildFile = "build.yaml"
+const DefinitionFile = "definition.yaml"
+const CollectionFile = "collection.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
@@ -43,9 +50,11 @@ type LuetCompiler struct {
 	Concurrency               int
 	CompressionType           CompressionImplementation
 	Options                   CompilerOptions
+	SolverOptions             solver.Options
+	BackedArgs                []string
 }
 
-func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions) Compiler {
+func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions, solvopts solver.Options) Compiler {
 	// The CompilerRecipe will gives us a tree with only build deps listed.
 	return &LuetCompiler{
 		Backend: backend,
@@ -58,11 +67,13 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *Compi
 		CompressionType: opt.CompressionType,
 		KeepImg:         opt.KeepImg,
 		Concurrency:     opt.Concurrency,
-		Clean:           opt.Clean,
 		Options:         *opt,
+		SolverOptions:   solvopts,
 	}
 }
-
+func (cs *LuetCompiler) SetBackendArgs(args []string) {
+	cs.BackedArgs = args
+}
 func (cs *LuetCompiler) SetConcurrency(i int) {
 	cs.Concurrency = i
 }
@@ -140,8 +151,6 @@ func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps Compilat
 }
 
 func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
-	Spinner(22)
-	defer SpinnerStop()
 	all := make(chan CompilationSpec)
 	artifacts := []Artifact{}
 	mutex := &sync.Mutex{}
@@ -174,7 +183,7 @@ func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpec
 	return artifacts, allErrors
 }
 
-func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string) error {
+func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, include bool) error {
 	var includeRegexp []*regexp.Regexp
 	for _, i := range includes {
 		r, e := regexp.Compile(i)
@@ -206,7 +215,7 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 			}
 		}
 
-		if !match {
+		if include && !match || !include && match {
 			toRemove = append(toRemove, currentpath)
 		}
 
@@ -228,33 +237,124 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 	return nil
 }
 
-func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec) (Artifact, error) {
-	if !cs.Clean {
-		if art, err := LoadArtifactFromYaml(p); err == nil {
-			Debug("Artifact reloaded. Skipping build")
-			return art, err
+func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p CompilationSpec, runnerOpts CompilerBackendOptions) (Artifact, error) {
+
+	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not create tempdir")
+	}
+	defer os.RemoveAll(rootfs) // clean up
+
+	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
+		ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not extract rootfs")
+	}
+
+	if p.GetPackageDir() != "" {
+		Info(":tophat: Packing from output dir", p.GetPackageDir())
+		rootfs = filepath.Join(rootfs, p.GetPackageDir())
+	}
+
+	if len(p.GetIncludes()) > 0 {
+		// strip from includes
+		cs.stripFromRootfs(p.GetIncludes(), rootfs, true)
+	}
+	if len(p.GetExcludes()) > 0 {
+		// strip from excludes
+		cs.stripFromRootfs(p.GetExcludes(), rootfs, false)
+	}
+	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
+	artifact.SetCompressionType(cs.CompressionType)
+
+	if err := artifact.Compress(rootfs, concurrency); err != nil {
+		return nil, errors.Wrap(err, "Error met while creating package archive")
+	}
+
+	artifact.SetCompileSpec(p)
+	return artifact, nil
+}
+
+func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions) (Artifact, error) {
+
+	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not create tempdir")
+	}
+	defer os.RemoveAll(rootfs) // clean up
+
+	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
+	if cs.Options.PullFirst && !cs.Backend.ImageExists(builderOpts.ImageName) && cs.Backend.ImageAvailable(builderOpts.ImageName) {
+		err := cs.Backend.DownloadImage(builderOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not pull image")
 		}
 	}
-	pkgTag := ":package:  " + p.GetPackage().GetName()
+
+	Info(pkgTag, ":hammer: Generating delta")
+	diffs, err := cs.Backend.Changes(builderOpts, runnerOpts)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not generate changes from layers")
+	}
+
+	Debug("Extracting image to grab files from delta")
+	if err := cs.Backend.ExtractRootfs(CompilerBackendOptions{
+		ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions); err != nil {
+		return nil, errors.Wrap(err, "Could not extract rootfs")
+	}
+	artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.CompressionType)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not generate deltas")
+	}
+
+	artifact.SetCompileSpec(p)
+	return artifact, nil
+}
+
+func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
+	concurrency int, keepPermissions bool,
+	p CompilationSpec) (CompilerBackendOptions, CompilerBackendOptions, error) {
+
+	var runnerOpts, builderOpts CompilerBackendOptions
+
+	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
+
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	// TODO: We should use the image tag, or pass by the package assertion hash which is unique
+	// and identifies the deptree of the package.
+
+	fp := p.GetPackage().HashFingerprint(helpers.StripRegistryFromImage(packageImage))
+
+	if buildertaggedImage == "" {
+		buildertaggedImage = cs.ImageRepository + ":builder-" + fp
+		Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image)
+	}
+
+	// TODO:  Cleanup, not actually hit
+	if packageImage == "" {
+		packageImage = cs.ImageRepository + ":builder-invalid" + fp
+	}
 
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
 	// and we build all the images first.
 
 	err := os.MkdirAll(p.Rel("build"), os.ModePerm)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for building")
+		return builderOpts, runnerOpts, errors.Wrap(err, "Error met while creating tempdir for building")
 	}
 	buildDir, err := ioutil.TempDir(p.Rel("build"), "pack")
 	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for building")
+		return builderOpts, runnerOpts, errors.Wrap(err, "Error met while creating tempdir for building")
 	}
 	defer os.RemoveAll(buildDir) // clean up
 
 	// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
 	err = helpers.CopyDir(p.GetPackage().GetPath(), buildDir)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not copy package sources")
-
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not copy package sources")
 	}
 
 	// Copy file into the build context, the compilespec might have requested to do so.
@@ -265,159 +365,129 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 	}
 
-	fp := p.GetPackage().HashFingerprint()
-	if buildertaggedImage == "" {
-		buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder"
-	}
-	if packageImage == "" {
-		packageImage = cs.ImageRepository + "-" + fp
-	}
-
-	Info(pkgTag, "Generating :whale: definition for builder image from", image)
-
 	// First we create the builder image
-	p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile"))
-	builderOpts := CompilerBackendOptions{
+	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")); err != nil {
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
+	}
+
+	if len(p.GetPreBuildSteps()) == 0 {
+		buildertaggedImage = image
+	}
+
+	// Then we write the step image, which uses the builder one
+	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")); err != nil {
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
+	}
+
+	builderOpts = CompilerBackendOptions{
 		ImageName:      buildertaggedImage,
 		SourcePath:     buildDir,
 		DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
+		BackendArgs:    cs.BackedArgs,
 	}
-
-	buildBuilderImage := true
-	if cs.Options.PullFirst {
-		if err := cs.Backend.DownloadImage(builderOpts); err == nil {
-			buildBuilderImage = false
-		}
-	}
-
-	if buildBuilderImage {
-		if err = cs.Backend.BuildImage(builderOpts); err != nil {
-			return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
-		}
-	}
-
-	if err = cs.Backend.ExportImage(builderOpts); err != nil {
-		return nil, errors.Wrap(err, "Could not export image")
-	}
-
-	if !cs.Options.KeepImageExport {
-		defer os.Remove(builderOpts.Destination)
-	}
-
-	if cs.Options.Push && buildBuilderImage {
-		if err = cs.Backend.Push(builderOpts); err != nil {
-			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
-		}
-	}
-	// Then we write the step image, which uses the builder one
-	p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
-	runnerOpts := CompilerBackendOptions{
+	runnerOpts = CompilerBackendOptions{
 		ImageName:      packageImage,
 		SourcePath:     buildDir,
 		DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
+		BackendArgs:    cs.BackedArgs,
 	}
 
-	// if !keepPackageImg {
-	// 	err = cs.Backend.ImageDefinitionToTar(runnerOpts)
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "Could not export image to tar")
-	// 	}
-	// } else {
-	buildPackageImage := true
-	if cs.Options.PullFirst {
-		//Best effort pull
-		if err := cs.Backend.DownloadImage(runnerOpts); err == nil {
-			buildPackageImage = false
+	buildAndPush := func(opts CompilerBackendOptions) error {
+		buildImage := true
+		if cs.Options.PullFirst {
+			bus.Manager.Publish(bus.EventImagePrePull, opts)
+			err := cs.Backend.DownloadImage(opts)
+			if err == nil {
+				buildImage = false
+			} else {
+				Warning("Failed to download '" + opts.ImageName + "'. Will keep going and build the image unless you use --fatal")
+				Warning(err.Error())
+			}
+			bus.Manager.Publish(bus.EventImagePostPull, opts)
+		}
+		if buildImage {
+			bus.Manager.Publish(bus.EventImagePreBuild, opts)
+			if err := cs.Backend.BuildImage(opts); err != nil {
+				return errors.Wrap(err, "Could not build image: "+image+" "+opts.DockerFileName)
+			}
+			bus.Manager.Publish(bus.EventImagePostBuild, opts)
+			if cs.Options.Push {
+				bus.Manager.Publish(bus.EventImagePrePush, opts)
+				if err = cs.Backend.Push(opts); err != nil {
+					return errors.Wrap(err, "Could not push image: "+image+" "+opts.DockerFileName)
+				}
+				bus.Manager.Publish(bus.EventImagePostPush, opts)
+			}
+		}
+		return nil
+	}
+	if len(p.GetPreBuildSteps()) != 0 {
+		Info(pkgTag, ":whale: Generating 'builder' image from", image, "as", buildertaggedImage, "with prelude steps")
+		if err := buildAndPush(builderOpts); err != nil {
+			return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
 		}
 	}
 
-	if buildPackageImage {
-		if err := cs.Backend.BuildImage(runnerOpts); err != nil {
-			return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
-		}
+	// Even if we might not have any steps to build, we do that so we can tag the image used in this moment and use that to cache it in a registry, or in the system.
+	// acting as a docker tag.
+	Info(pkgTag, ":whale: Generating 'package' image from", buildertaggedImage, "as", packageImage, "with build steps")
+	if err := buildAndPush(runnerOpts); err != nil {
+		return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
 	}
 
-	if err := cs.Backend.ExportImage(runnerOpts); err != nil {
-		return nil, errors.Wrap(err, "Failed exporting image")
-	}
+	return builderOpts, runnerOpts, nil
+}
 
-	if !cs.Options.KeepImageExport {
-		defer os.Remove(runnerOpts.Destination)
-	}
+func (cs *LuetCompiler) genArtifact(p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions, concurrency int, keepPermissions bool) (Artifact, error) {
 
-	if cs.Options.Push && buildPackageImage {
-		err = cs.Backend.Push(runnerOpts)
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
-		}
-	}
-	//	}
-
-	var diffs []ArtifactLayer
+	// generate Artifact
 	var artifact Artifact
+	var rootfs string
+	var err error
+	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
 
-	if !p.ImageUnpack() {
-		// we have to get diffs only if spec is not unpacked
-		diffs, err = cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
+	// We can't generate delta in this case. It implies the package is a virtual, and nothing has to be done really
+	if p.EmptyPackage() {
+		fakePackage := p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")
+
+		rootfs, err = ioutil.TempDir(p.GetOutputPath(), "rootfs")
 		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate changes from layers")
+			return nil, errors.Wrap(err, "Could not create tempdir")
 		}
-	}
+		defer os.RemoveAll(rootfs) // clean up
 
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not create tempdir")
-	}
-	defer os.RemoveAll(rootfs) // clean up
-
-	// TODO: Compression and such
-	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
-		ImageName:  packageImage,
-		SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not extract rootfs")
-	}
-
-	if !keepImg {
-		// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
-		// TODO: Handle caching and optionally do not remove things
-		err = cs.Backend.RemoveImage(builderOpts)
-		if err != nil {
-			Warning("Could not remove image ", builderOpts.ImageName)
-			//	return nil, errors.Wrap(err, "Could not remove image")
-		}
-		err = cs.Backend.RemoveImage(runnerOpts)
-		if err != nil {
-			Warning("Could not remove image ", builderOpts.ImageName)
-			//	return nil, errors.Wrap(err, "Could not remove image")
-		}
-	}
-
-	if p.ImageUnpack() {
-
-		if len(p.GetIncludes()) > 0 {
-			// strip from includes
-			cs.stripIncludesFromRootfs(p.GetIncludes(), rootfs)
-		}
-		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
+		artifact := NewPackageArtifact(fakePackage)
 		artifact.SetCompressionType(cs.CompressionType)
-		err = artifact.Compress(rootfs, concurrency)
-		if err != nil {
+
+		if err := artifact.Compress(rootfs, concurrency); err != nil {
 			return nil, errors.Wrap(err, "Error met while creating package archive")
 		}
 
 		artifact.SetCompileSpec(p)
-	} else {
-		Info(pkgTag, "Generating delta")
+		artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
 
-		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
+		err = artifact.WriteYaml(p.GetOutputPath())
 		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate deltas")
+			return artifact, errors.Wrap(err, "Failed while writing metadata file")
 		}
+		Info(pkgTag, "   :white_check_mark: done (empty virtual package)")
+		return artifact, nil
+	}
 
-		artifact.SetCompileSpec(p)
+	if p.UnpackedPackage() {
+		// Take content of container as a base for our package files
+		artifact, err = cs.unpackFs(concurrency, keepPermissions, p, runnerOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while extracting image")
+		}
+	} else {
+		// Generate delta between the two images
+		artifact, err = cs.unpackDelta(concurrency, keepPermissions, p, builderOpts, runnerOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while generating delta")
+		}
 	}
 
 	filelist, err := artifact.FileList()
@@ -426,6 +496,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	}
 
 	artifact.SetFiles(filelist)
+	artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
 
 	err = artifact.WriteYaml(p.GetOutputPath())
 	if err != nil {
@@ -436,9 +507,116 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	return artifact, nil
 }
 
+func (cs *LuetCompiler) waitForImage(image string) {
+	if cs.Options.PullFirst && cs.Options.Wait && !cs.Backend.ImageAvailable(image) {
+		Info(fmt.Sprintf("Waiting for image %s to be available... :zzz:", image))
+		Spinner(22)
+		defer SpinnerStop()
+		for !cs.Backend.ImageAvailable(image) {
+			Info(fmt.Sprintf("Image %s not available yet, sleeping", image))
+			time.Sleep(5 * time.Second)
+		}
+	}
+}
+
+func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string,
+	concurrency int,
+	keepPermissions, keepImg bool,
+	p CompilationSpec, generateArtifact bool) (Artifact, error) {
+
+	// If it is a virtual, check if we have to generate an empty artifact or not.
+	if generateArtifact && p.IsVirtual() {
+		return cs.genArtifact(p, CompilerBackendOptions{}, CompilerBackendOptions{}, concurrency, keepPermissions)
+	} else if p.IsVirtual() {
+		return &PackageArtifact{}, nil
+	}
+
+	if !generateArtifact {
+		exists := cs.Backend.ImageExists(packageImage)
+		if art, err := LoadArtifactFromYaml(p); err == nil && exists { // If YAML is correctly loaded, and both images exists, no reason to rebuild.
+			Debug("Artifact reloaded from YAML. Skipping build")
+			return art, err
+		}
+		cs.waitForImage(packageImage)
+		if cs.Options.PullFirst && cs.Backend.ImageAvailable(packageImage) {
+			return &PackageArtifact{}, nil
+		}
+	}
+
+	builderOpts, runnerOpts, err := cs.buildPackageImage(image, buildertaggedImage, packageImage, concurrency, keepPermissions, p)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed building package image")
+	}
+
+	if !keepImg {
+		defer func() {
+			// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
+			if err := cs.Backend.RemoveImage(builderOpts); err != nil {
+				Warning("Could not remove image ", builderOpts.ImageName)
+			}
+			if err := cs.Backend.RemoveImage(runnerOpts); err != nil {
+				Warning("Could not remove image ", runnerOpts.ImageName)
+			}
+		}()
+	}
+
+	if !generateArtifact {
+		return &PackageArtifact{}, nil
+	}
+
+	return cs.genArtifact(p, builderOpts, runnerOpts, concurrency, keepPermissions)
+}
+
+func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) {
+	compilerSpecs := NewLuetCompilationspecs()
+
+	w := db.World()
+
+	for _, p := range w {
+		spec, err := cs.FromPackage(p)
+		if err != nil {
+			return nil, err
+		}
+		if dst != "" {
+			spec.SetOutputPath(dst)
+		}
+		compilerSpecs.Add(spec)
+	}
+
+	switch minimum {
+	case true:
+		return cs.ComputeMinimumCompilableSet(compilerSpecs.Unique().All()...)
+	default:
+		return compilerSpecs.Unique().All(), nil
+	}
+}
+
+// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
+func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error) {
+	// Generate a set with all the deps of the provided specs
+	// we will use that set to remove the deps from the list of provided compilation specs
+	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
+	result := []CompilationSpec{}
+	for _, spec := range p {
+		ass, err := cs.ComputeDepTree(spec)
+		if err != nil {
+			return result, errors.Wrap(err, "computin specs deptree")
+		}
+
+		allDependencies = append(allDependencies, ass.Drop(spec.GetPackage())...)
+	}
+
+	for _, spec := range p {
+		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
+			result = append(result, spec)
+		}
+	}
+	return result, nil
+}
+
 func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
 
-	s := solver.NewResolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
+	s := solver.NewResolver(cs.SolverOptions, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
 
 	solution, err := s.Install(pkg.Packages{p.GetPackage()})
 	if err != nil {
@@ -454,9 +632,8 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
 	for _, assertion := range dependencies { //highly dependent on the order
 		if assertion.Value {
 			nthsolution := dependencies.Cut(assertion.Package)
-
 			assertion.Hash = solver.PackageHash{
-				BuildHash:   nthsolution.Drop(assertion.Package).AssertionHash(),
+				BuildHash:   nthsolution.HashFrom(assertion.Package),
 				PackageHash: nthsolution.AssertionHash(),
 			}
 			assertions = append(assertions, assertion)
@@ -479,18 +656,30 @@ func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifa
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
 	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
-	if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" {
-		Error("Package with no deps and no seed image supplied, bailing out")
-		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() + "with no deps and no seed image supplied, bailing out")
+	Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
+	if !p.HasImageSource() && !p.EmptyPackage() {
+		return nil,
+			fmt.Errorf(
+				"%s is invalid: package has no dependencies and no seed image supplied while it has steps defined",
+				p.GetPackage().GetFingerPrint(),
+			)
 	}
 
 	targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
 	targetPackageHash := cs.ImageRepository + ":" + targetAssertion.Hash.PackageHash
 
+	bus.Manager.Publish(bus.EventPackagePreBuild, struct {
+		CompileSpec CompilationSpec
+		Assert      solver.PackageAssert
+	}{
+		CompileSpec: p,
+		Assert:      *targetAssertion,
+	})
+
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p)
+		return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@@ -504,6 +693,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	depsN := 0
 	currentN := 0
 
+	packageDeps := !cs.Options.PackageTargetOnly
 	if !cs.Options.NoDeps {
 		Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
 		for _, assertion := range dependencies { //highly dependent on the order
@@ -513,8 +703,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 
 		for _, assertion := range dependencies { //highly dependent on the order
 			currentN++
-			pkgTag := fmt.Sprintf(":package:  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
-			Info(pkgTag, "   :zap:  Building dependency")
+			pkgTag := fmt.Sprintf(":package: %d/%d %s ⤑ :hammer: build %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
+			Info(pkgTag, " starts")
 			compileSpec, err := cs.FromPackage(assertion.Package)
 			if err != nil {
 				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
@@ -526,10 +716,18 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from", buildImageHash)
 			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
 
+			bus.Manager.Publish(bus.EventPackagePreBuild, struct {
+				CompileSpec CompilationSpec
+				Assert      solver.PackageAssert
+			}{
+				CompileSpec: compileSpec,
+				Assert:      assertion,
+			})
+
 			lastHash = currentPackageImageHash
 			if compileSpec.GetImage() != "" {
 				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
-				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 				if err != nil {
 					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				}
@@ -539,14 +737,23 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 			}
 
 			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
-			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 			if err != nil {
 				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				//	deperrs = append(deperrs, err)
 				//		break // stop at first error
 			}
+
+			bus.Manager.Publish(bus.EventPackagePostBuild, struct {
+				CompileSpec CompilationSpec
+				Artifact    Artifact
+			}{
+				CompileSpec: compileSpec,
+				Artifact:    artifact,
+			})
+
 			departifacts = append(departifacts, artifact)
-			Info(pkgTag, ":collision: Done")
+			Info(pkgTag, ":white_check_mark: Done")
 		}
 
 	} else if len(dependencies) > 0 {
@@ -554,20 +761,31 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	}
 
 	if !cs.Options.OnlyDeps {
-		Info(":package:", p.GetPackage().HumanReadableString(), ":cyclone:  Building package target from:", lastHash)
-		artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p)
+		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
+		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", lastHash)
+		artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 		if err != nil {
 			return artifact, err
 		}
 		artifact.SetDependencies(departifacts)
 		artifact.SetSourceAssertion(p.GetSourceAssertion())
 
+		bus.Manager.Publish(bus.EventPackagePostBuild, struct {
+			CompileSpec CompilationSpec
+			Artifact    Artifact
+		}{
+			CompileSpec: p,
+			Artifact:    artifact,
+		})
+
 		return artifact, err
 	} else {
 		return departifacts[len(departifacts)-1], nil
 	}
 }
 
+type templatedata map[string]interface{}
+
 func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 
 	pack, err := cs.Database.FindPackageCandidate(p)
@@ -575,16 +793,51 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 		return nil, err
 	}
 
-	buildFile := pack.Rel(BuildFile)
-	if !helpers.Exists(buildFile) {
-		return nil, errors.New("No build file present for " + p.GetFingerPrint())
+	var dataresult []byte
+
+	val := pack.Rel(DefinitionFile)
+	if _, err := os.Stat(pack.Rel(CollectionFile)); err == nil {
+		val = pack.Rel(CollectionFile)
+
+		data, err := ioutil.ReadFile(val)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+val)
+		}
+
+		dataBuild, err := ioutil.ReadFile(pack.Rel(BuildFile))
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+val)
+		}
+		packsRaw, err := pkg.GetRawPackages(data)
+
+		raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
+
+		d := map[string]interface{}{}
+		if len(cs.Options.BuildValuesFile) > 0 {
+			defBuild, err := ioutil.ReadFile(cs.Options.BuildValuesFile)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+val)
+			}
+			err = yaml.Unmarshal(defBuild, &d)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+val)
+			}
+		}
+
+		dat, err := helpers.RenderHelm(string(dataBuild), raw, d)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
+		}
+		dataresult = []byte(dat)
+	} else {
+		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, cs.Options.BuildValuesFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
+		}
+		dataresult = []byte(out)
 	}
 
-	dat, err := ioutil.ReadFile(buildFile)
-	if err != nil {
-		return nil, err
-	}
-	return NewLuetCompilationSpec(dat, pack)
+	return NewLuetCompilationSpec(dataresult, pack)
 }
 
 func (cs *LuetCompiler) GetBackend() CompilerBackend {

pkg/compiler/compiler_test.go

@@ -23,6 +23,7 @@ import (
 	sd "github.com/mudler/luet/pkg/compiler/backend"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -38,7 +39,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -49,8 +50,8 @@ var _ = Describe("Compiler", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			compiler.SetConcurrency(2)
@@ -82,7 +83,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -108,6 +109,27 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Templated packages", func() {
+		It("Renders", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/templates")
+			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+			pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec, err := compiler.FromPackage(pkg)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetImage()).To(Equal("b:bar"))
+		})
+	})
+
 	Context("Reconstruct image tree", func() {
 		It("Compiles it", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -120,7 +142,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -177,7 +199,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -219,7 +241,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -243,6 +265,146 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
+		It("Compiles and excludes files", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludes")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles includes and excludes files", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludesincludes")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+		})
+
+		It("Compiles and excludes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles includes and excludes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludeincludeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
 		It("Compiles and includes ony wanted files also from unpacked packages", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
@@ -254,7 +416,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -292,7 +454,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -333,7 +495,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -377,7 +539,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -419,7 +581,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -457,7 +619,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
 			Expect(err).ToNot(HaveOccurred())
@@ -496,7 +658,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
@@ -548,7 +710,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -591,7 +753,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -615,6 +777,61 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Packages which conents are a package folder", func() {
+		It("Compiles it in parallel", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/package_dir")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{
+				Name:     "dironly",
+				Category: "test",
+				Version:  "1.0",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{
+				Name:     "dironly_filter",
+				Category: "test",
+				Version:  "1.0",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			tmpdir2, err := ioutil.TempDir("", "tree2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir2) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir2)
+
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(2))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(0))
+
+			Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test2"))).To(BeTrue())
+
+			Expect(helpers.Untar(spec2.Rel("dironly_filter-test-1.0.package.tar"), tmpdir2, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec2.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
+		})
+	})
+
 	Context("Compression", func() {
 		It("Builds packages in gzip", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -624,7 +841,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -651,6 +868,25 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Compilation of whole tree", func() {
+		It("doesn't include dependencies that would be compiled anyway", func() {
+			// As some specs are dependent from each other, don't pull it in if they would
+			// be eventually
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/includeimage")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(specs)).To(Equal(1))
+
+			Expect(specs[0].GetPackage().GetFingerPrint()).To(Equal("b-test-1.0"))
+		})
+	})
+
 	Context("File list", func() {
 		It("is generated after the compilation process and annotated in the metadata", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -660,7 +896,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())

pkg/compiler/interface.go

@@ -28,11 +28,14 @@ type Compiler interface {
 	CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
+	ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error)
 	SetConcurrency(i int)
 	FromPackage(pkg.Package) (CompilationSpec, error)
-
+	FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error)
 	SetBackend(CompilerBackend)
 	GetBackend() CompilerBackend
+
+	SetBackendArgs([]string)
 	SetCompressionType(t CompressionImplementation)
 }
 
@@ -41,6 +44,8 @@ type CompilerBackendOptions struct {
 	SourcePath     string
 	DockerFileName string
 	Destination    string
+	Context        string
+	BackendArgs    []string
 }
 
 type CompilerOptions struct {
@@ -48,12 +53,14 @@ type CompilerOptions struct {
 	PullFirst, KeepImg, Push bool
 	Concurrency              int
 	CompressionType          CompressionImplementation
-	Clean                    bool
-	KeepImageExport          bool
 
-	OnlyDeps      bool
-	NoDeps        bool
-	SolverOptions config.LuetSolverOptions
+	Wait            bool
+	OnlyDeps        bool
+	NoDeps          bool
+	SolverOptions   config.LuetSolverOptions
+	BuildValuesFile string
+
+	PackageTargetOnly bool
 }
 
 func NewDefaultCompilerOptions() *CompilerOptions {
@@ -64,7 +71,6 @@ func NewDefaultCompilerOptions() *CompilerOptions {
 		CompressionType: None,
 		KeepImg:         true,
 		Concurrency:     runtime.NumCPU(),
-		Clean:           true,
 		OnlyDeps:        false,
 		NoDeps:          false,
 	}
@@ -74,7 +80,7 @@ type CompilerBackend interface {
 	BuildImage(CompilerBackendOptions) error
 	ExportImage(CompilerBackendOptions) error
 	RemoveImage(CompilerBackendOptions) error
-	Changes(fromImage, toImage string) ([]ArtifactLayer, error)
+	Changes(fromImage, toImage CompilerBackendOptions) ([]ArtifactLayer, error)
 	ImageDefinitionToTar(CompilerBackendOptions) error
 	ExtractRootfs(opts CompilerBackendOptions, keepPerms bool) error
 
@@ -82,6 +88,9 @@ type CompilerBackend interface {
 	DownloadImage(opts CompilerBackendOptions) error
 
 	Push(opts CompilerBackendOptions) error
+	ImageAvailable(string) bool
+
+	ImageExists(string) bool
 }
 
 type Artifact interface {
@@ -104,9 +113,13 @@ type Artifact interface {
 
 	SetFiles(f []string)
 	GetFiles() []string
+	GetFileName() string
 
 	GetChecksums() Checksums
 	SetChecksums(c Checksums)
+
+	GenerateFinalImage(string, CompilerBackend, bool) (CompilerBackendOptions, error)
+	GetUncompressedName() string
 }
 
 type ArtifactNode struct {
@@ -141,6 +154,7 @@ type ArtifactLayersSummary struct {
 type CompilationSpec interface {
 	ImageUnpack() bool // tells if the definition is just an image
 	GetIncludes() []string
+	GetExcludes() []string
 
 	RenderBuildImage() (string, error)
 	WriteBuildImageDefinition(string) error
@@ -168,6 +182,14 @@ type CompilationSpec interface {
 
 	GetRetrieve() []string
 	CopyRetrieves(dest string) error
+
+	SetPackageDir(string)
+	GetPackageDir() string
+
+	EmptyPackage() bool
+	UnpackedPackage() bool
+	HasImageSource() bool
+	IsVirtual() bool
 }
 
 type CompilationSpecs interface {

pkg/compiler/spec.go

@@ -95,12 +95,14 @@ type LuetCompilationSpec struct {
 	Seed            string                    `json:"seed"`
 	Package         *pkg.DefaultPackage       `json:"package"`
 	SourceAssertion solver.PackagesAssertions `json:"-"`
+	PackageDir      string                    `json:"package_dir" yaml:"package_dir"`
 
 	Retrieve []string `json:"retrieve"`
 
 	OutputPath string   `json:"-"` // Where the build processfiles go
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
+	Excludes   []string `json:"excludes"`
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
@@ -112,17 +114,25 @@ func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
 	spec.Package = p.(*pkg.DefaultPackage)
 	return &spec, nil
 }
-func (a *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
-	return a.SourceAssertion
+func (cs *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
+	return cs.SourceAssertion
 }
 
-func (a *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) {
-	a.SourceAssertion = as
+func (cs *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) {
+	cs.SourceAssertion = as
 }
 func (cs *LuetCompilationSpec) GetPackage() pkg.Package {
 	return cs.Package
 }
 
+func (cs *LuetCompilationSpec) GetPackageDir() string {
+	return cs.PackageDir
+}
+
+func (cs *LuetCompilationSpec) SetPackageDir(s string) {
+	cs.PackageDir = s
+}
+
 func (cs *LuetCompilationSpec) BuildSteps() []string {
 	return cs.Steps
 }
@@ -139,10 +149,20 @@ func (cs *LuetCompilationSpec) GetIncludes() []string {
 	return cs.Includes
 }
 
+func (cs *LuetCompilationSpec) GetExcludes() []string {
+	return cs.Excludes
+}
+
 func (cs *LuetCompilationSpec) GetRetrieve() []string {
 	return cs.Retrieve
 }
 
+// IsVirtual returns true if the spec is virtual.
+// A spec is virtual if the package is empty, and it has no image source to unpack from.
+func (cs *LuetCompilationSpec) IsVirtual() bool {
+	return cs.EmptyPackage() && !cs.HasImageSource()
+}
+
 func (cs *LuetCompilationSpec) GetSeedImage() string {
 	return cs.Seed
 }
@@ -171,6 +191,27 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 	cs.Seed = s
 }
 
+func (cs *LuetCompilationSpec) EmptyPackage() bool {
+	return len(cs.BuildSteps()) == 0 && len(cs.GetPreBuildSteps()) == 0 && !cs.UnpackedPackage()
+}
+
+func (cs *LuetCompilationSpec) UnpackedPackage() bool {
+	// If package_dir was specified in the spec, we want to treat the content of the directory
+	// as the root of our archive.  ImageUnpack is implied to be true. override it
+	unpack := cs.ImageUnpack()
+	if cs.GetPackageDir() != "" {
+		unpack = true
+	}
+	return unpack
+}
+
+// HasImageSource returns true when the compilation spec has an image source.
+// a compilation spec has an image source when it depends on other packages or have a source image
+// explictly supplied
+func (cs *LuetCompilationSpec) HasImageSource() bool {
+	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != ""
+}
+
 func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
 	var err error
 	if len(cs.Retrieve) > 0 {
@@ -189,10 +230,9 @@ func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
 	return err
 }
 
-// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
-func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
+func (cs *LuetCompilationSpec) genDockerfile(image string, steps []string) string {
 	spec := `
-FROM ` + cs.GetSeedImage() + `
+FROM ` + image + `
 COPY . /luetbuild
 WORKDIR /luetbuild
 ENV PACKAGE_NAME=` + cs.Package.GetName() + `
@@ -217,31 +257,22 @@ ADD ` + s + ` /luetbuild/`
 ENV ` + s
 	}
 
-	for _, s := range cs.GetPreBuildSteps() {
+	for _, s := range steps {
 		spec = spec + `
 RUN ` + s
 	}
-	return spec, nil
+	return spec
 }
 
-// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
+// RenderBuildImage renders the dockerfile of the image used as a pre-build step
+func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
+	return cs.genDockerfile(cs.GetSeedImage(), cs.GetPreBuildSteps()), nil
+
+}
+
+// RenderStepImage renders the dockerfile used for the image used for building the package
 func (cs *LuetCompilationSpec) RenderStepImage(image string) (string, error) {
-	spec := `
-FROM ` + image + `
-ENV PACKAGE_NAME=` + cs.Package.GetName() + `
-ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
-ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
-
-	for _, s := range cs.Env {
-		spec = spec + `
-ENV ` + s
-	}
-	for _, s := range cs.BuildSteps() {
-		spec = spec + `
-RUN ` + s
-	}
-
-	return spec, nil
+	return cs.genDockerfile(image, cs.BuildSteps()), nil
 }
 
 func (cs *LuetCompilationSpec) WriteBuildImageDefinition(path string) error {

pkg/compiler/spec_test.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -50,6 +51,26 @@ var _ = Describe("Spec", func() {
 			Expect(newSpec2.All()).To(Equal([]CompilationSpec{testSpec3}))
 
 		})
+		Context("virtuals", func() {
+			When("is empty", func() {
+				It("is virtual", func() {
+					spec := &LuetCompilationSpec{}
+					Expect(spec.IsVirtual()).To(BeTrue())
+				})
+			})
+			When("has defined steps", func() {
+				It("is not a virtual", func() {
+					spec := &LuetCompilationSpec{Steps: []string{"foo"}}
+					Expect(spec.IsVirtual()).To(BeFalse())
+				})
+			})
+			When("has defined image", func() {
+				It("is not a virtual", func() {
+					spec := &LuetCompilationSpec{Image: "foo"}
+					Expect(spec.IsVirtual()).To(BeFalse())
+				})
+			})
+		})
 	})
 
 	Context("Simple package build definition", func() {
@@ -61,7 +82,7 @@ var _ = Describe("Spec", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -95,6 +116,8 @@ ENV test=1`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
@@ -114,7 +137,7 @@ RUN echo bar > /test2`))
 
 		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 		spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
 		Expect(err).ToNot(HaveOccurred())
 
@@ -167,9 +190,13 @@ ENV test=1`))
 
 		Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=a
 ENV PACKAGE_VERSION=1.0
 ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
 ENV test=1
 RUN echo foo > /test
 RUN echo bar > /test2`))

pkg/config/config.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/mudler/luet/pkg/helpers"
+	pkg "github.com/mudler/luet/pkg/package"
 	solver "github.com/mudler/luet/pkg/solver"
 
 	v "github.com/spf13/viper"
@@ -37,9 +38,20 @@ var LuetCfg = NewLuetConfig(v.GetViper())
 var AvailableResolvers = strings.Join([]string{solver.QLearningResolverType}, " ")
 
 type LuetLoggingConfig struct {
-	Path       string `mapstructure:"path"`
-	JsonFormat bool   `mapstructure:"json_format"`
-	Level      string `mapstructure:"level"`
+	// Path of the logfile
+	Path string `mapstructure:"path"`
+	// Enable/Disable logging to file
+	EnableLogFile bool `mapstructure:"enable_logfile"`
+	// Enable JSON format logging in file
+	JsonFormat bool `mapstructure:"json_format"`
+
+	// Log level
+	Level string `mapstructure:"level"`
+
+	// Enable emoji
+	EnableEmoji bool `mapstructure:"enable_emoji"`
+	// Enable/Disable color in logging
+	Color bool `mapstructure:"color"`
 }
 
 type LuetGeneralConfig struct {
@@ -53,10 +65,20 @@ type LuetGeneralConfig struct {
 }
 
 type LuetSolverOptions struct {
-	Type        string  `mapstructure:"type"`
-	LearnRate   float32 `mapstructure:"rate"`
-	Discount    float32 `mapstructure:"discount"`
-	MaxAttempts int     `mapstructure:"max_attempts"`
+	Type           string            `mapstructure:"type"`
+	LearnRate      float32           `mapstructure:"rate"`
+	Discount       float32           `mapstructure:"discount"`
+	MaxAttempts    int               `mapstructure:"max_attempts"`
+	Implementation solver.SolverType `mapstructure:"implementation"`
+}
+
+func (opts LuetSolverOptions) ResolverIsSet() bool {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		return true
+	default:
+		return false
+	}
 }
 
 func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
@@ -85,7 +107,7 @@ type LuetSystemConfig struct {
 	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
 }
 
-func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
 	dbpath = filepath.Join(dbpath, "repos/"+name)
 	err := os.MkdirAll(dbpath, os.ModePerm)
@@ -95,7 +117,7 @@ func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	return dbpath
 }
 
-func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+func (sc *LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
 	dbpath := filepath.Join(sc.Rootfs,
 		sc.DatabasePath)
 	err := os.MkdirAll(dbpath, os.ModePerm)
@@ -105,7 +127,7 @@ func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
 	return dbpath
 }
 
-func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
 	var cachepath string
 	if sc.PkgsCachePath != "" {
 		cachepath = sc.PkgsCachePath
@@ -123,6 +145,10 @@ func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
 	return
 }
 
+func (sc *LuetSystemConfig) GetRootFsAbs() (string, error) {
+	return filepath.Abs(sc.Rootfs)
+}
+
 type LuetRepository struct {
 	Name           string            `json:"name" yaml:"name" mapstructure:"name"`
 	Description    string            `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
@@ -189,9 +215,14 @@ type LuetConfig struct {
 	System  LuetSystemConfig  `mapstructure:"system"`
 	Solver  LuetSolverOptions `mapstructure:"solver"`
 
-	RepositoriesConfDir []string         `mapstructure:"repos_confdir"`
-	CacheRepositories   []LuetRepository `mapstructure:"repetitors"`
-	SystemRepositories  []LuetRepository `mapstructure:"repositories"`
+	RepositoriesConfDir  []string         `mapstructure:"repos_confdir"`
+	ConfigProtectConfDir []string         `mapstructure:"config_protect_confdir"`
+	ConfigProtectSkip    bool             `mapstructure:"config_protect_skip"`
+	ConfigFromHost       bool             `mapstructure:"config_from_host"`
+	CacheRepositories    []LuetRepository `mapstructure:"repetitors"`
+	SystemRepositories   []LuetRepository `mapstructure:"repositories"`
+
+	ConfigProtectConfFiles []ConfigProtectConfFile
 }
 
 func NewLuetConfig(viper *v.Viper) *LuetConfig {
@@ -200,13 +231,16 @@ func NewLuetConfig(viper *v.Viper) *LuetConfig {
 	}
 
 	GenDefault(viper)
-	return &LuetConfig{Viper: viper}
+	return &LuetConfig{Viper: viper, ConfigProtectConfFiles: nil}
 }
 
 func GenDefault(viper *v.Viper) {
 	viper.SetDefault("logging.level", "info")
-	viper.SetDefault("logging.path", "")
+	viper.SetDefault("logging.enable_logfile", false)
+	viper.SetDefault("logging.path", "/var/log/luet.log")
 	viper.SetDefault("logging.json_format", false)
+	viper.SetDefault("logging.enable_emoji", true)
+	viper.SetDefault("logging.color", true)
 
 	viper.SetDefault("general.concurrency", runtime.NumCPU())
 	viper.SetDefault("general.debug", false)
@@ -217,7 +251,7 @@ func GenDefault(viper *v.Viper) {
 
 	u, err := user.Current()
 	// os/user doesn't work in from scratch environments
-	if err != nil || u.Uid == "0" {
+	if err != nil || (u != nil && u.Uid == "0") {
 		viper.SetDefault("general.same_owner", true)
 	} else {
 		viper.SetDefault("general.same_owner", false)
@@ -230,6 +264,10 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("system.pkgs_cache_path", "packages")
 
 	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
+	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
+	viper.SetDefault("config_protect_skip", false)
+	// TODO: Set default to false when we are ready for migration.
+	viper.SetDefault("config_from_host", true)
 	viper.SetDefault("cache_repositories", []string{})
 	viper.SetDefault("system_repositories", []string{})
 
@@ -239,6 +277,16 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("solver.max_attempts", 9000)
 }
 
+func (c *LuetConfig) GetSystemDB() pkg.PackageDatabase {
+	switch LuetCfg.GetSystem().DatabaseEngine {
+	case "boltdb":
+		return pkg.NewBoltDatabase(
+			filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+	default:
+		return pkg.NewInMemoryDatabase(true)
+	}
+}
+
 func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
 	c.SystemRepositories = append(c.SystemRepositories, r)
 }
@@ -259,6 +307,18 @@ func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
 	return &c.Solver
 }
 
+func (c *LuetConfig) GetConfigProtectConfFiles() []ConfigProtectConfFile {
+	return c.ConfigProtectConfFiles
+}
+
+func (c *LuetConfig) AddConfigProtectConfFile(file *ConfigProtectConfFile) {
+	if c.ConfigProtectConfFiles == nil {
+		c.ConfigProtectConfFiles = []ConfigProtectConfFile{*file}
+	} else {
+		c.ConfigProtectConfFiles = append(c.ConfigProtectConfFiles, *file)
+	}
+}
+
 func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
 	var ans *LuetRepository = nil
 
@@ -318,9 +378,13 @@ func (c *LuetLoggingConfig) SetLogLevel(s string) {
 func (c *LuetLoggingConfig) String() string {
 	ans := fmt.Sprintf(`
 logging:
+  enable_logfile: %t
   path: %s
   json_format: %t
-  level: %s`, c.Path, c.JsonFormat, c.Level)
+  color: %t
+  enable_emoji: %t
+  level: %s`, c.EnableLogFile, c.Path, c.JsonFormat,
+		c.Color, c.EnableEmoji, c.Level)
 
 	return ans
 }

pkg/config/config_protect.go

@@ -0,0 +1,119 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+type ConfigProtectConfFile struct {
+	Filename string
+
+	Name        string   `mapstructure:"name" yaml:"name" json:"name"`
+	Directories []string `mapstructure:"dirs" yaml:"dirs" json:"dirs"`
+}
+
+func NewConfigProtectConfFile(filename string) *ConfigProtectConfFile {
+	return &ConfigProtectConfFile{
+		Filename:    filename,
+		Name:        "",
+		Directories: []string{},
+	}
+}
+
+func (c *ConfigProtectConfFile) String() string {
+	return fmt.Sprintf("[%s] filename: %s, dirs: %s", c.Name, c.Filename,
+		c.Directories)
+}
+
+type ConfigProtect struct {
+	AnnotationDir string
+	MapProtected  map[string]bool
+}
+
+func NewConfigProtect(annotationDir string) *ConfigProtect {
+	if len(annotationDir) > 0 && annotationDir[0:1] != "/" {
+		annotationDir = "/" + annotationDir
+	}
+	return &ConfigProtect{
+		AnnotationDir: annotationDir,
+		MapProtected:  make(map[string]bool, 0),
+	}
+}
+
+func (c *ConfigProtect) AddAnnotationDir(d string) {
+	c.AnnotationDir = d
+}
+
+func (c *ConfigProtect) GetAnnotationDir() string {
+	return c.AnnotationDir
+}
+
+func (c *ConfigProtect) Map(files []string) {
+	if LuetCfg.ConfigProtectSkip {
+		return
+	}
+
+	for _, file := range files {
+
+		if file[0:1] != "/" {
+			file = "/" + file
+		}
+
+		if len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
+				for _, dir := range conf.Directories {
+					// Note file is without / at begin (on unpack)
+					if strings.HasPrefix(file, filepath.Clean(dir)) {
+						// docker archive modifier works with path without / at begin.
+						c.MapProtected[file] = true
+						goto nextFile
+					}
+				}
+			}
+		}
+
+		if c.AnnotationDir != "" && strings.HasPrefix(file, filepath.Clean(c.AnnotationDir)) {
+			c.MapProtected[file] = true
+		}
+	nextFile:
+	}
+
+}
+
+func (c *ConfigProtect) Protected(file string) bool {
+	if file[0:1] != "/" {
+		file = "/" + file
+	}
+	_, ans := c.MapProtected[file]
+	return ans
+}
+
+func (c *ConfigProtect) GetProtectFiles(withSlash bool) []string {
+	ans := []string{}
+
+	for key, _ := range c.MapProtected {
+		if withSlash {
+			ans = append(ans, key)
+		} else {
+			ans = append(ans, key[1:])
+		}
+	}
+	return ans
+}

pkg/config/config_protect_test.go

@@ -0,0 +1,118 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+
+	Context("Test config protect", func() {
+
+		It("Protect1", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("/etc")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+		It("Protect2", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{},
+			))
+		})
+
+		It("Protect3: Annotation dir without initial slash", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("etc")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+	})
+
+})

pkg/config/config_test.go

@@ -54,6 +54,13 @@ var _ = Describe("Config", func() {
 			defer os.Remove(tmpFile.Name())
 		})
 
+		It("Config1", func() {
+			cfg := config.LuetCfg
+
+			cfg.GetLogging().Color = false
+			Expect(cfg.GetLogging().Color).To(BeFalse())
+		})
+
 	})
 
 })

pkg/helpers/archive.go

@@ -17,6 +17,7 @@ package helpers
 
 import (
 	"archive/tar"
+	"bytes"
 	"io"
 	"os"
 	"path/filepath"
@@ -49,6 +50,146 @@ func Tar(src, dest string) error {
 	return err
 }
 
+type TarModifierWrapperFunc func(path, dst string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
+type TarModifierWrapper struct {
+	DestinationPath string
+	Modifier        TarModifierWrapperFunc
+}
+
+func NewTarModifierWrapper(dst string, modifier TarModifierWrapperFunc) *TarModifierWrapper {
+	return &TarModifierWrapper{
+		DestinationPath: dst,
+		Modifier:        modifier,
+	}
+}
+
+func (m *TarModifierWrapper) GetModifier() archive.TarModifierFunc {
+	return func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+		return m.Modifier(m.DestinationPath, path, header, content)
+	}
+}
+
+func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modifier *TarModifierWrapper) error {
+	var ans error
+
+	if len(protectedFiles) <= 0 {
+		return Untar(src, dst, sameOwner)
+	}
+
+	// POST: we have files to protect. I create a ReplaceFileTarWrapper
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	// Create modifier map
+	mods := make(map[string]archive.TarModifierFunc)
+	for _, file := range protectedFiles {
+		mods[file] = modifier.GetModifier()
+	}
+
+	if sameOwner {
+		// PRE: i have root privileged.
+
+		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
+
+		opts := &archive.TarOptions{
+			// NOTE: NoLchown boolean is used for chmod of the symlink
+			// Probably it's needed set this always to true.
+			NoLchown:        true,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+			ContinueOnError: true,
+		}
+
+		ans = archive.Untar(replacerArchive, dst, opts)
+	} else {
+		ans = unTarIgnoreOwner(dst, in, mods)
+	}
+
+	return ans
+}
+
+func unTarIgnoreOwner(dest string, in io.ReadCloser, mods map[string]archive.TarModifierFunc) error {
+	tr := tar.NewReader(in)
+	for {
+		header, err := tr.Next()
+
+		var data []byte
+		var headerReplaced = false
+
+		switch {
+		case err == io.EOF:
+			goto tarEof
+		case err != nil:
+			return err
+		case header == nil:
+			continue
+		}
+
+		// the target location where the dir/file should be created
+		target := filepath.Join(dest, header.Name)
+		if mods != nil {
+			modifier, ok := mods[header.Name]
+			if ok {
+				header, data, err = modifier(header.Name, header, tr)
+				if err != nil {
+					return err
+				}
+
+				// Override target path
+				target = filepath.Join(dest, header.Name)
+				headerReplaced = true
+			}
+
+		}
+
+		// Check the file type
+		switch header.Typeflag {
+
+		// if its a dir and it doesn't exist create it
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return err
+				}
+			}
+
+			// handle creation of file
+		case tar.TypeReg:
+
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+
+			// copy over contents
+			if headerReplaced {
+				_, err = io.Copy(f, bytes.NewReader(data))
+			} else {
+				_, err = io.Copy(f, tr)
+			}
+			if err != nil {
+				return err
+			}
+
+			// manually close here after each file operation; defering would cause each
+			// file close to wait until all operations have completed.
+			f.Close()
+
+		case tar.TypeSymlink:
+			source := header.Linkname
+			err := os.Symlink(source, target)
+			if err != nil {
+				return err
+			}
+		}
+	}
+tarEof:
+
+	return nil
+}
+
 // Untar just a wrapper around the docker functions
 func Untar(src, dest string, sameOwner bool) error {
 	var ans error
@@ -72,62 +213,7 @@ func Untar(src, dest string, sameOwner bool) error {
 
 		ans = archive.Untar(in, dest, opts)
 	} else {
-
-		// TODO: replace with https://github.com/mholt/archiver ?
-		var fileReader io.ReadCloser = in
-
-		tr := tar.NewReader(fileReader)
-		for {
-			header, err := tr.Next()
-
-			switch {
-			case err == io.EOF:
-				goto tarEof
-			case err != nil:
-				return err
-			case header == nil:
-				continue
-			}
-
-			// the target location where the dir/file should be created
-			target := filepath.Join(dest, header.Name)
-
-			// Check the file type
-			switch header.Typeflag {
-
-			// if its a dir and it doesn't exist create it
-			case tar.TypeDir:
-				if _, err := os.Stat(target); err != nil {
-					if err := os.MkdirAll(target, 0755); err != nil {
-						return err
-					}
-				}
-
-				// handle creation of file
-			case tar.TypeReg:
-				f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
-				if err != nil {
-					return err
-				}
-
-				// copy over contents
-				if _, err := io.Copy(f, tr); err != nil {
-					return err
-				}
-
-				// manually close here after each file operation; defering would cause each
-				// file close to wait until all operations have completed.
-				f.Close()
-
-			case tar.TypeSymlink:
-				source := header.Linkname
-				err := os.Symlink(source, target)
-				if err != nil {
-					return err
-				}
-			}
-		}
-	tarEof:
+		ans = unTarIgnoreOwner(dest, in, nil)
 	}
 
 	return ans

pkg/helpers/archive_test.go

@@ -0,0 +1,134 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/archive"
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// Code from moby/moby pkg/archive/archive_test
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(filepath.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+		if makeLinks {
+			if err := os.Link(filepath.Join(targetPath, fileName), filepath.Join(targetPath, fileName+"-link")); err != nil {
+				return 0, err
+			}
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}
+
+func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	// If the destination path already exists I rename target file name with postfix.
+	var basePath string
+
+	// Read data. TODO: We need change archive callback to permit to return a Reader
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	if header != nil {
+
+		switch header.Typeflag {
+		case tar.TypeReg:
+			basePath = filepath.Base(path)
+		default:
+			// Nothing to do. I return original reader
+			return header, buffer.Bytes(), nil
+		}
+
+		if basePath == "file-0" {
+			name := filepath.Join(filepath.Join(filepath.Dir(path), fmt.Sprintf("._cfg%04d_%s", 1, basePath)))
+			return &tar.Header{
+				Mode:       header.Mode,
+				Typeflag:   header.Typeflag,
+				PAXRecords: header.PAXRecords,
+				Name:       name,
+			}, buffer.Bytes(), nil
+		} else if basePath == "file-1" {
+			return header, []byte("newcontent"), nil
+		}
+
+		// else file not present
+	}
+
+	return header, buffer.Bytes(), nil
+}
+
+var _ = Describe("Helpers Archive", func() {
+	Context("Untar Protect", func() {
+
+		It("Detect existing and not-existing files", func() {
+
+			archiveSourceDir, err := ioutil.TempDir("", "archive-source")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(archiveSourceDir)
+
+			_, err = prepareUntarSourceDirectory(10, archiveSourceDir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			targetDir, err := ioutil.TempDir("", "archive-target")
+			Expect(err).ToNot(HaveOccurred())
+			//	defer os.RemoveAll(targetDir)
+
+			sourceArchive, err := archive.TarWithOptions(archiveSourceDir, &archive.TarOptions{})
+			Expect(err).ToNot(HaveOccurred())
+			defer sourceArchive.Close()
+
+			tarModifier := NewTarModifierWrapper(targetDir, tarModifierWrapperFunc)
+			mods := make(map[string]archive.TarModifierFunc)
+			mods["file-0"] = tarModifier.GetModifier()
+			mods["file-1"] = tarModifier.GetModifier()
+			mods["file-9999"] = tarModifier.GetModifier()
+
+			replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
+			//replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
+			opts := &archive.TarOptions{
+				// NOTE: NoLchown boolean is used for chmod of the symlink
+				// Probably it's needed set this always to true.
+				NoLchown:        true,
+				ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+				ContinueOnError: true,
+			}
+
+			err = archive.Untar(replacerArchive, targetDir, opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
+		})
+	})
+})

pkg/helpers/file.go

@@ -16,14 +16,57 @@
 package helpers
 
 import (
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
+	"strings"
+	"syscall"
 	"time"
 
 	copy "github.com/otiai10/copy"
+	"github.com/pkg/errors"
 )
 
+func OrderFiles(target string, files []string) ([]string, []string) {
+
+	var newFiles []string
+	var notPresent []string
+
+	for _, f := range files {
+		target := filepath.Join(target, f)
+		fi, err := os.Lstat(target)
+		if err != nil {
+			notPresent = append(notPresent, f)
+			continue
+		}
+		if m := fi.Mode(); !m.IsDir() {
+			newFiles = append(newFiles, f)
+		}
+	}
+
+	dirs := []string{}
+
+	for _, f := range files {
+		target := filepath.Join(target, f)
+		fi, err := os.Lstat(target)
+		if err != nil {
+			continue
+		}
+		if m := fi.Mode(); m.IsDir() {
+			dirs = append(dirs, f)
+		}
+	}
+
+	// Compare how many sub paths there are, and push at the end the ones that have less subpaths
+	sort.Slice(dirs, func(i, j int) bool {
+		return len(strings.Split(dirs[i], string(os.PathSeparator))) > len(strings.Split(dirs[j], string(os.PathSeparator)))
+	})
+
+	return append(newFiles, dirs...), notPresent
+}
+
 func ListDir(dir string) ([]string, error) {
 	content := []string{}
 
@@ -40,6 +83,20 @@ func ListDir(dir string) ([]string, error) {
 	return content, err
 }
 
+// DirectoryIsEmpty Checks wether the directory is empty or not
+func DirectoryIsEmpty(dir string) (bool, error) {
+	f, err := os.Open(dir)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	if _, err = f.Readdirnames(1); err == io.EOF {
+		return true, nil
+	}
+	return false, nil
+}
+
 // Touch creates an empty file
 func Touch(f string) error {
 	_, err := os.Stat(f)
@@ -77,14 +134,15 @@ func Read(file string) (string, error) {
 	return string(dat), nil
 }
 
-func ensureDir(fileName string) {
+func EnsureDir(fileName string) error {
 	dirName := filepath.Dir(fileName)
 	if _, serr := os.Stat(dirName); serr != nil {
 		merr := os.MkdirAll(dirName, os.ModePerm) // FIXME: It should preserve permissions from src to dst instead
 		if merr != nil {
-			panic(merr)
+			return merr
 		}
 	}
+	return nil
 }
 
 // CopyFile copies the contents of the file named src to the file named
@@ -93,7 +151,30 @@ func ensureDir(fileName string) {
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
 func CopyFile(src, dst string) (err error) {
-	return copy.Copy(src, dst)
+	// Workaround for https://github.com/otiai10/copy/issues/47
+	fi, err := os.Lstat(src)
+	if err != nil {
+		return errors.Wrap(err, "error reading file info")
+	}
+
+	fm := fi.Mode()
+	switch {
+	case fm&os.ModeNamedPipe != 0:
+		EnsureDir(dst)
+		if err := syscall.Mkfifo(dst, uint32(fi.Mode())); err != nil {
+			return errors.Wrap(err, "failed creating pipe")
+		}
+		if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
+			if err := os.Chown(dst, int(stat.Uid), int(stat.Gid)); err != nil {
+				return errors.Wrap(err, "failed chowning file")
+			}
+		}
+		return nil
+	}
+
+	return copy.Copy(src, dst, copy.Options{
+		Sync:      true,
+		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }
 
 func IsDirectory(path string) (bool, error) {
@@ -110,5 +191,7 @@ func IsDirectory(path string) (bool, error) {
 func CopyDir(src string, dst string) (err error) {
 	src = filepath.Clean(src)
 	dst = filepath.Clean(dst)
-	return copy.Copy(src, dst)
+	return copy.Copy(src, dst, copy.Options{
+		Sync:      true,
+		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }

pkg/helpers/file_test.go

@@ -16,6 +16,10 @@
 package helpers_test
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -28,4 +32,72 @@ var _ = Describe("Helpers", func() {
 			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
 		})
 	})
+
+	Context("DirectoryIsEmpty", func() {
+		It("Detects empty directory", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+			Expect(DirectoryIsEmpty(testDir)).To(BeTrue())
+		})
+		It("Detects directory with files", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+			err = Touch(filepath.Join(testDir, "foo"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(DirectoryIsEmpty(testDir)).To(BeFalse())
+		})
+	})
+
+	Context("Orders dir and files correctly", func() {
+		It("puts files first and folders at end", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			err = ioutil.WriteFile(filepath.Join(testDir, "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(testDir, "baz"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "bar"), 0755)
+			Expect(err).ToNot(HaveOccurred())
+			err = ioutil.WriteFile(filepath.Join(testDir, "bar", "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "baz2"), 0755)
+			Expect(err).ToNot(HaveOccurred())
+			err = ioutil.WriteFile(filepath.Join(testDir, "baz2", "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			ordered, notExisting := OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
+
+			Expect(ordered).To(Equal([]string{"baz", "bar/foo", "foo", "baz2/foo", "bar", "baz2"}))
+			Expect(notExisting).To(Equal([]string{"notexisting"}))
+		})
+
+		It("orders correctly when there are folders with folders", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			err = os.MkdirAll(filepath.Join(testDir, "bar"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+			err = os.MkdirAll(filepath.Join(testDir, "foo"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "bar"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz", "fa"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			ordered, _ := OrderFiles(testDir, []string{"foo", "foo/bar", "bar", "foo/baz/fa", "foo/baz"})
+			Expect(ordered).To(Equal([]string{"foo/baz/fa", "foo/bar", "foo/baz", "foo", "bar"}))
+		})
+	})
 })

pkg/helpers/helm.go

@@ -0,0 +1,71 @@
+package helpers
+
+import (
+	"io/ioutil"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+)
+
+// RenderHelm renders the template string with helm
+func RenderHelm(template string, values, d map[string]interface{}) (string, error) {
+	c := &chart.Chart{
+		Metadata: &chart.Metadata{
+			Name:    "",
+			Version: "",
+		},
+		Templates: []*chart.File{
+			{Name: "templates", Data: []byte(template)},
+		},
+		Values: map[string]interface{}{"Values": values},
+	}
+
+	v, err := chartutil.CoalesceValues(c, map[string]interface{}{"Values": d})
+	if err != nil {
+		return "", errors.Wrap(err, "while rendering template")
+	}
+	out, err := engine.Render(c, v)
+	if err != nil {
+		return "", errors.Wrap(err, "while rendering template")
+	}
+
+	return out["templates"], nil
+}
+
+type templatedata map[string]interface{}
+
+func RenderFiles(toTemplate, valuesFile string, defaultFile string) (string, error) {
+	raw, err := ioutil.ReadFile(toTemplate)
+	if err != nil {
+		return "", errors.Wrap(err, "reading file "+toTemplate)
+	}
+
+	if !Exists(valuesFile) {
+		return "", errors.Wrap(err, "file not existing "+valuesFile)
+	}
+	val, err := ioutil.ReadFile(valuesFile)
+	if err != nil {
+		return "", errors.Wrap(err, "reading file "+valuesFile)
+	}
+
+	var values templatedata
+	d := templatedata{}
+	if len(defaultFile) > 0 {
+		def, err := ioutil.ReadFile(defaultFile)
+		if err != nil {
+			return "", errors.Wrap(err, "reading file "+valuesFile)
+		}
+		if err = yaml.Unmarshal(def, &d); err != nil {
+			return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
+		}
+	}
+
+	if err = yaml.Unmarshal(val, &values); err != nil {
+		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
+	}
+
+	return RenderHelm(string(raw), values, d)
+}

pkg/helpers/helm_test.go

@@ -0,0 +1,147 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func writeFile(path string, content string) {
+	err := ioutil.WriteFile(path, []byte(content), 0644)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+var _ = Describe("Helpers", func() {
+	Context("RenderHelm", func() {
+		It("Renders templates", func() {
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foobar"))
+		})
+		It("Renders templates with overrides", func() {
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foobar"))
+		})
+
+		It("Renders templates", func() {
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foobar"))
+		})
+
+		It("Render files default overrides", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+			writeFile(d, `
+foo: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("baz"))
+
+		})
+
+		It("Render files from values", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+			writeFile(d, `
+faa: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bar"))
+
+		})
+
+		It("Render files from values if no default", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, "")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bar"))
+		})
+
+		It("doesn't interpolate if no one provides the values", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foao: "bar"
+`)
+			writeFile(d, `
+faa: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal(""))
+
+		})
+	})
+})

pkg/helpers/match.go

@@ -0,0 +1,49 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"regexp"
+)
+
+func MapMatchRegex(m *map[string]string, r *regexp.Regexp) bool {
+	ans := false
+
+	if m != nil {
+		for k, v := range *m {
+			if r.MatchString(k + "=" + v) {
+				ans = true
+				break
+			}
+		}
+	}
+
+	return ans
+}
+
+func MapHasKey(m *map[string]string, label string) bool {
+	ans := false
+	if m != nil {
+		for k, _ := range *m {
+			if k == label {
+				ans = true
+				break
+			}
+		}
+	}
+	return ans
+}

pkg/helpers/references.go

@@ -0,0 +1,30 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       David Cassany <dcassany@suse.com>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"github.com/asaskevich/govalidator"
+	"strings"
+)
+
+func StripRegistryFromImage(image string) string {
+	img := strings.SplitN(image, "/", 2)
+	if len(img) == 2 && govalidator.IsURL(img[0]) {
+		return img[1]
+	}
+	return image
+}

pkg/helpers/references_test.go

@@ -0,0 +1,44 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       David Cassany <dcassany@suse.com>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Helpers", func() {
+	Context("StripRegistryFromImage", func() {
+		It("Strips the domain name", func() {
+			out := StripRegistryFromImage("valid.domain.org/base/image:tag")
+			Expect(out).To(Equal("base/image:tag"))
+		})
+		It("Strips the domain name when port is included", func() {
+			out := StripRegistryFromImage("valid.domain.org:5000/base/image:tag")
+			Expect(out).To(Equal("base/image:tag"))
+		})
+		It("Does not strip the domain name", func() {
+			out := StripRegistryFromImage("not-a-domain/base/image:tag")
+			Expect(out).To(Equal("not-a-domain/base/image:tag"))
+		})
+		It("Does not strip the domain name on invalid domains", func() {
+			out := StripRegistryFromImage("-invaliddomain.org/base/image:tag")
+			Expect(out).To(Equal("-invaliddomain.org/base/image:tag"))
+		})
+	})
+})

pkg/helpers/sys.go

@@ -16,7 +16,9 @@
 package helpers
 
 import (
+	"os"
 	"os/exec"
+	"os/user"
 	"syscall"
 
 	"github.com/pkg/errors"
@@ -30,3 +32,17 @@ func Exec(cmd string, args []string, env []string) error {
 	}
 	return syscall.Exec(path, args, env)
 }
+
+func GetHomeDir() (ans string) {
+	// os/user doesn't work in from scratch environments
+	u, err := user.Current()
+	if err == nil {
+		ans = u.HomeDir
+	} else {
+		ans = ""
+	}
+	if os.Getenv("HOME") != "" {
+		ans = os.Getenv("HOME")
+	}
+	return ans
+}

pkg/installer/client/docker.go

@@ -0,0 +1,190 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/docker/go-units"
+	"github.com/pkg/errors"
+
+	imgworker "github.com/mudler/luet/pkg/installer/client/imgworker"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+type DockerClient struct {
+	RepoData RepoData
+}
+
+func NewDockerClient(r RepoData) *DockerClient {
+	return &DockerClient{RepoData: r}
+}
+
+func downloadAndExtractDockerImage(image, dest string) error {
+	temp, err := config.LuetCfg.GetSystem().TempDir("contentstore")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(temp)
+	Debug("Temporary directory", temp)
+	c, err := imgworker.New(temp)
+	if err != nil {
+		return errors.Wrapf(err, "failed creating client")
+	}
+	defer c.Close()
+
+	// FROM Slightly adapted from genuinetools/img https://github.com/genuinetools/img/blob/54d0ca981c1260546d43961a538550eef55c87cf/pull.go
+	Debug("Pulling image", image)
+	listedImage, err := c.Pull(image)
+	if err != nil {
+		return errors.Wrapf(err, "failed listing images")
+
+	}
+	Debug("Pulled:", listedImage.Target.Digest)
+	Debug("Size:", units.BytesSize(float64(listedImage.ContentSize)))
+	Debug("Unpacking", image, "to", dest)
+	os.RemoveAll(dest)
+	return c.Unpack(image, dest)
+}
+
+func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+	//var u *url.URL = nil
+	var err error
+	var temp string
+
+	Spinner(22)
+	defer SpinnerStop()
+
+	var resultingArtifact compiler.Artifact
+	artifactName := path.Base(artifact.GetPath())
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
+	Debug("Cache file", cacheFile)
+	if err := helpers.EnsureDir(cacheFile); err != nil {
+		return nil, errors.Wrapf(err, "could not create cache folder %s for %s", config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), cacheFile)
+	}
+	ok := false
+
+	// TODO:
+	// Files are in URI/packagename:version (GetPackageImageName() method)
+	// use downloadAndExtract .. and egenrate an archive to consume. Checksum should be already checked while downloading the image
+	// with the above functions, because Docker images already contain such metadata
+	// - Check how verification is done when calling DownloadArtifact outside, similarly we need to check DownloadFile, and how verification
+	// is done in such cases (see repository.go)
+
+	// Check if file is already in cache
+	if helpers.Exists(cacheFile) {
+		Debug("Cache hit for artifact", artifactName)
+		resultingArtifact = artifact
+		resultingArtifact.SetPath(cacheFile)
+		resultingArtifact.SetChecksums(compiler.Checksums{})
+	} else {
+
+		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
+		if err != nil {
+			return nil, err
+		}
+		defer os.RemoveAll(temp)
+
+		for _, uri := range c.RepoData.Urls {
+
+			imageName := fmt.Sprintf("%s:%s", uri, artifact.GetCompileSpec().GetPackage().GetFingerPrint())
+			Info("Downloading image", imageName)
+
+			// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
+			err = downloadAndExtractDockerImage(imageName, temp)
+			if err != nil {
+				Debug("Failed download of image", imageName)
+				continue
+			}
+			Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile)
+
+			newart := artifact
+			// We discard checksum, that are checked while during pull and unpack
+			newart.SetChecksums(compiler.Checksums{})
+			newart.SetPath(cacheFile)                    // First set to cache file
+			newart.SetPath(newart.GetUncompressedName()) // Calculate the real path from cacheFile
+			err = newart.Compress(temp, 1)
+			if err != nil {
+				Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))
+				continue
+			}
+			resultingArtifact = newart
+
+			ok = true
+			break
+		}
+
+		if !ok {
+			return nil, err
+		}
+	}
+
+	return resultingArtifact, nil
+}
+
+func (c *DockerClient) DownloadFile(name string) (string, error) {
+	var file *os.File = nil
+	var err error
+	var temp string
+
+	// Files should be in URI/repository:<file>
+	ok := false
+
+	temp, err = config.LuetCfg.GetSystem().TempDir("tree")
+	if err != nil {
+		return "", err
+	}
+
+	for _, uri := range c.RepoData.Urls {
+
+		file, err = config.LuetCfg.GetSystem().TempFile("DockerClient")
+		if err != nil {
+			continue
+		}
+
+		Debug("Downloading file", name, "from", uri)
+
+		imageName := fmt.Sprintf("%s:%s", uri, name)
+		//imageName := fmt.Sprintf("%s/%s:%s", uri, "repository", name)
+		err = downloadAndExtractDockerImage(imageName, temp)
+		if err != nil {
+			Debug("Failed download of image", imageName)
+			continue
+		}
+
+		Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name())
+		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
+
+	if !ok {
+		return "", err
+	}
+
+	return file.Name(), err
+}

pkg/installer/client/docker_test.go

@@ -0,0 +1,77 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	compiler "github.com/mudler/luet/pkg/compiler"
+	helpers "github.com/mudler/luet/pkg/helpers"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/mudler/luet/pkg/installer/client"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// This test expect that the repository defined in UNIT_TEST_DOCKER_IMAGE is in zstd format.
+// the repository is built by the 01_simple_docker.sh integration test file.
+// This test also require root. At the moment, unpacking docker images with 'img' requires root permission to
+// mount/unmount layers.
+var _ = Describe("Docker client", func() {
+	Context("With repository", func() {
+		repoImage := os.Getenv("UNIT_TEST_DOCKER_IMAGE")
+		var repoURL []string
+		var c *DockerClient
+		BeforeEach(func() {
+			if repoImage == "" {
+				Skip("UNIT_TEST_DOCKER_IMAGE not specified")
+			}
+			repoURL = []string{repoImage}
+			c = NewDockerClient(RepoData{Urls: repoURL})
+		})
+
+		It("Downloads single files", func() {
+			f, err := c.DownloadFile("repository.yaml")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(f)).To(ContainSubstring("Test Repo"))
+			os.RemoveAll(f)
+		})
+
+		It("Downloads artifacts", func() {
+			f, err := c.DownloadArtifact(&compiler.PackageArtifact{
+				Path: "test.tar",
+				CompileSpec: &compiler.LuetCompilationSpec{
+					Package: &pkg.DefaultPackage{
+						Name:     "c",
+						Category: "test",
+						Version:  "1.0",
+					},
+				},
+			})
+			Expect(err).ToNot(HaveOccurred())
+			tmpdir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			Expect(f.Unpack(tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Read(filepath.Join(tmpdir, "c"))).To(Equal("c\n"))
+			Expect(helpers.Read(filepath.Join(tmpdir, "cd"))).To(Equal("c\n"))
+			os.RemoveAll(f.GetPath())
+		})
+	})
+})

pkg/installer/client/http.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"time"
 
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -30,6 +31,8 @@ import (
 	"github.com/mudler/luet/pkg/helpers"
 
 	"github.com/cavaliercoder/grab"
+
+	"github.com/schollz/progressbar/v3"
 )
 
 type HttpClient struct {
@@ -75,7 +78,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 
 	// Check if file is already in cache
 	if helpers.Exists(cacheFile) {
-		Info("Use artifact", artifactName, "from cache.")
+		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 
 		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
@@ -87,7 +90,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 		client := grab.NewClient()
 
 		for _, uri := range c.RepoData.Urls {
-			Info("Downloading artifact", artifactName, "from", uri)
+			Debug("Downloading artifact", artifactName, "from", uri)
 
 			u, err = url.Parse(uri)
 			if err != nil {
@@ -101,20 +104,61 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 			}
 
 			resp := client.Do(req)
+
+			bar := progressbar.NewOptions64(
+				resp.Size(),
+				progressbar.OptionSetDescription(
+					fmt.Sprintf("[cyan] %s - [reset]",
+						filepath.Base(resp.Request.HTTPRequest.URL.RequestURI()))),
+				progressbar.OptionSetRenderBlankState(true),
+				progressbar.OptionEnableColorCodes(config.LuetCfg.GetLogging().Color),
+				progressbar.OptionClearOnFinish(),
+				progressbar.OptionShowBytes(true),
+				progressbar.OptionShowCount(),
+				progressbar.OptionSetPredictTime(true),
+				progressbar.OptionFullWidth(),
+				progressbar.OptionSetTheme(progressbar.Theme{
+					Saucer:        "[white]=[reset]",
+					SaucerHead:    "[white]>[reset]",
+					SaucerPadding: " ",
+					BarStart:      "[",
+					BarEnd:        "]",
+				}))
+
+			bar.Reset()
+			// start download loop
+			t := time.NewTicker(500 * time.Millisecond)
+			defer t.Stop()
+
+		download_loop:
+
+			for {
+				select {
+				case <-t.C:
+					bar.Set64(resp.BytesComplete())
+
+				case <-resp.Done:
+					// download is complete
+					break download_loop
+				}
+			}
+
 			if err = resp.Err(); err != nil {
 				continue
 			}
 
-			Info("Downloaded", artifactName, "of",
-				fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
-				fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
-
-			Debug("Copying file ", filepath.Join(temp, artifactName), "to", cacheFile)
-			err = helpers.CopyFile(filepath.Join(temp, artifactName), cacheFile)
 			if err != nil {
 				continue
 			}
 
+			Info("\nDownloaded", artifactName, "of",
+				fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
+				fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
+
+			Debug("\nCopying file ", filepath.Join(temp, artifactName), "to", cacheFile)
+			err = helpers.CopyFile(filepath.Join(temp, artifactName), cacheFile)
+
+			bar.Finish()
 			ok = true
 			break
 		}
@@ -158,7 +202,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 		}
 		u.Path = path.Join(u.Path, name)
 
-		Info("Downloading", u.String())
+		Debug("Downloading", u.String())
 
 		req, err = c.PrepareReq(temp, u.String())
 		if err != nil {

pkg/installer/client/imgworker/client.go

@@ -0,0 +1,78 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/containerd/namespaces"
+	"github.com/genuinetools/img/types"
+	"github.com/moby/buildkit/control"
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/util/appcontext"
+	"github.com/moby/buildkit/worker/base"
+	"github.com/pkg/errors"
+)
+
+// Client holds the information for the client we will use for communicating
+// with the buildkit controller.
+type Client struct {
+	backend   string
+	localDirs map[string]string
+	root      string
+
+	sessionManager *session.Manager
+	controller     *control.Controller
+	opts           *base.WorkerOpt
+
+	sess *session.Session
+	ctx  context.Context
+}
+
+// New returns a new client for communicating with the buildkit controller.
+func New(root string) (*Client, error) {
+	// Native backend is fine, our images have just one layer. No need to depend on anything
+	backend := types.NativeBackend
+
+	// Create the root/
+	root = filepath.Join(root, "runc", backend)
+	if err := os.MkdirAll(root, 0700); err != nil {
+		return nil, err
+	}
+	c := &Client{
+		backend:   types.NativeBackend,
+		root:      root,
+		localDirs: nil,
+	}
+
+	if err := c.prepare(); err != nil {
+		return nil, errors.Wrapf(err, "failed preparing client")
+	}
+
+	// Create the start of the client.
+	return c, nil
+}
+
+func (c *Client) Close() {
+	c.sess.Close()
+}
+
+func (c *Client) prepare() error {
+	ctx := appcontext.Context()
+	sess, sessDialer, err := c.Session(ctx)
+	if err != nil {
+		return errors.Wrapf(err, "failed creating Session")
+	}
+	ctx = session.NewContext(ctx, sess.ID())
+	ctx = namespaces.WithNamespace(ctx, "buildkit")
+
+	c.ctx = ctx
+	c.sess = sess
+
+	go func() {
+		sess.Run(ctx, sessDialer)
+	}()
+	return nil
+}

pkg/installer/client/imgworker/pull.go

@@ -0,0 +1,129 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"fmt"
+
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
+	"github.com/docker/distribution/reference"
+	"github.com/moby/buildkit/cache"
+	"github.com/moby/buildkit/exporter"
+	imageexporter "github.com/moby/buildkit/exporter/containerimage"
+	"github.com/moby/buildkit/source"
+	"github.com/moby/buildkit/source/containerimage"
+)
+
+// ListedImage represents an image structure returuned from ListImages.
+// It extends containerd/images.Image with extra fields.
+type ListedImage struct {
+	images.Image
+	ContentSize int64
+}
+
+// Pull retrieves an image from a remote registry.
+func (c *Client) Pull(image string) (*ListedImage, error) {
+
+	ctx := c.ctx
+
+	sm, err := c.getSessionManager()
+	if err != nil {
+		return nil, err
+	}
+	// Parse the image name and tag.
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return nil, fmt.Errorf("parsing image name %q failed: %v", image, err)
+	}
+	// Add the latest lag if they did not provide one.
+	named = reference.TagNameOnly(named)
+	image = named.String()
+
+	// Get the identifier for the image.
+	identifier, err := source.NewImageIdentifier(image)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the worker opts.
+	opt, err := c.createWorkerOpt()
+	if err != nil {
+		return nil, fmt.Errorf("creating worker opt failed: %v", err)
+	}
+
+	cm, err := cache.NewManager(cache.ManagerOpt{
+		Snapshotter:    opt.Snapshotter,
+		MetadataStore:  opt.MetadataStore,
+		ContentStore:   opt.ContentStore,
+		LeaseManager:   opt.LeaseManager,
+		GarbageCollect: opt.GarbageCollect,
+		Applier:        opt.Applier,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the source for the pull.
+	srcOpt := containerimage.SourceOpt{
+		Snapshotter:   opt.Snapshotter,
+		ContentStore:  opt.ContentStore,
+		Applier:       opt.Applier,
+		CacheAccessor: cm,
+		ImageStore:    opt.ImageStore,
+		RegistryHosts: opt.RegistryHosts,
+		LeaseManager:  opt.LeaseManager,
+	}
+	src, err := containerimage.NewSource(srcOpt)
+	if err != nil {
+		return nil, err
+	}
+	s, err := src.Resolve(ctx, identifier, sm)
+	if err != nil {
+		return nil, err
+	}
+	ref, err := s.Snapshot(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the exporter for the pull.
+	iw, err := imageexporter.NewImageWriter(imageexporter.WriterOpt{
+		Snapshotter:  opt.Snapshotter,
+		ContentStore: opt.ContentStore,
+		Differ:       opt.Differ,
+	})
+	if err != nil {
+		return nil, err
+	}
+	expOpt := imageexporter.Opt{
+		SessionManager: sm,
+		ImageWriter:    iw,
+		Images:         opt.ImageStore,
+		RegistryHosts:  opt.RegistryHosts,
+		LeaseManager:   opt.LeaseManager,
+	}
+	exp, err := imageexporter.New(expOpt)
+	if err != nil {
+		return nil, err
+	}
+	e, err := exp.Resolve(ctx, map[string]string{"name": image})
+	if err != nil {
+		return nil, err
+	}
+	if _, err := e.Export(ctx, exporter.Source{Ref: ref}); err != nil {
+		return nil, err
+	}
+
+	// Get the image.
+	img, err := opt.ImageStore.Get(ctx, image)
+	if err != nil {
+		return nil, fmt.Errorf("getting image %s from image store failed: %v", image, err)
+	}
+	size, err := img.Size(ctx, opt.ContentStore, platforms.Default())
+	if err != nil {
+		return nil, fmt.Errorf("calculating size of image %s failed: %v", img.Name, err)
+	}
+
+	return &ListedImage{Image: img, ContentSize: size}, nil
+}

pkg/installer/client/imgworker/session.go

@@ -0,0 +1,51 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"context"
+	"os"
+
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/session/auth/authprovider"
+	"github.com/moby/buildkit/session/filesync"
+	"github.com/moby/buildkit/session/testutil"
+	"github.com/pkg/errors"
+)
+
+func (c *Client) getSessionManager() (*session.Manager, error) {
+	if c.sessionManager == nil {
+		var err error
+		c.sessionManager, err = session.NewManager()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return c.sessionManager, nil
+}
+
+// Session creates the session manager and returns the session and it's
+// dialer.
+func (c *Client) Session(ctx context.Context) (*session.Session, session.Dialer, error) {
+	m, err := c.getSessionManager()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to create session manager")
+	}
+	sessionName := "img"
+	s, err := session.NewSession(ctx, sessionName, "")
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to create session")
+	}
+	syncedDirs := make([]filesync.SyncedDir, 0, len(c.localDirs))
+	for name, d := range c.localDirs {
+		syncedDirs = append(syncedDirs, filesync.SyncedDir{Name: name, Dir: d})
+	}
+	s.Allow(filesync.NewFSSyncProvider(syncedDirs))
+	s.Allow(authprovider.NewDockerAuthProvider(os.Stderr))
+	return s, sessionDialer(s, m), err
+}
+
+func sessionDialer(s *session.Session, m *session.Manager) session.Dialer {
+	// FIXME: rename testutil
+	return session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn)))
+}

pkg/installer/client/imgworker/unpack.go

@@ -0,0 +1,82 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+)
+
+// TODO: this requires root permissions to mount/unmount layers, althrought it shouldn't be required.
+// See how backends are unpacking images without asking for root permissions.
+
+// Unpack exports an image to a rootfs destination directory.
+func (c *Client) Unpack(image, dest string) error {
+
+	ctx := c.ctx
+	if len(dest) < 1 {
+		return errors.New("destination directory for rootfs cannot be empty")
+	}
+
+	if _, err := os.Stat(dest); err == nil {
+		return fmt.Errorf("destination directory already exists: %s", dest)
+	}
+
+	// Parse the image name and tag.
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return fmt.Errorf("parsing image name %q failed: %v", image, err)
+	}
+	// Add the latest lag if they did not provide one.
+	named = reference.TagNameOnly(named)
+	image = named.String()
+
+	// Create the worker opts.
+	opt, err := c.createWorkerOpt()
+	if err != nil {
+		return fmt.Errorf("creating worker opt failed: %v", err)
+	}
+
+	if opt.ImageStore == nil {
+		return errors.New("image store is nil")
+	}
+
+	img, err := opt.ImageStore.Get(ctx, image)
+	if err != nil {
+		return fmt.Errorf("getting image %s from image store failed: %v", image, err)
+	}
+
+	manifest, err := images.Manifest(ctx, opt.ContentStore, img.Target, platforms.Default())
+	if err != nil {
+		return fmt.Errorf("getting image manifest failed: %v", err)
+	}
+
+	for _, desc := range manifest.Layers {
+		logrus.Debugf("Unpacking layer %s", desc.Digest.String())
+
+		// Read the blob from the content store.
+		layer, err := opt.ContentStore.ReaderAt(ctx, desc)
+		if err != nil {
+			return fmt.Errorf("getting reader for digest %s failed: %v", desc.Digest.String(), err)
+		}
+
+		// Unpack the tarfile to the rootfs path.
+		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
+		if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{
+			NoLchown:        true,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+		}); err != nil {
+			return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err)
+		}
+	}
+
+	return nil
+}

pkg/installer/client/imgworker/workeropts.go

@@ -0,0 +1,106 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/containerd/containerd/content/local"
+	"github.com/containerd/containerd/diff/apply"
+	"github.com/containerd/containerd/diff/walking"
+	ctdmetadata "github.com/containerd/containerd/metadata"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/remotes/docker"
+	ctdsnapshot "github.com/containerd/containerd/snapshots"
+	"github.com/containerd/containerd/snapshots/native"
+	"github.com/moby/buildkit/cache/metadata"
+	containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
+	"github.com/moby/buildkit/util/binfmt_misc"
+	"github.com/moby/buildkit/util/leaseutil"
+	"github.com/moby/buildkit/worker/base"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	bolt "go.etcd.io/bbolt"
+)
+
+// createWorkerOpt creates a base.WorkerOpt to be used for a new worker.
+func (c *Client) createWorkerOpt() (opt base.WorkerOpt, err error) {
+
+	if c.opts != nil {
+		return *c.opts, nil
+	}
+
+	// Create the metadata store.
+	md, err := metadata.NewStore(filepath.Join(c.root, "metadata.db"))
+	if err != nil {
+		return opt, err
+	}
+
+	snapshotRoot := filepath.Join(c.root, "snapshots")
+
+	s, err := native.NewSnapshotter(snapshotRoot)
+	if err != nil {
+		return opt, fmt.Errorf("creating %s snapshotter failed: %v", c.backend, err)
+	}
+
+	// Create the content store locally.
+	contentStore, err := local.NewStore(filepath.Join(c.root, "content"))
+	if err != nil {
+		return opt, err
+	}
+
+	// Open the bolt database for metadata.
+	db, err := bolt.Open(filepath.Join(c.root, "containerdmeta.db"), 0644, nil)
+	if err != nil {
+		return opt, err
+	}
+
+	// Create the new database for metadata.
+	mdb := ctdmetadata.NewDB(db, contentStore, map[string]ctdsnapshot.Snapshotter{
+		c.backend: s,
+	})
+	if err := mdb.Init(context.TODO()); err != nil {
+		return opt, err
+	}
+
+	// Create the image store.
+	imageStore := ctdmetadata.NewImageStore(mdb)
+
+	contentStore = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit")
+
+	id, err := base.ID(c.root)
+	if err != nil {
+		return opt, err
+	}
+
+	xlabels := base.Labels("oci", c.backend)
+
+	var supportedPlatforms []specs.Platform
+	for _, p := range binfmt_misc.SupportedPlatforms(false) {
+		parsed, err := platforms.Parse(p)
+		if err != nil {
+			return opt, err
+		}
+		supportedPlatforms = append(supportedPlatforms, platforms.Normalize(parsed))
+	}
+
+	opt = base.WorkerOpt{
+		ID:             id,
+		Labels:         xlabels,
+		MetadataStore:  md,
+		Snapshotter:    containerdsnapshot.NewSnapshotter(c.backend, mdb.Snapshotter(c.backend), "buildkit", nil),
+		ContentStore:   contentStore,
+		Applier:        apply.NewFileSystemApplier(contentStore),
+		Differ:         walking.NewWalkingDiff(contentStore),
+		ImageStore:     imageStore,
+		Platforms:      supportedPlatforms,
+		RegistryHosts:  docker.ConfigureDefaultRegistries(),
+		LeaseManager:   leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"),
+		GarbageCollect: mdb.GarbageCollect,
+	}
+
+	c.opts = &opt
+
+	return opt, err
+}

pkg/installer/client/local.go

@@ -38,15 +38,26 @@ func NewLocalClient(r RepoData) *LocalClient {
 func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
 	var err error
 
+	rootfs := ""
 	artifactName := path.Base(artifact.GetPath())
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 
+	if !config.LuetCfg.ConfigFromHost {
+		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Check if file is already in cache
 	if helpers.Exists(cacheFile) {
-		Info("Use artifact", artifactName, "from cache.")
+		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 		ok := false
 		for _, uri := range c.RepoData.Urls {
+
+			uri = filepath.Join(rootfs, uri)
+
 			Info("Downloading artifact", artifactName, "from", uri)
 
 			//defer os.Remove(file.Name())
@@ -72,8 +83,20 @@ func (c *LocalClient) DownloadFile(name string) (string, error) {
 	var err error
 	var file *os.File = nil
 
+	rootfs := ""
+
+	if !config.LuetCfg.ConfigFromHost {
+		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return "", err
+		}
+	}
+
 	ok := false
 	for _, uri := range c.RepoData.Urls {
+
+		uri = filepath.Join(rootfs, uri)
+
 		Info("Downloading file", name, "from", uri)
 		file, err = config.LuetCfg.GetSystem().TempFile("localclient")
 		if err != nil {

pkg/installer/config_protect.go

@@ -0,0 +1,100 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"io/ioutil"
+	"path"
+	"path/filepath"
+	"regexp"
+
+	"github.com/ghodss/yaml"
+
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+func LoadConfigProtectConfs(c *LuetConfig) error {
+	var regexConfs = regexp.MustCompile(`.yml$`)
+	var err error
+
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, cdir := range c.ConfigProtectConfDir {
+		cdir = filepath.Join(rootfs, cdir)
+
+		Debug("Parsing Config Protect Directory", cdir, "...")
+
+		files, err := ioutil.ReadDir(cdir)
+		if err != nil {
+			Debug("Skip dir", cdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexConfs.MatchString(file.Name()) {
+				Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(cdir, file.Name()))
+			if err != nil {
+				Warning("On read file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := LoadConfigProtectConFile(file.Name(), content)
+			if err != nil {
+				Warning("On parse file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Directories) == 0 {
+				Warning("Invalid config protect file", file.Name())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddConfigProtectConfFile(r)
+		}
+	}
+	return nil
+
+}
+
+func LoadConfigProtectConFile(filename string, data []byte) (*ConfigProtectConfFile, error) {
+	ans := NewConfigProtectConfFile(filename)
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}

pkg/installer/finalizer.go

@@ -47,7 +47,7 @@ func (f *LuetFinalizer) RunInstall(s *System) error {
 
 	for _, c := range f.Install {
 		toRun := append(args, c)
-		Info("Executing finalizer on ", s.Target, cmd, toRun)
+		Info(":shell: Executing finalizer on ", s.Target, cmd, toRun)
 		if s.Target == "/" {
 			cmd := exec.Command(cmd, toRun...)
 			stdoutStderr, err := cmd.CombinedOutput()

pkg/installer/installer.go

@@ -16,6 +16,7 @@
 package installer
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -23,24 +24,29 @@ import (
 	"strings"
 	"sync"
 
+	. "github.com/logrusorgru/aurora"
+	"github.com/mudler/luet/pkg/bus"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
-	"github.com/mudler/luet/pkg/tree"
 
 	"github.com/pkg/errors"
 )
 
 type LuetInstallerOptions struct {
-	SolverOptions               config.LuetSolverOptions
-	Concurrency                 int
-	NoDeps                      bool
-	OnlyDeps                    bool
-	Force                       bool
-	PreserveSystemEssentialData bool
+	SolverOptions                                                  config.LuetSolverOptions
+	Concurrency                                                    int
+	NoDeps                                                         bool
+	OnlyDeps                                                       bool
+	Force                                                          bool
+	PreserveSystemEssentialData                                    bool
+	FullUninstall, FullCleanUninstall                              bool
+	CheckConflicts                                                 bool
+	SolverUpgrade, RemoveUnavailableOnUpgrade, UpgradeNewRevisions bool
+	Ask                                                            bool
 }
 
 type LuetInstaller struct {
@@ -59,22 +65,31 @@ func NewLuetInstaller(opts LuetInstallerOptions) Installer {
 	return &LuetInstaller{Options: opts}
 }
 
-func (l *LuetInstaller) Upgrade(s *System) error {
-	syncedRepos, err := l.SyncRepositories(true)
-	if err != nil {
-		return err
-	}
+// computeUpgrade returns the packages to be uninstalled and installed in a system to perform an upgrade
+// based on the system repositories
+func (l *LuetInstaller) computeUpgrade(syncedRepos Repositories, s *System) (pkg.Packages, pkg.Packages, error) {
+	toInstall := pkg.Packages{}
+	var uninstall pkg.Packages
+	var err error
 	// First match packages against repositories by priority
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	// compute a "big" world
-	solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
-	uninstall, solution, err := solv.Upgrade(false)
-	if err != nil {
-		return errors.Wrap(err, "Failed solving solution for upgrade")
+	solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+	var solution solver.PackagesAssertions
+
+	if l.Options.SolverUpgrade {
+		uninstall, solution, err = solv.UpgradeUniverse(l.Options.RemoveUnavailableOnUpgrade)
+		if err != nil {
+			return uninstall, toInstall, errors.Wrap(err, "Failed solving solution for upgrade")
+		}
+	} else {
+		uninstall, solution, err = solv.Upgrade(l.Options.FullUninstall, true)
+		if err != nil {
+			return uninstall, toInstall, errors.Wrap(err, "Failed solving solution for upgrade")
+		}
 	}
 
-	toInstall := pkg.Packages{}
 	for _, assertion := range solution {
 		// Be sure to filter from solutions packages already installed in the system
 		if _, err := s.Database.FindPackage(assertion.Package); err != nil && assertion.Value {
@@ -82,7 +97,93 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 		}
 	}
 
-	return l.swap(syncedRepos, uninstall, toInstall, s)
+	if l.Options.UpgradeNewRevisions {
+		for _, p := range s.Database.World() {
+			matches := syncedRepos.PackageMatches(pkg.Packages{p})
+			if len(matches) == 0 {
+				// Package missing. the user should run luet upgrade --universe
+				continue
+			}
+			for _, artefact := range matches[0].Repo.GetIndex() {
+				if artefact.GetCompileSpec().GetPackage() == nil {
+					return uninstall, toInstall, errors.New("Package in compilespec empty")
+
+				}
+				if artefact.GetCompileSpec().GetPackage().Matches(p) && artefact.GetCompileSpec().GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
+					toInstall = append(toInstall, matches[0].Package).Unique()
+					uninstall = append(uninstall, p).Unique()
+				}
+			}
+		}
+	}
+
+	return uninstall, toInstall, nil
+}
+
+func packsToList(p pkg.Packages) string {
+	var packs []string
+
+	for _, pp := range p {
+		packs = append(packs, pp.HumanReadableString())
+	}
+	return strings.Join(packs, " ")
+}
+
+func matchesToList(artefacts map[string]ArtifactMatch) string {
+	var packs []string
+
+	for fingerprint, match := range artefacts {
+		packs = append(packs, fmt.Sprintf("%s (%s)", fingerprint, match.Repository.GetName()))
+	}
+	return strings.Join(packs, " ")
+}
+
+// Upgrade upgrades a System based on the Installer options. Returns error in case of failure
+func (l *LuetInstaller) Upgrade(s *System) error {
+
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+
+	Info(":thinking: Computing upgrade, please hang tight... :zzz:")
+	if l.Options.UpgradeNewRevisions {
+		Info(":memo: note: will consider new build revisions while upgrading")
+	}
+
+	Spinner(32)
+	uninstall, toInstall, err := l.computeUpgrade(syncedRepos, s)
+	if err != nil {
+		return errors.Wrap(err, "failed computing upgrade")
+	}
+	SpinnerStop()
+
+	if len(uninstall) > 0 {
+		Info(":recycle: Packages that are going to be removed from the system:\n ", Yellow(packsToList(uninstall)).BgBlack().String())
+	}
+
+	if len(toInstall) > 0 {
+		Info(":zap:Packages that are going to be installed in the system:\n ", Green(packsToList(toInstall)).BgBlack().String())
+	}
+
+	if len(toInstall) == 0 && len(uninstall) == 0 {
+		Info("Nothing to do")
+		return nil
+	}
+
+	if l.Options.Ask {
+		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
+		if Ask() {
+			l.Options.Ask = false // Don't prompt anymore
+			return l.swap(syncedRepos, uninstall, toInstall, s, true)
+		} else {
+			return errors.New("Aborted by user")
+		}
+	}
+
+	Spinner(32)
+	defer SpinnerStop()
+	return l.swap(syncedRepos, uninstall, toInstall, s, true)
 }
 
 func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
@@ -112,19 +213,59 @@ func (l *LuetInstaller) Swap(toRemove pkg.Packages, toInstall pkg.Packages, s *S
 	if err != nil {
 		return err
 	}
-	return l.swap(syncedRepos, toRemove, toInstall, s)
+
+	toRemoveFinal := pkg.Packages{}
+	for _, p := range toRemove {
+		packs, _ := s.Database.FindPackages(p)
+		if len(packs) == 0 {
+			return errors.New("Package " + p.HumanReadableString() + " not found in the system")
+		}
+		for _, pp := range packs {
+			toRemoveFinal = append(toRemoveFinal, pp)
+		}
+	}
+
+	return l.swap(syncedRepos, toRemoveFinal, toInstall, s, false)
 }
 
-func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) error {
-	// First match packages against repositories by priority
+func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
+
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
+
 	toInstall = syncedRepos.ResolveSelectors(toInstall)
 
-	if err := l.download(syncedRepos, toInstall); err != nil {
-		return errors.Wrap(err, "Pre-downloading packages")
+	// First check what would have been done
+	installedtmp, err := s.Database.Copy()
+	if err != nil {
+		return nil, nil, nil, nil, errors.Wrap(err, "Failed create temporary in-memory db")
 	}
 
+	systemAfterChanges := &System{Database: installedtmp}
+
+	packs, err := l.computeUninstall(systemAfterChanges, toRemove...)
+	if err != nil && !l.Options.Force {
+		Error("Failed computing uninstall for ", packsToList(toRemove))
+		return nil, nil, nil, nil, errors.Wrap(err, "computing uninstall "+packsToList(toRemove))
+	}
+	for _, p := range packs {
+		err = systemAfterChanges.Database.RemovePackage(p)
+		if err != nil {
+			return nil, nil, nil, nil, errors.Wrap(err, "Failed removing package from database")
+		}
+	}
+
+	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, toInstall, systemAfterChanges)
+	for _, p := range toInstall {
+		assertions = append(assertions, solver.PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
+	}
+	return match, packages, assertions, allRepos, err
+}
+
+func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System, forceNodeps bool) error {
+	forced := l.Options.Force
+	nodeps := l.Options.NoDeps
+
 	// We don't want any conflict with the installed to raise during the upgrade.
 	// In this way we both force uninstalls and we avoid to check with conflicts
 	// against the current system state which is pending to deletion
@@ -132,25 +273,47 @@ func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, to
 	// if the old A results installed in the system. This is due to the fact that
 	// now the solver enforces the constraints and explictly denies two packages
 	// of the same version installed.
-	forced := false
-	if l.Options.Force {
-		forced = true
-	}
 	l.Options.Force = true
+	if forceNodeps {
+		l.Options.NoDeps = true
+	}
 
-	for _, u := range toRemove {
-		Info(":package: Marked for deletion", u.HumanReadableString())
+	match, packages, assertions, allRepos, err := l.computeSwap(syncedRepos, toRemove, toInstall, s)
+	if err != nil {
+		return errors.Wrap(err, "failed computing package replacement")
+	}
 
-		err := l.Uninstall(u, s)
-		if err != nil && !l.Options.Force {
-			Error("Failed uninstall for ", u.HumanReadableString())
-			return errors.Wrap(err, "uninstalling "+u.HumanReadableString())
+	if l.Options.Ask {
+		if len(toRemove) > 0 {
+			Info(":recycle: Packages that are going to be removed from the system:\n ", Yellow(packsToList(toRemove)).BgBlack().String())
 		}
 
-	}
-	l.Options.Force = forced
+		if len(match) > 0 {
+			Info("Packages that are going to be installed in the system: \n ", Green(matchesToList(match)).BgBlack().String())
+		}
 
-	return l.install(syncedRepos, toInstall, s)
+		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
+		if Ask() {
+			l.Options.Ask = false // Don't prompt anymore
+		} else {
+			return errors.New("Aborted by user")
+		}
+	}
+
+	// First match packages against repositories by priority
+	if err := l.download(syncedRepos, match); err != nil {
+		return errors.Wrap(err, "Pre-downloading packages")
+	}
+
+	err = l.Uninstall(s, toRemove...)
+	if err != nil && !l.Options.Force {
+		Error("Failed uninstall for ", packsToList(toRemove))
+		return errors.Wrap(err, "uninstalling "+packsToList(toRemove))
+	}
+
+	l.Options.Force = forced
+	l.Options.NoDeps = nodeps
+	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
 }
 
 func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
@@ -158,35 +321,54 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 	if err != nil {
 		return err
 	}
-	return l.install(syncedRepos, cp, s)
-}
 
-func (l *LuetInstaller) download(syncedRepos Repositories, cp pkg.Packages) error {
-	toDownload := map[string]ArtifactMatch{}
-
-	// FIXME: This can be optimized. We don't need to re-match this to the repository
-	// But we could just do it once
-
-	// Gathers things to download
-	for _, currentPack := range cp {
-		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
-		if len(matches) == 0 {
-			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
-		}
-	A:
-		for _, artefact := range matches[0].Repo.GetIndex() {
-			if artefact.GetCompileSpec().GetPackage() == nil {
-				return errors.New("Package in compilespec empty")
+	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, cp, s)
+	if err != nil {
+		return err
+	}
 
+	// Check if we have to process something, or return to the user an error
+	if len(match) == 0 {
+		Info("No packages to install")
+		return nil
+	}
+	// Resolvers might decide to remove some packages from being installed
+	if !l.Options.SolverOptions.ResolverIsSet() {
+		for _, p := range cp {
+			found := false
+			vers, _ := s.Database.FindPackageVersions(p) // If was installed, it is found, as it was filtered
+			if len(vers) >= 1 {
+				found = true
+				continue
 			}
-			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
 
-				toDownload[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
+			for _, m := range match {
+				if m.Package.GetName() == p.GetName() {
+					found = true
+				}
+			}
 
-				break A
+			if !found {
+				return fmt.Errorf("Package '%s' not found", p.HumanReadableString())
 			}
 		}
 	}
+	Info("Packages that are going to be installed in the system: \n ", Green(matchesToList(match)).BgBlack().String())
+
+	if l.Options.Ask {
+		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
+		if Ask() {
+			l.Options.Ask = false // Don't prompt anymore
+			return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+		} else {
+			return errors.New("Aborted by user")
+		}
+	}
+	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+}
+
+func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string]ArtifactMatch) error {
+
 	// Download packages into cache in parallel.
 	all := make(chan ArtifactMatch)
 
@@ -229,7 +411,7 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 					if err != nil {
 						return err
 					}
-					Info("Found package:", p.HumanReadableString())
+					Info(":mag: Found package:", p.HumanReadableString())
 					toMerge = append(toMerge, ArtifactMatch{Artifact: artefact, Package: p})
 					break FILES
 				}
@@ -250,23 +432,25 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 			return errors.Wrap(err, "Failed creating package")
 		}
 		s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.GetFiles()})
-		Info("Reclaimed package:", pack.HumanReadableString())
+		Info(":zap:Reclaimed package:", pack.HumanReadableString())
 	}
 	Info("Done!")
 
 	return nil
 }
 
-func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *System) error {
+func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
 	var p pkg.Packages
+	toInstall := map[string]ArtifactMatch{}
+	allRepos := pkg.NewInMemoryDatabase(false)
+	var solution solver.PackagesAssertions
 
 	// Check if the package is installed first
 	for _, pi := range cp {
-
 		vers, _ := s.Database.FindPackageVersions(pi)
 
 		if len(vers) >= 1 {
-			Warning("Filtering out package " + pi.HumanReadableString() + ", it has other versions already installed. Uninstall one of them first ")
+			//	Warning("Filtering out package " + pi.HumanReadableString() + ", it has other versions already installed. Uninstall one of them first ")
 			continue
 			//return errors.New("Package " + pi.GetFingerPrint() + " has other versions already installed. Uninstall one of them first: " + strings.Join(vers, " "))
 
@@ -275,8 +459,7 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 	}
 
 	if len(p) == 0 {
-		Warning("No package to install, bailing out with no errors")
-		return nil
+		return toInstall, p, solution, allRepos, nil
 	}
 	// First get metas from all repos (and decodes trees)
 
@@ -284,50 +467,52 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 	//	matches := syncedRepos.PackageMatches(p)
 
 	// compute a "big" world
-	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	p = syncedRepos.ResolveSelectors(p)
-	toInstall := map[string]ArtifactMatch{}
 	var packagesToInstall pkg.Packages
 	var err error
-	var solution solver.PackagesAssertions
 
 	if !l.Options.NoDeps {
-		solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		solution, err = solv.Install(p)
+		/// TODO: PackageAssertions needs to be a map[fingerprint]pack so lookup is in O(1)
 		if err != nil && !l.Options.Force {
-			return errors.Wrap(err, "Failed solving solution for package")
+			return toInstall, p, solution, allRepos, errors.Wrap(err, "Failed solving solution for package")
 		}
 		// Gathers things to install
 		for _, assertion := range solution {
 			if assertion.Value {
+				if _, err := s.Database.FindPackage(assertion.Package); err == nil {
+					// skip matching if it is installed already
+					continue
+				}
 				packagesToInstall = append(packagesToInstall, assertion.Package)
 			}
 		}
 	} else if !l.Options.OnlyDeps {
 		for _, currentPack := range p {
+			if _, err := s.Database.FindPackage(currentPack); err == nil {
+				// skip matching if it is installed already
+				continue
+			}
 			packagesToInstall = append(packagesToInstall, currentPack)
 		}
 	}
-
 	// Gathers things to install
 	for _, currentPack := range packagesToInstall {
 		// Check if package is already installed.
-		if _, err := s.Database.FindPackage(currentPack); err == nil {
-			// skip matching if it is installed already
-			continue
-		}
+
 		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
 		if len(matches) == 0 {
-			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
+			return toInstall, p, solution, allRepos, errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
 		}
 	A:
 		for _, artefact := range matches[0].Repo.GetIndex() {
 			if artefact.GetCompileSpec().GetPackage() == nil {
-				return errors.New("Package in compilespec empty")
-
+				return toInstall, p, solution, allRepos, errors.New("Package in compilespec empty")
 			}
 			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+				currentPack.SetBuildTimestamp(artefact.GetCompileSpec().GetPackage().GetBuildTimestamp())
 				// Filter out already installed
 				if _, err := s.Database.FindPackage(currentPack); err != nil {
 					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
@@ -336,26 +521,18 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 			}
 		}
 	}
+	return toInstall, p, solution, allRepos, nil
+}
+
+func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
 	// Install packages into rootfs in parallel.
+	if err := l.download(syncedRepos, toInstall); err != nil {
+		return errors.Wrap(err, "Downloading packages")
+	}
+
 	all := make(chan ArtifactMatch)
 
-	var wg = new(sync.WaitGroup)
-
-	// Download first
-	for i := 0; i < l.Options.Concurrency; i++ {
-		wg.Add(1)
-		go l.downloadWorker(i, wg, all)
-	}
-
-	for _, c := range toInstall {
-		all <- c
-	}
-	close(all)
-	wg.Wait()
-
-	all = make(chan ArtifactMatch)
-
-	wg = new(sync.WaitGroup)
+	wg := new(sync.WaitGroup)
 
 	// Do the real install
 	for i := 0; i < l.Options.Concurrency; i++ {
@@ -375,9 +552,9 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Failed creating package")
 		}
+		bus.Manager.Publish(bus.EventPackageInstall, c)
 	}
-	executedFinalizer := map[string]bool{}
-
+	var toFinalize []pkg.Package
 	if !l.Options.NoDeps {
 		// TODO: Lower those errors as warning
 		for _, w := range p {
@@ -389,36 +566,17 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 		ORDER:
 			for _, ass := range ordered {
 				if ass.Value {
-
 					installed, ok := toInstall[ass.Package.GetFingerPrint()]
 					if !ok {
 						// It was a dep already installed in the system, so we can skip it safely
 						continue ORDER
 					}
-
 					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
 					if err != nil {
 						return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
 					}
-					if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
-						Info("Executing finalizer for " + ass.Package.HumanReadableString())
-						finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
-						if err != nil && !l.Options.Force {
-							return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
-						}
-						if _, exists := executedFinalizer[ass.Package.GetFingerPrint()]; !exists {
-							finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
-							if err != nil && !l.Options.Force {
-								return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
-							}
-							err = finalizer.RunInstall(s)
-							if err != nil && !l.Options.Force {
-								return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
-							}
-							executedFinalizer[ass.Package.GetFingerPrint()] = true
-						}
-					}
 
+					toFinalize = append(toFinalize, treePackage)
 				}
 			}
 
@@ -429,29 +587,11 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 			if err != nil {
 				return errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
 			}
-			if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
-				Info("Executing finalizer for " + c.Package.HumanReadableString())
-				finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
-				if err != nil && !l.Options.Force {
-					return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
-				}
-				if _, exists := executedFinalizer[c.Package.GetFingerPrint()]; !exists {
-					finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
-					if err != nil && !l.Options.Force {
-						return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
-					}
-					err = finalizer.RunInstall(s)
-					if err != nil && !l.Options.Force {
-						return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
-					}
-					executedFinalizer[c.Package.GetFingerPrint()] = true
-				}
-			}
+			toFinalize = append(toFinalize, treePackage)
 		}
 	}
 
-	return nil
-
+	return s.ExecuteFinalizers(toFinalize)
 }
 
 func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) {
@@ -462,7 +602,7 @@ func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, err
 	}
 
 	err = artifact.Verify()
-	if err != nil && !l.Options.Force {
+	if err != nil {
 		return nil, errors.Wrap(err, "Artifact integrity check failure")
 	}
 	return artifact, nil
@@ -496,15 +636,11 @@ func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan Artif
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
 		_, err := l.downloadPackage(p)
-		if err != nil && !l.Options.Force {
-			//TODO: Uninstall, rollback.
-			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
-			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
-		}
-		if err == nil {
-			Info(":package: ", p.Package.HumanReadableString(), "downloaded")
-		} else if err != nil && l.Options.Force {
-			Info(":package: ", p.Package.HumanReadableString(), "downloaded with failures (force download)")
+		if err != nil {
+			Fatal("Failed downloading package "+p.Package.GetName(), err.Error())
+			return errors.Wrap(err, "Failed downloading package "+p.Package.GetName())
+		} else {
+			Info(":package: Package ", p.Package.HumanReadableString(), "downloaded")
 		}
 	}
 
@@ -523,9 +659,9 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
 		}
 		if err == nil {
-			Info(":package: ", p.Package.HumanReadableString(), "installed")
+			Info(":package: Package ", p.Package.HumanReadableString(), "installed")
 		} else if err != nil && l.Options.Force {
-			Info(":package: ", p.Package.HumanReadableString(), "installed with failures (force install)")
+			Info(":package: Package ", p.Package.HumanReadableString(), "installed with failures (forced install)")
 		}
 	}
 
@@ -533,16 +669,39 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 }
 
 func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
+	var cp *config.ConfigProtect
+	annotationDir := ""
+
 	files, err := s.Database.GetPackageFiles(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed getting installed files")
 	}
 
-	// Remove from target
-	for _, f := range files {
-		target := filepath.Join(s.Target, f)
-		Debug("Removing", target)
+	if !config.LuetCfg.ConfigProtectSkip {
 
+		if p.HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := p.GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
+			}
+		}
+
+		cp = config.NewConfigProtect(annotationDir)
+		cp.Map(files)
+	}
+
+	toRemove, notPresent := helpers.OrderFiles(s.Target, files)
+
+	// Remove from target
+	for _, f := range toRemove {
+		target := filepath.Join(s.Target, f)
+
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Preserving protected file:", f)
+			continue
+		}
+
+		Debug("Removing", target)
 		if l.Options.PreserveSystemEssentialData &&
 			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) ||
 			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath()) {
@@ -550,11 +709,41 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 			continue
 		}
 
-		err := os.Remove(target)
+		fi, err := os.Lstat(target)
 		if err != nil {
-			Warning("Failed removing file (not present in the system target ?)", target)
+			Warning("File not found (it was before?) ", err.Error())
+			continue
+		}
+		switch mode := fi.Mode(); {
+		case mode.IsDir():
+			files, err := ioutil.ReadDir(target)
+			if err != nil {
+				Warning("Failed reading folder", target, err.Error())
+			}
+			if len(files) != 0 {
+				Debug("Preserving not-empty folder", target)
+				continue
+			}
+		}
+
+		if err = os.Remove(target); err != nil {
+			Warning("Failed removing file (maybe not present in the system target anymore ?)", target, err.Error())
 		}
 	}
+
+	for _, f := range notPresent {
+		target := filepath.Join(s.Target, f)
+
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Preserving protected file:", f)
+			continue
+		}
+
+		if err = os.Remove(target); err != nil {
+			Debug("Failed removing file (not present in the system target)", target, err.Error())
+		}
+	}
+
 	err = s.Database.RemovePackageFiles(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed removing package files from database")
@@ -564,42 +753,98 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		return errors.Wrap(err, "Failed removing package from database")
 	}
 
-	Info(p.GetFingerPrint(), "Removed")
+	bus.Manager.Publish(bus.EventPackageUnInstall, p)
+
+	Info(":recycle: ", p.GetFingerPrint(), "Removed :heavy_check_mark:")
 	return nil
 }
 
-func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
+func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.Packages, error) {
+
+	var toUninstall pkg.Packages
 	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) TODO - mark the uninstallation in db
 	// Get installed definition
-
-	checkConflicts := true
-	if l.Options.Force == true {
+	checkConflicts := l.Options.CheckConflicts
+	full := l.Options.FullUninstall
+	if l.Options.Force == true { // IF forced, we want to remove the package and all its requires
 		checkConflicts = false
+		full = false
+	}
+
+	// Create a temporary DB with the installed packages
+	// so the solver is much faster finding the deptree
+	// First check what would have been done
+	installedtmp, err := s.Database.Copy()
+	if err != nil {
+		return toUninstall, errors.Wrap(err, "Failed create temporary in-memory db")
 	}
 
 	if !l.Options.NoDeps {
-		solv := solver.NewResolver(s.Database, s.Database, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
-		solution, err := solv.Uninstall(p, checkConflicts)
-		if err != nil && !l.Options.Force {
-			return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		var solution pkg.Packages
+		var err error
+		if l.Options.FullCleanUninstall {
+			solution, err = solv.UninstallUniverse(packs)
+			if err != nil {
+				return toUninstall, errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+			}
+		} else {
+			solution, err = solv.Uninstall(checkConflicts, full, packs...)
+			if err != nil && !l.Options.Force {
+				return toUninstall, errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+			}
 		}
+
 		for _, p := range solution {
-			Info("Uninstalling", p.HumanReadableString())
+			toUninstall = append(toUninstall, p)
+		}
+	} else {
+		toUninstall = append(toUninstall, packs...)
+	}
+
+	return toUninstall, nil
+}
+func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
+
+	for _, p := range packs {
+		if packs, _ := s.Database.FindPackages(p); len(packs) == 0 {
+			return errors.New("Package not found in the system")
+		}
+
+	}
+
+	Spinner(32)
+	toUninstall, err := l.computeUninstall(s, packs...)
+	if err != nil {
+		return errors.Wrap(err, "while computing uninstall")
+	}
+	SpinnerStop()
+
+	uninstall := func() error {
+		for _, p := range toUninstall {
 			err := l.uninstall(p, s)
 			if err != nil && !l.Options.Force {
 				return errors.Wrap(err, "Uninstall failed")
 			}
 		}
-	} else {
-		Info("Uninstalling", p.HumanReadableString(), "without deps")
-		err := l.uninstall(p, s)
-		if err != nil && !l.Options.Force {
-			return errors.Wrap(err, "Uninstall failed")
-		}
-		Info(":package: ", p.HumanReadableString(), "uninstalled")
+		return nil
 	}
-	return nil
 
+	if len(toUninstall) == 0 {
+		Info("Nothing to do")
+		return nil
+	}
+
+	if l.Options.Ask {
+		Info(":recycle: Packages that are going to be removed from the system:\n   ", Yellow(packsToList(toUninstall)).BgBlack().String())
+		if Ask() {
+			l.Options.Ask = false // Don't prompt anymore
+			return uninstall()
+		} else {
+			return errors.New("Aborted by user")
+		}
+	}
+	return uninstall()
 }
 
 func (l *LuetInstaller) Repositories(r []Repository) { l.PackageRepositories = r }

pkg/installer/installer_test.go

@@ -24,6 +24,9 @@ import (
 	compiler "github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/installer"
+	solver "github.com/mudler/luet/pkg/solver"
+
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
@@ -31,6 +34,18 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+func stubRepo(tmpdir, tree string) (installer.Repository, error) {
+	return GenerateRepository(
+		"test",
+		"description",
+		"disk",
+		[]string{tmpdir},
+		1,
+		tmpdir,
+		[]string{tree},
+		pkg.NewInMemoryDatabase(false), nil, "", false, false)
+}
+
 var _ = Describe("Installer", func() {
 	Context("Writes a repository definition", func() {
 		It("Writes a repo and can install packages from it", func() {
@@ -47,7 +62,7 @@ var _ = Describe("Installer", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -58,8 +73,8 @@ var _ = Describe("Installer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -82,13 +97,13 @@ var _ = Describe("Installer", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -132,7 +147,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p.GetName()).To(Equal("b"))
 
-			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			err = inst.Uninstall(system, &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
@@ -163,7 +178,7 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
 			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
-				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -174,8 +189,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -198,7 +213,9 @@ urls:
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
 			treeFile := NewDefaultTreeRepositoryFile()
 			treeFile.SetCompressionType(compiler.None)
 			repo.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
@@ -207,7 +224,7 @@ urls:
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -251,7 +268,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p.GetName()).To(Equal("b"))
 
-			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			err = inst.Uninstall(system, &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
@@ -281,7 +298,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -292,8 +309,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -316,13 +333,20 @@ urls:
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository(
+				"test",
+				"description",
+				"disk",
+				[]string{tmpdir}, 1,
+				tmpdir,
+				[]string{"../../tests/fixtures/buildable"},
+				pkg.NewInMemoryDatabase(false), nil, "", false, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -371,7 +395,7 @@ urls:
 			Expect(files).To(Equal([]string{"artifact42", "test5", "test6"}))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			err = inst.Uninstall(system, &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
@@ -399,7 +423,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -410,8 +434,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -434,13 +458,21 @@ urls:
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository(
+				"test",
+				"description",
+				"disk",
+				[]string{tmpdir},
+				1,
+				tmpdir,
+				[]string{"../../tests/fixtures/buildable"},
+				pkg.NewInMemoryDatabase(false), nil, "", false, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -486,7 +518,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -500,9 +532,9 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 
-			repo, err = GenerateRepository("test", "description", "disk", []string{tmpdir2}, 1, tmpdir2, "../../tests/fixtures/alpine", pkg.NewInMemoryDatabase(false))
+			repo, err = stubRepo(tmpdir2, "../../tests/fixtures/alpine")
 			Expect(err).ToNot(HaveOccurred())
-			err = repo.Write(tmpdir2, false)
+			err = repo.Write(tmpdir2, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			fakeroot, err = ioutil.TempDir("", "fakeroot")
@@ -545,7 +577,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -569,13 +601,13 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -663,8 +695,8 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 			Expect(len(generalRecipeNewRepo.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
-			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -695,18 +727,18 @@ urls:
 			_, errs = c2.CompileParallel(false, compiler.NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade_old_repo", pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			repoupgrade, err := GenerateRepository("test", "description", "disk", []string{tmpdirnewrepo}, 1, tmpdirnewrepo, "../../tests/fixtures/upgrade_new_repo", pkg.NewInMemoryDatabase(false))
+			repoupgrade, err := stubRepo(tmpdirnewrepo, "../../tests/fixtures/upgrade_new_repo")
 			Expect(err).ToNot(HaveOccurred())
-			err = repoupgrade.Write(tmpdirnewrepo, false)
+			err = repoupgrade.Write(tmpdirnewrepo, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
@@ -795,7 +827,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -818,13 +850,13 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
@@ -897,6 +929,47 @@ urls:
 
 	})
 
+	Context("Uninstallation", func() {
+		It("fails if package is required by others which are installed", func() {
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1, CheckConflicts: true})
+
+			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B := pkg.NewPackage("calamares", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+			C := pkg.NewPackage("kpmcore", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+			Z := pkg.NewPackage("chromium", "", []*pkg.DefaultPackage{A}, []*pkg.DefaultPackage{})
+			F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{Z, B}, []*pkg.DefaultPackage{})
+
+			Z.SetVersion("86.0.4240.193+2")
+			Z.SetCategory("www-client")
+			B.SetVersion("3.2.32.1+5")
+			B.SetCategory("app-admin")
+			C.SetVersion("4.2.0+2")
+			C.SetCategory("sys-libs-5")
+			D.SetVersion("5.19.5+9")
+			D.SetCategory("layers")
+
+			for _, p := range []pkg.Package{A, B, C, D, Z, F} {
+				_, err := systemDB.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = inst.Uninstall(system, D)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
 	Context("Existing files", func() {
 		It("Reclaims them", func() {
 			//repo:=NewLuetSystemRepository()
@@ -912,7 +985,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -935,13 +1008,13 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade", pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
@@ -1017,7 +1090,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1037,13 +1110,13 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/upgrade_old_repo", pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -1109,7 +1182,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(3))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1123,10 +1196,10 @@ urls:
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err = GenerateRepository("test", "description", "disk", []string{tmpdir2}, 1, tmpdir2, "../../tests/fixtures/upgrade_new_repo", pkg.NewInMemoryDatabase(false))
+			repo, err = stubRepo(tmpdir2, "../../tests/fixtures/upgrade_new_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			err = repo.Write(tmpdir2, false)
+			err = repo.Write(tmpdir2, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			inst = NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})

pkg/installer/interface.go

@@ -24,7 +24,7 @@ import (
 
 type Installer interface {
 	Install(pkg.Packages, *System) error
-	Uninstall(pkg.Package, *System) error
+	Uninstall(*System, ...pkg.Package) error
 	Upgrade(s *System) error
 	Reclaim(s *System) error
 
@@ -51,7 +51,7 @@ type Repository interface {
 	SetIndex(i compiler.ArtifactIndex)
 	GetTree() tree.Builder
 	SetTree(tree.Builder)
-	Write(path string, resetRevision bool) error
+	Write(path string, resetRevision, force bool) error
 	Sync(bool) (Repository, error)
 	GetTreePath() string
 	SetTreePath(string)
@@ -70,6 +70,8 @@ type Repository interface {
 	SetPriority(int)
 	GetRepositoryFile(string) (LuetRepositoryFile, error)
 	SetRepositoryFile(string, LuetRepositoryFile)
-
+	SetName(p string)
 	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
+	GetBackend() compiler.CompilerBackend
+	SetBackend(b compiler.CompilerBackend)
 }

pkg/installer/repository.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mudler/luet/pkg/bus"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
@@ -35,7 +36,6 @@ import (
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/ghodss/yaml"
-	. "github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 )
 
@@ -46,6 +46,10 @@ const (
 
 	REPOFILE_TREE_KEY = "tree"
 	REPOFILE_META_KEY = "meta"
+
+	DiskRepositoryType   = "disk"
+	HttpRepositoryType   = "http"
+	DockerRepositoryType = "docker"
 )
 
 type LuetRepositoryFile struct {
@@ -60,6 +64,9 @@ type LuetSystemRepository struct {
 	Index           compiler.ArtifactIndex        `json:"index"`
 	Tree            tree.Builder                  `json:"-"`
 	RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"`
+	Backend         compiler.CompilerBackend      `json:"-"`
+	PushImages      bool                          `json:"-"`
+	ForcePush       bool                          `json:"-"`
 }
 
 type LuetSystemRepositorySerialized struct {
@@ -157,42 +164,89 @@ func NewDefaultMetaRepositoryFile() LuetRepositoryFile {
 	}
 }
 
+// SetFileName sets the name of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this set the filename that the client will pull
 func (f *LuetRepositoryFile) SetFileName(n string) {
 	f.FileName = n
 }
 
+// GetFileName returns the name of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this gets the filename that the client will pull
 func (f *LuetRepositoryFile) GetFileName() string {
 	return f.FileName
 }
+
+// SetCompressionType sets the compression type of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this sets the compression type that the client will use to uncompress the artifact
 func (f *LuetRepositoryFile) SetCompressionType(c compiler.CompressionImplementation) {
 	f.CompressionType = c
 }
+
+// GetCompressionType gets the compression type of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this gets the compression type that the client will use to uncompress the artifact
 func (f *LuetRepositoryFile) GetCompressionType() compiler.CompressionImplementation {
 	return f.CompressionType
 }
+
+// SetChecksums sets the checksum of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this sets the checksums that the client will use to verify the artifact
 func (f *LuetRepositoryFile) SetChecksums(c compiler.Checksums) {
 	f.Checksums = c
 }
+
+// GetChecksums gets the checksum of the repository file.
+// Each repository can ship arbitrary file that will be downloaded by the client
+// in case of need, this gets the checksums that the client will use to verify the artifact
 func (f *LuetRepositoryFile) GetChecksums() compiler.Checksums {
 	return f.Checksums
 }
 
-func GenerateRepository(name, descr, t string, urls []string, priority int, src, treeDir string, db pkg.PackageDatabase) (Repository, error) {
+// GenerateRepository generates a new repository from the given argument.
+// If the repository is of the docker type, it will also push the package images.
+// In case the repository is local, it will build the package Index
+func GenerateRepository(name, descr, t string, urls []string,
+	priority int, src string, treesDir []string, db pkg.PackageDatabase,
+	b compiler.CompilerBackend, imagePrefix string, pushImages, force bool) (Repository, error) {
 
 	tr := tree.NewInstallerRecipe(db)
-	err := tr.Load(treeDir)
-	if err != nil {
-		return nil, err
+
+	for _, treeDir := range treesDir {
+		err := tr.Load(treeDir)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	art, err := buildPackageIndex(src, tr.GetDatabase())
-	if err != nil {
-		return nil, err
+	// if !strings.HasSuffix(imagePrefix, "/") {
+	// 	imagePrefix = imagePrefix + "/"
+	// }
+
+	var art []compiler.Artifact
+	var err error
+	switch t {
+	case DiskRepositoryType, HttpRepositoryType:
+		art, err = buildPackageIndex(src, tr.GetDatabase())
+		if err != nil {
+			return nil, err
+		}
+
+	case DockerRepositoryType:
+		art, err = generatePackageImages(b, imagePrefix, src, tr.GetDatabase(), pushImages, force)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	return NewLuetSystemRepository(
+	repo := NewLuetSystemRepository(
 		config.NewLuetRepository(name, t, descr, urls, priority, true, false),
-		art, tr), nil
+		art, tr, pushImages, force)
+	repo.SetBackend(b)
+	return repo, nil
 }
 
 func NewSystemRepository(repo config.LuetRepository) Repository {
@@ -202,12 +256,14 @@ func NewSystemRepository(repo config.LuetRepository) Repository {
 	}
 }
 
-func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder) Repository {
+func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder, pushImages, force bool) Repository {
 	return &LuetSystemRepository{
 		LuetRepository:  repo,
 		Index:           art,
 		Tree:            builder,
 		RepositoryFiles: map[string]LuetRepositoryFile{},
+		PushImages:      pushImages,
+		ForcePush:       force,
 	}
 }
 
@@ -241,6 +297,71 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 	return r, err
 }
 
+func pushImage(b compiler.CompilerBackend, image string, force bool) error {
+	if b.ImageAvailable(image) && !force {
+		Debug("Image", image, "already present, skipping")
+		return nil
+	}
+	return b.Push(compiler.CompilerBackendOptions{ImageName: image})
+}
+
+func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]compiler.Artifact, error) {
+	Info("Generating docker images for packages in", imagePrefix)
+	var art []compiler.Artifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil // Skip with no errors
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		artifact, err := compiler.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				artifact.GetCompileSpec().GetPackage().HumanReadableString()))
+			return nil
+		}
+
+		packageImage := fmt.Sprintf("%s:%s", imagePrefix, artifact.GetCompileSpec().GetPackage().GetFingerPrint())
+
+		if imagePush && b.ImageAvailable(packageImage) && !force {
+			Info("Image", packageImage, "already present, skipping. use --force-push to override")
+		} else {
+			Info("Generating final image", packageImage,
+				"for package ", artifact.GetCompileSpec().GetPackage().HumanReadableString())
+			if opts, err := artifact.GenerateFinalImage(packageImage, b, true); err != nil {
+				return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+			}
+		}
+		if imagePush {
+			if err := pushImage(b, packageImage, force); err != nil {
+				return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage)
+			}
+		}
+
+		art = append(art, artifact)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
 func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) {
 
 	var art []compiler.Artifact
@@ -263,6 +384,8 @@ func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact
 		// We want to include packages that are ONLY referenced in the tree.
 		// the ones which aren't should be deleted. (TODO: by another cli command?)
 		if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				artifact.GetCompileSpec().GetPackage().HumanReadableString()))
 			return nil
 		}
 
@@ -300,6 +423,18 @@ func (r *LuetSystemRepository) GetType() string {
 func (r *LuetSystemRepository) SetType(p string) {
 	r.LuetRepository.Type = p
 }
+
+func (r *LuetSystemRepository) GetBackend() compiler.CompilerBackend {
+	return r.Backend
+}
+func (r *LuetSystemRepository) SetBackend(b compiler.CompilerBackend) {
+	r.Backend = b
+}
+
+func (r *LuetSystemRepository) SetName(p string) {
+	r.LuetRepository.Name = p
+}
+
 func (r *LuetSystemRepository) AddUrl(p string) {
 	r.LuetRepository.Urls = append(r.LuetRepository.Urls, p)
 }
@@ -390,7 +525,7 @@ func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repos
 	return repo, err
 }
 
-func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
+func (r *LuetSystemRepository) genLocalRepo(dst string, resetRevision bool) error {
 	err := os.MkdirAll(dst, os.ModePerm)
 	if err != nil {
 		return err
@@ -417,6 +552,14 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 		r.Name, r.Revision, r.LastUpdate,
 	))
 
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
+
 	// Create tree and repository file
 	archive, err := config.LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
@@ -497,31 +640,256 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 		return err
 	}
 
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
 	return nil
 }
 
+func (r *LuetSystemRepository) genDockerRepo(imagePrefix string, resetRevision, force bool) error {
+	// - Iterate over meta, build final images, push them if necessary
+	//   - while pushing, check if image already exists, and if exist push them only if --force is supplied
+	// - Generate final images for metadata and push
+
+	imageRepository := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
+
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
+
+	repoTemp, err := config.LuetCfg.GetSystem().TempDir("repo")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for repository")
+	}
+	defer os.RemoveAll(repoTemp) // clean up
+
+	if r.GetBackend().ImageAvailable(imageRepository) {
+		if err := r.GetBackend().DownloadImage(compiler.CompilerBackendOptions{ImageName: imageRepository}); err != nil {
+			return errors.Wrapf(err, "while downloading '%s'", imageRepository)
+		}
+
+		if err := r.GetBackend().ExtractRootfs(compiler.CompilerBackendOptions{ImageName: imageRepository, Destination: repoTemp}, false); err != nil {
+			return errors.Wrapf(err, "while extracting '%s'", imageRepository)
+		}
+	}
+
+	repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE)
+	if resetRevision {
+		r.Revision = 0
+	} else {
+		if _, err := os.Stat(repospec); !os.IsNotExist(err) {
+			// Read existing file for retrieve revision
+			spec, err := r.ReadSpecFile(repospec, false)
+			if err != nil {
+				return err
+			}
+			r.Revision = spec.GetRevision()
+		}
+	}
+	r.Revision++
+
+	Info(fmt.Sprintf(
+		"For repository %s creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imageRepository,
+	})
+
+	// Create tree and repository file
+	archive, err := config.LuetCfg.GetSystem().TempDir("archive")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for archive")
+	}
+	defer os.RemoveAll(archive) // clean up
+	err = r.GetTree().Save(archive)
+	if err != nil {
+		return errors.Wrap(err, "Error met while saving the tree")
+	}
+
+	treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY)
+	if err != nil {
+		treeFile = NewDefaultTreeRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
+	}
+
+	a := compiler.NewPackageArtifact(filepath.Join(repoTemp, treeFile.GetFileName()))
+	a.SetCompressionType(treeFile.GetCompressionType())
+	err = a.Compress(archive, 1)
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating package archive")
+	}
+
+	// Update the tree name with the name created by compression selected.
+	treeFile.SetFileName(a.GetFileName())
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	treeFile.SetChecksums(a.GetChecksums())
+	r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
+
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	treeArchive, err := compiler.CreateArtifactForFile(a.GetPath())
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	imageTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName())
+	Debug("Generating image", imageTree)
+	if opts, err := treeArchive.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
+	}
+	if r.PushImages {
+		if err := pushImage(r.GetBackend(), imageTree, true); err != nil {
+			return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree)
+		}
+	}
+
+	// Create Metadata struct and serialized repository
+	meta, serialized := r.Serialize()
+
+	// Create metadata file and repository file
+	metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaTmpDir) // clean up
+
+	metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY)
+	if err != nil {
+		metaFile = NewDefaultMetaRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	}
+
+	repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE)
+	// Create repository.meta.yaml file
+	err = meta.WriteFile(repoMetaSpec)
+	if err != nil {
+		return err
+	}
+
+	// create temp dir for metafile
+	metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaDir) // clean up
+
+	a = compiler.NewPackageArtifact(filepath.Join(metaDir, metaFile.GetFileName()))
+	a.SetCompressionType(metaFile.GetCompressionType())
+	err = a.Compress(metaTmpDir, 1)
+	if err != nil {
+		return errors.Wrap(err, "Error met while archiving repository metadata")
+	}
+
+	metaFile.SetFileName(a.GetFileName())
+	r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for metadata")
+	}
+	metaFile.SetChecksums(a.GetChecksums())
+
+	// Files are downloaded as-is from docker images
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	metaArchive, err := compiler.CreateArtifactForFile(a.GetPath())
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	imageMetaTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName())
+	if opts, err := metaArchive.GenerateFinalImage(imageMetaTree, r.GetBackend(), false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+	}
+	if r.PushImages {
+		if err := pushImage(r.GetBackend(), imageMetaTree, true); err != nil {
+			return errors.Wrapf(err, "Failed while pushing image: '%s'", imageMetaTree)
+		}
+	}
+	data, err := yaml.Marshal(serialized)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(repospec, data, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	tempRepoFile := filepath.Join(metaDir, REPOSITORY_SPECFILE+".tar")
+	if err := helpers.Tar(repospec, tempRepoFile); err != nil {
+		return errors.Wrap(err, "Error met while archiving repository file")
+	}
+
+	a = compiler.NewPackageArtifact(tempRepoFile)
+	imageRepo := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
+	if opts, err := a.GenerateFinalImage(imageRepo, r.GetBackend(), false); err != nil {
+		return errors.Wrap(err, "Failed generating repository image"+opts.ImageName)
+	}
+	if r.PushImages {
+		if err := pushImage(r.GetBackend(), imageRepo, true); err != nil {
+			return errors.Wrapf(err, "Failed while pushing image: '%s'", imageRepo)
+		}
+	}
+
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imagePrefix,
+	})
+	return nil
+}
+
+// Write writes the repository metadata to the supplied destination
+func (r *LuetSystemRepository) Write(dst string, resetRevision, force bool) error {
+	switch r.GetType() {
+	case DiskRepositoryType, HttpRepositoryType:
+		return r.genLocalRepo(dst, resetRevision)
+	case DockerRepositoryType:
+		return r.genDockerRepo(dst, resetRevision, force)
+	}
+	return errors.New("invalid repository type")
+}
+
 func (r *LuetSystemRepository) Client() Client {
 	switch r.GetType() {
-	case "disk":
+	case DiskRepositoryType:
 		return client.NewLocalClient(client.RepoData{Urls: r.GetUrls()})
-	case "http":
+	case HttpRepositoryType:
 		return client.NewHttpClient(
 			client.RepoData{
 				Urls:           r.GetUrls(),
 				Authentication: r.GetAuthentication(),
 			})
-	}
 
+	case DockerRepositoryType:
+		return client.NewDockerClient(
+			client.RepoData{
+				Urls:           r.GetUrls(),
+				Authentication: r.GetAuthentication(),
+			})
+	}
 	return nil
 }
+
 func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	var repoUpdated bool = false
 	var treefs, metafs string
+	aurora := GetAurora()
 
 	Debug("Sync of the repository", r.Name, "in progress...")
 	c := r.Client()
 	if c == nil {
-		return nil, errors.New("No client could be generated from repository.")
+		return nil, errors.New("no client could be generated from repository")
 	}
 
 	// Retrieve remote repository.yaml for retrieve revision and date
@@ -535,7 +903,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	if err != nil {
 		return nil, err
 	}
-	// Remove temporary file that contains repository.html.
+	// Remove temporary file that contains repository.yaml
 	// Example: /tmp/HttpClient236052003
 	defer os.RemoveAll(file)
 
@@ -565,15 +933,10 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 		}
-		// Note: If we always remove them, later on, no other structure can access
-		// to the tree for e.g. to retrieve finalizers
-
 		metafs, err = config.LuetCfg.GetSystem().TempDir("metafs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met whilte creating tempdir for metafs")
 		}
-		//defer os.RemoveAll(metafs)
-
 	}
 
 	// POST: treeFile and metaFile are present. I check this inside
@@ -584,17 +947,18 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	if !repoUpdated {
 
 		// Get Tree
-		a := compiler.NewPackageArtifact(treeFile.GetFileName())
-		artifactTree, err := c.DownloadArtifact(a)
+		downloadedTreeFile, err := c.DownloadFile(treeFile.GetFileName())
 		if err != nil {
 			return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName())
 		}
-		defer os.Remove(artifactTree.GetPath())
+		defer os.Remove(downloadedTreeFile)
 
-		artifactTree.SetChecksums(treeFile.GetChecksums())
-		artifactTree.SetCompressionType(treeFile.GetCompressionType())
+		// Treat the file as artifact, in order to verify it
+		treeFileArtifact := compiler.NewPackageArtifact(downloadedTreeFile)
+		treeFileArtifact.SetChecksums(treeFile.GetChecksums())
+		treeFileArtifact.SetCompressionType(treeFile.GetCompressionType())
 
-		err = artifactTree.Verify()
+		err = treeFileArtifact.Verify()
 		if err != nil {
 			return nil, errors.Wrap(err, "Tree integrity check failure")
 		}
@@ -602,17 +966,17 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
 
 		// Get Repository Metadata
-		a = compiler.NewPackageArtifact(metaFile.GetFileName())
-		artifactMeta, err := c.DownloadArtifact(a)
+		downloadedMeta, err := c.DownloadFile(metaFile.GetFileName())
 		if err != nil {
 			return nil, errors.Wrap(err, "While downloading "+metaFile.GetFileName())
 		}
-		defer os.Remove(artifactMeta.GetPath())
+		defer os.Remove(downloadedMeta)
 
-		artifactMeta.SetChecksums(metaFile.GetChecksums())
-		artifactMeta.SetCompressionType(metaFile.GetCompressionType())
+		metaFileArtifact := compiler.NewPackageArtifact(downloadedMeta)
+		metaFileArtifact.SetChecksums(metaFile.GetChecksums())
+		metaFileArtifact.SetCompressionType(metaFile.GetCompressionType())
 
-		err = artifactMeta.Verify()
+		err = metaFileArtifact.Verify()
 		if err != nil {
 			return nil, errors.Wrap(err, "Metadata integrity check failure")
 		}
@@ -632,7 +996,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		}
 		Debug("Decompress tree of the repository " + r.Name + "...")
 
-		err = artifactTree.Unpack(treefs, true)
+		err = treeFileArtifact.Unpack(treefs, true)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while unpacking tree")
 		}
@@ -640,7 +1004,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		// FIXME: It seems that tar with only one file doesn't create destination
 		//       directory. I create directory directly for now.
 		os.MkdirAll(metafs, os.ModePerm)
-		err = artifactMeta.Unpack(metafs, true)
+		err = metaFileArtifact.Unpack(metafs, true)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while unpacking metadata")
 		}
@@ -648,9 +1012,10 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64)
 
 		InfoC(
-			Bold(Red(":house: Repository "+repo.GetName()+" revision: ")).String() +
-				Bold(Green(repo.GetRevision())).String() + " - " +
-				Bold(Green(time.Unix(tsec, 0).String())).String(),
+			aurora.Bold(
+				aurora.Red(":house: Repository "+repo.GetName()+" revision: ")).String() +
+				aurora.Bold(aurora.Green(repo.GetRevision())).String() + " - " +
+				aurora.Bold(aurora.Green(time.Unix(tsec, 0).String())).String(),
 		)
 
 	} else {
@@ -681,10 +1046,15 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	repo.SetAuthentication(r.GetAuthentication())
 	repo.SetType(r.GetType())
 	repo.SetPriority(r.GetPriority())
+	repo.SetName(r.GetName())
 	InfoC(
-		Bold(Yellow(":information_source: Repository "+repo.GetName()+" priority: ")).String() +
-			Bold(Green(repo.GetPriority())).String() + " - type " +
-			Bold(Green(repo.GetType())).String(),
+		aurora.Yellow(":information_source:").String() +
+			aurora.Magenta("Repository: ").String() +
+			aurora.Green(aurora.Bold(repo.GetName()).String()).String() +
+			aurora.Magenta(" Priority: ").String() +
+			aurora.Bold(aurora.Green(repo.GetPriority())).String() +
+			aurora.Magenta(" Type: ").String() +
+			aurora.Bold(aurora.Green(repo.GetType())).String(),
 	)
 	return repo, nil
 }
@@ -796,13 +1166,11 @@ PACKAGE:
 				c, err := r.GetTree().GetDatabase().FindPackageCandidate(pack)
 				// If FindPackageCandidate returns the same package, it means it couldn't find one.
 				// Skip this repository and keep looking.
-				if c.String() == pack.String() {
+				if err != nil { //c.String() == pack.String() {
 					continue REPOSITORY
 				}
-				if err == nil {
-					matches = append(matches, c)
-					continue PACKAGE
-				}
+				matches = append(matches, c)
+				continue PACKAGE
 			} else {
 				// If it's not a selector, just append it
 				matches = append(matches, pack)

pkg/installer/repository_test.go

@@ -19,20 +19,36 @@ import (
 
 	//	. "github.com/mudler/luet/pkg/installer"
 
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
 	"github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	config "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
+func dockerStubRepo(tmpdir, tree, image string, push, force bool) (installer.Repository, error) {
+	return GenerateRepository(
+		"test",
+		"description",
+		"docker",
+		[]string{image},
+		1,
+		tmpdir,
+		[]string{tree},
+		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force)
+}
+
 var _ = Describe("Repository", func() {
 	Context("Generation", func() {
 		It("Generate repository metadata", func() {
@@ -48,7 +64,7 @@ var _ = Describe("Repository", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -59,8 +75,8 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			compiler.SetConcurrency(1)
@@ -83,13 +99,13 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
@@ -116,11 +132,11 @@ var _ = Describe("Repository", func() {
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -132,8 +148,8 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
@@ -165,13 +181,13 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
@@ -215,6 +231,166 @@ urls:
 			Expect(matches).To(Equal([]PackageMatch{{Repo: repo1, Package: package1}}))
 
 		})
+	})
+	Context("Docker repository", func() {
+		repoImage := os.Getenv("UNIT_TEST_DOCKER_IMAGE_REPOSITORY")
 
+		BeforeEach(func() {
+			if repoImage == "" {
+				Skip("UNIT_TEST_DOCKER_IMAGE_REPOSITORY not specified")
+			}
+		})
+
+		It("generates images", func() {
+			b := backend.NewSimpleDockerBackend()
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			localcompiler.SetConcurrency(1)
+
+			artifact, err := localcompiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/buildable", repoImage, true, true)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(repoImage, false, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "tree.tar.gz"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.meta.yaml.tar"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.yaml"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "b-test-1.0"))).To(BeTrue())
+
+			extracted, err := ioutil.TempDir("", "extracted")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(extracted) // clean up
+
+			c := repo.Client()
+
+			f, err := c.DownloadFile("repository.yaml")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(f)).To(ContainSubstring("name: test"))
+
+			a, err := c.DownloadArtifact(&compiler.PackageArtifact{
+				Path: "test.tar",
+				CompileSpec: &compiler.LuetCompilationSpec{
+					Package: &pkg.DefaultPackage{
+						Name:     "b",
+						Category: "test",
+						Version:  "1.0",
+					},
+				},
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(a.Unpack(extracted, false)).ToNot(HaveOccurred())
+			Expect(helpers.Read(filepath.Join(extracted, "test6"))).To(Equal("artifact6\n"))
+		})
+
+		It("generates images of virtual packages", func() {
+			b := backend.NewSimpleDockerBackend()
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/virtuals")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(5))
+
+			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.99"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			localcompiler.SetConcurrency(1)
+
+			artifact, err := localcompiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/virtuals", repoImage, true, true)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(repoImage, false, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "tree.tar.gz"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.meta.yaml.tar"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.yaml"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "a-test-1.99"))).To(BeTrue())
+
+			extracted, err := ioutil.TempDir("", "extracted")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(extracted) // clean up
+
+			c := repo.Client()
+
+			f, err := c.DownloadFile("repository.yaml")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(f)).To(ContainSubstring("name: test"))
+
+			a, err := c.DownloadArtifact(&compiler.PackageArtifact{
+				Path: "test.tar",
+				CompileSpec: &compiler.LuetCompilationSpec{
+					Package: &pkg.DefaultPackage{
+						Name:     "a",
+						Category: "test",
+						Version:  "1.99",
+					},
+				},
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(a.Unpack(extracted, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.DirectoryIsEmpty(extracted)).To(BeFalse())
+			content, err := ioutil.ReadFile(filepath.Join(extracted, ".virtual"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(content)).To(Equal(""))
+		})
 	})
 })

pkg/installer/system.go

@@ -1,7 +1,11 @@
 package installer
 
 import (
+	"github.com/hashicorp/go-multierror"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
 )
 
 type System struct {
@@ -12,3 +16,38 @@ type System struct {
 func (s *System) World() (pkg.Packages, error) {
 	return s.Database.World(), nil
 }
+
+type templatedata map[string]interface{}
+
+func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
+	var errs error
+	executedFinalizer := map[string]bool{}
+	for _, p := range packs {
+		if helpers.Exists(p.Rel(tree.FinalizerFile)) {
+			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile), "")
+			if err != nil {
+				Warning("Failed rendering finalizer for ", p.HumanReadableString(), err.Error())
+				errs = multierror.Append(errs, err)
+				continue
+			}
+
+			if _, exists := executedFinalizer[p.GetFingerPrint()]; !exists {
+				executedFinalizer[p.GetFingerPrint()] = true
+				Info("Executing finalizer for " + p.HumanReadableString())
+				finalizer, err := NewLuetFinalizerFromYaml([]byte(out))
+				if err != nil {
+					Warning("Failed reading finalizer for ", p.HumanReadableString(), err.Error())
+					errs = multierror.Append(errs, err)
+					continue
+				}
+				err = finalizer.RunInstall(s)
+				if err != nil {
+					Warning("Failed running finalizer for ", p.HumanReadableString(), err.Error())
+					errs = multierror.Append(errs, err)
+					continue
+				}
+			}
+		}
+	}
+	return errs
+}

pkg/logger/logger.go

@@ -3,7 +3,12 @@ package logger
 import (
 	"fmt"
 	"os"
+	"path"
+	"regexp"
+	"runtime"
+	"strings"
 
+	"sync"
 	. "github.com/mudler/luet/pkg/config"
 
 	"github.com/briandowns/spinner"
@@ -15,7 +20,8 @@ import (
 
 var s *spinner.Spinner = nil
 var z *zap.Logger = nil
-
+var aurora Aurora = nil
+var spinnerLock = sync.Mutex{}
 func NewSpinner() {
 	if s == nil {
 		s = spinner.New(
@@ -24,6 +30,32 @@ func NewSpinner() {
 	}
 }
 
+func InitAurora() {
+	if aurora == nil {
+		aurora = NewAurora(LuetCfg.GetLogging().Color)
+	}
+}
+
+func GetAurora() Aurora {
+	return aurora
+}
+
+func Ask() bool {
+	var input string
+
+	Info("Do you want to continue with this operation? [y/N]: ")
+	_, err := fmt.Scanln(&input)
+	if err != nil {
+		return false
+	}
+	input = strings.ToLower(input)
+
+	if input == "y" || input == "yes" {
+		return true
+	}
+	return false
+}
+
 func ZapLogger() error {
 	var err error
 	if z == nil {
@@ -53,6 +85,8 @@ func ZapLogger() error {
 }
 
 func Spinner(i int) {
+	spinnerLock.Lock()
+	defer spinnerLock.Unlock()
 	var confLevel int
 	if LuetCfg.GetGeneral().Debug {
 		confLevel = 3
@@ -66,27 +100,31 @@ func Spinner(i int) {
 		i = 43
 	}
 
-	if !s.Active() {
+	if s != nil && !s.Active() {
 		//	s.UpdateCharSet(spinner.CharSets[i])
 		s.Start() // Start the spinner
 	}
 }
 
 func SpinnerText(suffix, prefix string) {
-	s.Lock()
-	defer s.Unlock()
-	if LuetCfg.GetGeneral().Debug {
-		fmt.Println(fmt.Sprintf("%s %s",
-			Bold(Cyan(prefix)).String(),
-			Bold(Magenta(suffix)).BgBlack().String(),
-		))
-	} else {
-		s.Suffix = Bold(Magenta(suffix)).BgBlack().String()
-		s.Prefix = Bold(Cyan(prefix)).String()
+	if s != nil {
+		s.Lock()
+		defer s.Unlock()
+		if LuetCfg.GetGeneral().Debug {
+			fmt.Println(fmt.Sprintf("%s %s",
+				Bold(Cyan(prefix)).String(),
+				Bold(Magenta(suffix)).BgBlack().String(),
+			))
+		} else {
+			s.Suffix = Bold(Magenta(suffix)).BgBlack().String()
+			s.Prefix = Bold(Cyan(prefix)).String()
+		}
 	}
 }
 
 func SpinnerStop() {
+	spinnerLock.Lock()
+	defer spinnerLock.Unlock()
 	var confLevel int
 	if LuetCfg.GetGeneral().Debug {
 		confLevel = 3
@@ -96,7 +134,9 @@ func SpinnerStop() {
 	if 2 > confLevel {
 		return
 	}
-	s.Stop()
+	if s != nil {
+		s.Stop()
+	}
 }
 
 func level2Number(level string) int {
@@ -138,7 +178,7 @@ func level2AtomicLevel(level string) zap.AtomicLevel {
 	}
 }
 
-func msg(level string, withoutColor bool, msg ...interface{}) {
+func Msg(level string, withoutColor, ln bool, msg ...interface{}) {
 	var message string
 	var confLevel, msgLevel int
 
@@ -158,58 +198,73 @@ func msg(level string, withoutColor bool, msg ...interface{}) {
 
 	var levelMsg string
 
-	if withoutColor {
+	if withoutColor || !LuetCfg.GetLogging().Color {
 		levelMsg = message
 	} else {
 		switch level {
 		case "warning":
-			levelMsg = Bold(Yellow(":construction: " + message)).BgBlack().String()
+			levelMsg = Yellow(":construction: warning" + message).BgBlack().String()
 		case "debug":
 			levelMsg = White(message).BgBlack().String()
 		case "info":
-			levelMsg = Bold(White(message)).BgBlack().String()
+			levelMsg = message
 		case "error":
-			levelMsg = Bold(Red(":bomb: " + message + ":fire:")).BgBlack().String()
+			levelMsg = Red(message).String()
 		}
 	}
 
-	levelMsg = emoji.Sprint(levelMsg)
+	if LuetCfg.GetLogging().EnableEmoji {
+		levelMsg = emoji.Sprint(levelMsg)
+	} else {
+		re := regexp.MustCompile(`[:][\w]+[:]`)
+		levelMsg = re.ReplaceAllString(levelMsg, "")
+	}
 
 	if z != nil {
 		log2File(level, message)
 	}
 
-	fmt.Println(levelMsg)
+	if ln {
+		fmt.Println(levelMsg)
+	} else {
+		fmt.Print(levelMsg)
+	}
+
 }
 
 func Warning(mess ...interface{}) {
-	msg("warning", false, mess...)
+	Msg("warning", false, true, mess...)
 	if LuetCfg.GetGeneral().FatalWarns {
 		os.Exit(2)
 	}
 }
 
 func Debug(mess ...interface{}) {
-	msg("debug", false, mess...)
+	pc, file, line, ok := runtime.Caller(1)
+	if ok {
+		mess = append([]interface{}{fmt.Sprintf("DEBUG (%s:#%d:%v)",
+			path.Base(file), line, runtime.FuncForPC(pc).Name())}, mess...)
+	}
+	Msg("debug", false, true, mess...)
 }
 
 func DebugC(mess ...interface{}) {
-	msg("debug", true, mess...)
+	Msg("debug", true, true, mess...)
 }
 
 func Info(mess ...interface{}) {
-	msg("info", false, mess...)
+	Msg("info", false, true, mess...)
 }
 
 func InfoC(mess ...interface{}) {
-	msg("info", true, mess...)
+	Msg("info", true, true, mess...)
 }
 
 func Error(mess ...interface{}) {
-	msg("error", false, mess...)
+	Msg("error", false, true, mess...)
 }
 
 func Fatal(mess ...interface{}) {
-	Error(mess)
+	Error(mess...)
 	os.Exit(1)
 }

pkg/package/annotations.go

@@ -0,0 +1,23 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+type AnnotationKey string
+
+const (
+	ConfigProtectAnnnotation AnnotationKey = "config_protect"
+)

pkg/package/database.go

@@ -28,6 +28,10 @@ type PackageDatabase interface {
 }
 
 type PackageSet interface {
+	Clone(PackageDatabase) error
+	Copy() (PackageDatabase, error)
+
+	GetRevdeps(p Package) (Packages, error)
 	GetPackages() []string //Ids
 	CreatePackage(pkg Package) (string, error)
 	GetPackage(ID string) (Package, error)

pkg/package/database_bench_test.go

@@ -0,0 +1,116 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	"io/ioutil"
+	"os"
+	"strconv"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Database  benchmark", func() {
+
+	Context("BoltDB", func() {
+
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		var db PackageSet
+
+		BeforeEach(func() {
+
+			tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			db = NewBoltDatabase(tmpfile.Name())
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+
+		Measure("it should be fast in computing world from a 50000 dataset", func(b Benchmarker) {
+			for i := 0; i < 50000; i++ {
+				a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			runtime := b.Time("runtime", func() {
+				packs := db.World()
+				Expect(len(packs)).To(Equal(50000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 30), "World() shouldn't take too long.")
+
+		}, 1)
+
+		Measure("it should be fast in computing world from a 100000 dataset", func(b Benchmarker) {
+			for i := 0; i < 100000; i++ {
+				a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			runtime := b.Time("runtime", func() {
+				packs := db.World()
+				Expect(len(packs)).To(Equal(100000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 30), "World() shouldn't take too long.")
+
+		}, 1)
+	})
+
+	Context("InMemory", func() {
+
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		var db PackageSet
+
+		BeforeEach(func() {
+
+			tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			db = NewInMemoryDatabase(false)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+
+		Measure("it should be fast in computing world from a 100000 dataset", func(b Benchmarker) {
+
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 100000; i++ {
+					a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					_, err := db.CreatePackage(a)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				packs := db.World()
+				Expect(len(packs)).To(Equal(100000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 10), "World() shouldn't take too long.")
+
+		}, 2)
+	})
+})

pkg/package/database_boltdb.go

@@ -47,6 +47,14 @@ func NewBoltDatabase(path string) PackageDatabase {
 	return &BoltDatabase{Path: path, ProvidesDatabase: map[string]map[string]Package{}}
 }
 
+func (db *BoltDatabase) Clone(to PackageDatabase) error {
+	return clone(db, to)
+}
+
+func (db *BoltDatabase) Copy() (PackageDatabase, error) {
+	return copy(db)
+}
+
 func (db *BoltDatabase) Get(s string) (string, error) {
 	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
@@ -86,6 +94,17 @@ func (db *BoltDatabase) Retrieve(ID string) ([]byte, error) {
 	return enc, nil
 }
 
+// GetRevdeps uses a new inmemory db to calcuate revdeps
+// TODO: Have a memory instance for boltdb, so we don't compute each time we get called
+// as this is REALLY expensive. But we don't perform usually those operations in a file db.
+func (db *BoltDatabase) GetRevdeps(p Package) (Packages, error) {
+	memory, err := db.Copy()
+	if err != nil {
+		return nil, errors.New("Failed copying bolt db to memory")
+	}
+	return memory.GetRevdeps(p)
+}
+
 func (db *BoltDatabase) FindPackage(tofind Package) (Package, error) {
 	// Provides: Return the replaced package here
 	if provided, err := db.getProvide(tofind); err == nil {
@@ -162,26 +181,17 @@ func (db *BoltDatabase) GetAllPackages(packages chan Package) error {
 		return err
 	}
 	defer bolt.Close()
-	// Fetching records one by one (useful when the bucket contains a lot of records)
-	//query := bolt.Select()
-
-	var packs []Package
+	var packs []DefaultPackage
 	err = bolt.All(&packs)
 	if err != nil {
 		return err
 	}
 
 	for _, r := range packs {
-		packages <- r
+		packages <- &r
 	}
 
 	return nil
-
-	// return query.Each(new(DefaultPackage), func(record interface{}) error {
-	// 	u := record.(*DefaultPackage)
-	// 	packages <- u
-	// 	return err
-	// })
 }
 
 // Encode encodes the package to string.
@@ -316,33 +326,41 @@ func (db *BoltDatabase) RemovePackage(p Package) error {
 }
 
 func (db *BoltDatabase) World() Packages {
+	var packs []DefaultPackage
 
-	var all []Package
-	// FIXME: This should all be locked in the db - for now forbid the solver to be run in threads.
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err == nil {
-			all = append(all, pack)
-		}
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return Packages([]Package{})
 	}
-	return Packages(all)
+	defer bolt.Close()
+	err = bolt.All(&packs)
+	if err != nil {
+		return Packages([]Package{})
+	}
+	models := make([]Package, len(packs))
+	for i, _ := range packs {
+		models[i] = &packs[i]
+	}
+
+	return Packages(models)
 }
 
 func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
 
 	required, err := db.FindPackage(p)
 	if err != nil {
-
+		err = nil
 		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
 		packages, err := p.Expand(db)
 		//	Info("Expanded", packages, err)
 		if err != nil || len(packages) == 0 {
 			required = p
+			err = errors.Wrap(err, "Candidate not found")
 		} else {
 			required = packages.Best(nil)
 
 		}
-		return required, nil
+		return required, err
 		//required = &DefaultPackage{Name: "test"}
 	}
 
@@ -353,10 +371,22 @@ func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
 // FindPackages return the list of the packages beloging to cat/name  (any versions in requested range)
 // FIXME: Optimize, see inmemorydb
 func (db *BoltDatabase) FindPackages(p Package) (Packages, error) {
+	if !p.IsSelector() {
+		pack, err := db.FindPackage(p)
+		if err != nil {
+			return []Package{}, err
+		}
+		return []Package{pack}, nil
+	}
+
 	// Provides: Treat as the replaced package here
 	if provided, err := db.getProvide(p); err == nil {
 		p = provided
+		if !provided.IsSelector() {
+			return Packages{provided}, nil
+		}
 	}
+
 	var versionsInWorld []Package
 	for _, w := range db.World() {
 		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
@@ -376,6 +406,11 @@ func (db *BoltDatabase) FindPackages(p Package) (Packages, error) {
 
 // FindPackageVersions return the list of the packages beloging to cat/name
 func (db *BoltDatabase) FindPackageVersions(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+
 	var versionsInWorld []Package
 	for _, w := range db.World() {
 		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
@@ -390,11 +425,7 @@ func (db *BoltDatabase) FindPackageVersions(p Package) (Packages, error) {
 func (db *BoltDatabase) FindPackageLabel(labelKey string) (Packages, error) {
 	var ans []Package
 
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err != nil {
-			return ans, err
-		}
+	for _, pack := range db.World() {
 		if pack.HasLabel(labelKey) {
 			ans = append(ans, pack)
 		}
@@ -410,11 +441,7 @@ func (db *BoltDatabase) FindPackageLabelMatch(pattern string) (Packages, error)
 		return nil, errors.New("Invalid regex " + pattern + "!")
 	}
 
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err != nil {
-			return ans, err
-		}
+	for _, pack := range db.World() {
 		if pack.MatchLabel(re) {
 			ans = append(ans, pack)
 		}
@@ -431,12 +458,7 @@ func (db *BoltDatabase) FindPackageMatch(pattern string) (Packages, error) {
 		return nil, errors.New("Invalid regex " + pattern + "!")
 	}
 
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err != nil {
-			return ans, err
-		}
-
+	for _, pack := range db.World() {
 		if re.MatchString(pack.HumanReadableString()) {
 			ans = append(ans, pack)
 		}

pkg/package/database_boltdb_test.go

@@ -0,0 +1,287 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or ItNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	"io/ioutil"
+	"os"
+	"regexp"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("BoltDB Database", func() {
+
+	tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+	defer os.Remove(tmpfile.Name()) // clean up
+	var db PackageDatabase
+
+	BeforeEach(func() {
+		tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		db = NewBoltDatabase(tmpfile.Name())
+	})
+	Context("Simple package", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		It("Find packages", func() {
+			ID, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.GetPackage(ID)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack).To(Equal(a))
+			ids := db.GetPackages()
+
+			Expect(ids).To(Equal([]string{"1"}))
+
+			pack, err = db.FindPackage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Expands correctly", func() {
+
+			a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+			re := regexp.MustCompile("project[0-9][=].*")
+			for _, p := range []Package{a1, a11, a01} {
+				_, err := db.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			lst, err := a.Expand(db)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(lst).To(ContainElement(a11))
+			Expect(lst).To(ContainElement(a1))
+			Expect(lst).ToNot(ContainElement(a01))
+			Expect(len(lst)).To(Equal(2))
+			p := lst.Best(nil)
+			Expect(p).To(Equal(a11))
+			// Test annotation with null map
+			Expect(a.MatchAnnotation(re)).To(Equal(false))
+		})
+
+		It("Find best package candidate", func() {
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+
+		})
+
+		It("Find specific package candidate", func() {
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Provides replaces definitions", func() {
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+			a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+			Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+
+			s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackage(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+		})
+
+		Context("Provides", func() {
+
+			It("replaces definitions", func() {
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				pack, err := db.FindPackage(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(pack).To(Equal(a3))
+			})
+
+			It("replaces definitions", func() {
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(a3))
+			})
+
+			It("replaces definitions", func() {
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of unexisting packages", func() {
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of a required package", func() {
+
+				c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = db.CreatePackage(c)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			When("Searching with selectors", func() {
+				It("replaces definitions of a required package", func() {
+
+					c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+					z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+					z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+					Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+					_, err := db.CreatePackage(z)
+					Expect(err).ToNot(HaveOccurred())
+					_, err = db.CreatePackage(c)
+					Expect(err).ToNot(HaveOccurred())
+
+					s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					packs, err := db.FindPackages(s)
+					Expect(err).ToNot(HaveOccurred())
+					Expect(packs).To(ContainElement(z))
+				})
+			})
+
+		})
+
+	})
+
+})

pkg/package/database_common.go

@@ -0,0 +1,38 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+import "github.com/pkg/errors"
+
+func clone(src, dst PackageDatabase) error {
+	for _, i := range src.World() {
+		_, err := dst.CreatePackage(i)
+		if err != nil {
+			return errors.Wrap(err, "Failed create package "+i.HumanReadableString())
+		}
+	}
+	return nil
+}
+
+func copy(src PackageDatabase) (PackageDatabase, error) {
+	dst := NewInMemoryDatabase(false)
+
+	if err := clone(src, dst); err != nil {
+		return dst, errors.Wrap(err, "Failed create temporary in-memory db")
+	}
+
+	return dst, nil
+}

pkg/package/database_mem.go

@@ -31,6 +31,8 @@ var DBInMemoryInstance = &InMemoryDatabase{
 	Database:         map[string]string{},
 	CacheNoVersion:   map[string]map[string]interface{}{},
 	ProvidesDatabase: map[string]map[string]Package{},
+	RevDepsDatabase:  map[string]map[string]Package{},
+	cached:           map[string]interface{}{},
 }
 
 type InMemoryDatabase struct {
@@ -39,6 +41,8 @@ type InMemoryDatabase struct {
 	FileDatabase     map[string][]string
 	CacheNoVersion   map[string]map[string]interface{}
 	ProvidesDatabase map[string]map[string]Package
+	RevDepsDatabase  map[string]map[string]Package
+	cached           map[string]interface{}
 }
 
 func NewInMemoryDatabase(singleton bool) PackageDatabase {
@@ -50,6 +54,8 @@ func NewInMemoryDatabase(singleton bool) PackageDatabase {
 			Database:         map[string]string{},
 			CacheNoVersion:   map[string]map[string]interface{}{},
 			ProvidesDatabase: map[string]map[string]Package{},
+			RevDepsDatabase:  map[string]map[string]Package{},
+			cached:           map[string]interface{}{},
 		}
 	}
 	return DBInMemoryInstance
@@ -125,6 +131,47 @@ func (db *InMemoryDatabase) GetAllPackages(packages chan Package) error {
 	return nil
 }
 
+func (db *InMemoryDatabase) getRevdeps(p Package, visited map[string]interface{}) (Packages, error) {
+	var versionsInWorld Packages
+	if _, ok := visited[p.HumanReadableString()]; ok {
+		return versionsInWorld, nil
+	}
+	visited[p.HumanReadableString()] = true
+
+	var res Packages
+	packs, err := db.FindPackages(p)
+	if err != nil {
+		return res, err
+	}
+	for _, pp := range packs {
+		//	db.Lock()
+		list := db.RevDepsDatabase[pp.GetFingerPrint()]
+		//	db.Unlock()
+		for _, revdep := range list {
+			dep, err := db.FindPackage(revdep)
+			if err != nil {
+				return res, err
+			}
+			res = append(res, dep)
+
+			packs, err := db.getRevdeps(dep, visited)
+			if err != nil {
+				return res, err
+			}
+			res = append(res, packs...)
+
+		}
+	}
+	return res.Unique(), nil
+}
+
+// GetRevdeps returns the package reverse dependencies,
+// matching also selectors in versions (>, <, >=, <=)
+// TODO: Code should use db explictly
+func (db *InMemoryDatabase) GetRevdeps(p Package) (Packages, error) {
+	return db.getRevdeps(p, make(map[string]interface{}))
+}
+
 // Encode encodes the package to string.
 // It returns an ID which can be used to retrieve the package later on.
 func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
@@ -143,9 +190,25 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 		return "", err
 	}
 
+	db.populateCaches(pd)
+
+	return ID, nil
+}
+
+func (db *InMemoryDatabase) populateCaches(p Package) {
+	pd, _ := p.(*DefaultPackage)
+
 	// Create extra cache between package -> []versions
 	db.Lock()
-	defer db.Unlock()
+	if db.cached == nil {
+		db.cached = map[string]interface{}{}
+	}
+
+	if _, ok := db.cached[p.GetFingerPrint()]; ok {
+		db.Unlock()
+		return
+	}
+	db.cached[p.GetFingerPrint()] = nil
 
 	// Provides: Store package provides, we will reuse this when walking deps
 	for _, provide := range pd.Provides {
@@ -157,21 +220,71 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 		db.ProvidesDatabase[provide.GetPackageName()][provide.GetVersion()] = p
 	}
 
-	_, ok = db.CacheNoVersion[p.GetPackageName()]
+	_, ok := db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
 		db.CacheNoVersion[p.GetPackageName()] = make(map[string]interface{})
 	}
 	db.CacheNoVersion[p.GetPackageName()][p.GetVersion()] = nil
 
-	return ID, nil
+	// Updating Revdeps
+	// Given that when we populate the cache we don't have the full db at hand
+	// We cycle over reverse dependency of a package to update their entry if they are matching
+	// the version selector
+	toUpdate, ok := db.RevDepsDatabase[pd.GetPackageName()]
+	if ok {
+		for _, pp := range toUpdate {
+			for _, re := range pp.GetRequires() {
+				if match, _ := pd.VersionMatchSelector(re.GetVersion(), nil); match {
+					_, ok = db.RevDepsDatabase[pd.GetFingerPrint()]
+					if !ok {
+						db.RevDepsDatabase[pd.GetFingerPrint()] = make(map[string]Package)
+					}
+					db.RevDepsDatabase[pd.GetFingerPrint()][pp.GetFingerPrint()] = pp
+				}
+			}
+		}
+	}
+	db.Unlock()
+
+	for _, re := range pd.GetRequires() {
+		packages, _ := db.FindPackages(re)
+		db.Lock()
+
+		for _, pa := range packages {
+			_, ok := db.RevDepsDatabase[pa.GetFingerPrint()]
+			if !ok {
+				db.RevDepsDatabase[pa.GetFingerPrint()] = make(map[string]Package)
+			}
+			db.RevDepsDatabase[pa.GetFingerPrint()][pd.GetFingerPrint()] = pd
+			_, ok = db.RevDepsDatabase[pa.GetPackageName()]
+			if !ok {
+				db.RevDepsDatabase[pa.GetPackageName()] = make(map[string]Package)
+			}
+			db.RevDepsDatabase[pa.GetPackageName()][pd.GetPackageName()] = pd
+		}
+		_, ok := db.RevDepsDatabase[re.GetFingerPrint()]
+		if !ok {
+			db.RevDepsDatabase[re.GetFingerPrint()] = make(map[string]Package)
+		}
+		db.RevDepsDatabase[re.GetFingerPrint()][pd.GetFingerPrint()] = pd
+		_, ok = db.RevDepsDatabase[re.GetPackageName()]
+		if !ok {
+			db.RevDepsDatabase[re.GetPackageName()] = make(map[string]Package)
+		}
+		db.RevDepsDatabase[re.GetPackageName()][pd.GetPackageName()] = pd
+
+		db.Unlock()
+	}
 }
 
 func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
+
 	db.Lock()
+
 	pa, ok := db.ProvidesDatabase[p.GetPackageName()][p.GetVersion()]
 	if !ok {
 		versions, ok := db.ProvidesDatabase[p.GetPackageName()]
-		db.Unlock()
+		defer db.Unlock()
 
 		if !ok {
 			return nil, errors.New("No versions found for package")
@@ -195,9 +308,18 @@ func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
 		return nil, errors.New("No package provides this")
 	}
 	db.Unlock()
+
 	return db.FindPackage(pa)
 }
 
+func (db *InMemoryDatabase) Clone(to PackageDatabase) error {
+	return clone(db, to)
+}
+
+func (db *InMemoryDatabase) Copy() (PackageDatabase, error) {
+	return copy(db)
+}
+
 func (db *InMemoryDatabase) encodePackage(p Package) (string, string, error) {
 	pd, ok := p.(*DefaultPackage)
 	if !ok {
@@ -225,7 +347,13 @@ func (db *InMemoryDatabase) FindPackage(p Package) (Package, error) {
 
 // FindPackages return the list of the packages beloging to cat/name
 func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+	db.Lock()
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
+	db.Unlock()
 	if !ok {
 		return nil, errors.New("No versions found for package")
 	}
@@ -242,29 +370,41 @@ func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) {
 
 // FindPackages return the list of the packages beloging to cat/name (any versions in requested range)
 func (db *InMemoryDatabase) FindPackages(p Package) (Packages, error) {
-
+	if !p.IsSelector() {
+		pack, err := db.FindPackage(p)
+		if err != nil {
+			return []Package{}, err
+		}
+		return []Package{pack}, nil
+	}
 	// Provides: Treat as the replaced package here
 	if provided, err := db.getProvide(p); err == nil {
 		p = provided
+		if !provided.IsSelector() {
+			return Packages{provided}, nil
+		}
 	}
+
+	db.Lock()
+	var matches []*DefaultPackage
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
+	for ve := range versions {
+		match, _ := p.SelectorMatchVersion(ve, nil)
+		if match {
+			matches = append(matches, &DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
+		}
+	}
+	db.Unlock()
 	if !ok {
 		return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 	}
 	var versionsInWorld []Package
-	for ve, _ := range versions {
-		match, err := p.SelectorMatchVersion(ve, nil)
+	for _, p := range matches {
+		w, err := db.FindPackage(p)
 		if err != nil {
-			return nil, errors.Wrap(err, "Error on match selector")
-		}
-
-		if match {
-			w, err := db.FindPackage(&DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
-			if err != nil {
-				return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")
-			}
-			versionsInWorld = append(versionsInWorld, w)
+			return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")
 		}
+		versionsInWorld = append(versionsInWorld, w)
 	}
 	return Packages(versionsInWorld), nil
 }
@@ -344,16 +484,17 @@ func (db *InMemoryDatabase) FindPackageCandidate(p Package) (Package, error) {
 
 	required, err := db.FindPackage(p)
 	if err != nil {
+		err = nil
 		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
 		packages, err := p.Expand(db)
 		//	Info("Expanded", packages, err)
 		if err != nil || len(packages) == 0 {
 			required = p
+			err = errors.Wrap(err, "Candidate not found")
 		} else {
 			required = packages.Best(nil)
-
 		}
-		return required, nil
+		return required, err
 		//required = &DefaultPackage{Name: "test"}
 	}
 

pkg/package/database_mem_test.go

@@ -0,0 +1,247 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Database", func() {
+
+	db := NewInMemoryDatabase(false)
+	Context("Simple package", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		//	a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		//		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+		//		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+		It("Saves and get data back correctly", func() {
+
+			ID, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.GetPackage(ID)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Gets all", func() {
+
+			ids := db.GetPackages()
+
+			Expect(ids).To(Equal([]string{"A-->=1.0"}))
+
+		})
+		It("Find packages", func() {
+
+			pack, err := db.FindPackage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Find best package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+
+		})
+
+		It("Find specific package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		Context("Provides", func() {
+
+			It("replaces definitions", func() {
+				db := NewInMemoryDatabase(false)
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				pack, err := db.FindPackage(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(pack).To(Equal(a3))
+			})
+
+			It("replaces definitions", func() {
+				db := NewInMemoryDatabase(false)
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(a3))
+			})
+
+			It("replaces definitions", func() {
+				db := NewInMemoryDatabase(false)
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of unexisting packages", func() {
+				db := NewInMemoryDatabase(false)
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of a required package", func() {
+				db := NewInMemoryDatabase(false)
+
+				c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = db.CreatePackage(c)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			When("Searching with selectors", func() {
+				It("replaces definitions of a required package", func() {
+					db := NewInMemoryDatabase(false)
+
+					c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+					z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+					z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+					Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+					_, err := db.CreatePackage(z)
+					Expect(err).ToNot(HaveOccurred())
+					_, err = db.CreatePackage(c)
+					Expect(err).ToNot(HaveOccurred())
+
+					s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					packs, err := db.FindPackages(s)
+					Expect(err).ToNot(HaveOccurred())
+					Expect(packs).To(ContainElement(z))
+				})
+			})
+
+		})
+
+	})
+
+})

pkg/package/database_test.go

@@ -1,127 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package pkg_test
-
-import (
-	. "github.com/mudler/luet/pkg/package"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Database", func() {
-
-	db := NewInMemoryDatabase(false)
-	Context("Simple package", func() {
-		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
-		//	a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-		//		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
-		//		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
-		It("Saves and get data back correctly", func() {
-
-			ID, err := db.CreatePackage(a)
-			Expect(err).ToNot(HaveOccurred())
-
-			pack, err := db.GetPackage(ID)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(pack).To(Equal(a))
-
-		})
-
-		It("Gets all", func() {
-
-			ids := db.GetPackages()
-
-			Expect(ids).To(Equal([]string{"A-->=1.0"}))
-
-		})
-		It("Find packages", func() {
-
-			pack, err := db.FindPackage(a)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(pack).To(Equal(a))
-
-		})
-
-		It("Find best package candidate", func() {
-			db := NewInMemoryDatabase(false)
-			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
-			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
-			_, err := db.CreatePackage(a)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a1)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a3)
-			Expect(err).ToNot(HaveOccurred())
-			s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
-
-			pack, err := db.FindPackageCandidate(s)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(pack).To(Equal(a3))
-
-		})
-
-		It("Find specific package candidate", func() {
-			db := NewInMemoryDatabase(false)
-			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
-			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
-			_, err := db.CreatePackage(a)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a1)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a3)
-			Expect(err).ToNot(HaveOccurred())
-			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
-
-			pack, err := db.FindPackageCandidate(s)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(pack).To(Equal(a))
-
-		})
-
-		It("Provides replaces definitions", func() {
-			db := NewInMemoryDatabase(false)
-			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
-			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
-
-			a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
-			Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
-
-			_, err := db.CreatePackage(a)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a1)
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.CreatePackage(a3)
-			Expect(err).ToNot(HaveOccurred())
-
-			s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-
-			pack, err := db.FindPackage(s)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(pack).To(Equal(a3))
-		})
-	})
-
-})

pkg/package/package.go

@@ -26,11 +26,13 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/mudler/luet/pkg/helpers"
+	version "github.com/mudler/luet/pkg/versioner"
+
 	gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/crillab/gophersat/bf"
 	"github.com/ghodss/yaml"
 	"github.com/jinzhu/copier"
-	version "github.com/mudler/luet/pkg/versioner"
 	"github.com/pkg/errors"
 )
 
@@ -38,16 +40,16 @@ import (
 // FIXME: Currently some of the methods are returning DefaultPackages due to JSON serialization of the package
 type Package interface {
 	Encode(PackageDatabase) (string, error)
+	Related(definitiondb PackageDatabase) Packages
 
 	BuildFormula(PackageDatabase, PackageDatabase) ([]bf.Formula, error)
-	IsFlagged(bool) Package
-	Flagged() bool
+
 	GetFingerPrint() string
 	GetPackageName() string
+	GetPackageImageName() string
 	Requires([]*DefaultPackage) Package
 	Conflicts([]*DefaultPackage) Package
 	Revdeps(PackageDatabase) Packages
-	ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages
 	LabelDeps(PackageDatabase, string) Packages
 
 	GetProvides() []*DefaultPackage
@@ -59,9 +61,11 @@ type Package interface {
 	SetCategory(string)
 
 	GetName() string
+	SetName(string)
 	GetCategory() string
 
 	GetVersion() string
+	SetVersion(string)
 	RequiresContains(PackageDatabase, Package) (bool, error)
 	Matches(m Package) bool
 	BumpBuildVersion() error
@@ -91,13 +95,24 @@ type Package interface {
 	HasLabel(string) bool
 	MatchLabel(*regexp.Regexp) bool
 
+	AddAnnotation(string, string)
+	GetAnnotations() map[string]string
+	HasAnnotation(string) bool
+	MatchAnnotation(*regexp.Regexp) bool
+
+	IsHidden() bool
 	IsSelector() bool
 	VersionMatchSelector(string, version.Versioner) (bool, error)
 	SelectorMatchVersion(string, version.Versioner) (bool, error)
 
 	String() string
 	HumanReadableString() string
-	HashFingerprint() string
+	HashFingerprint(string) string
+
+	SetBuildTimestamp(s string)
+	GetBuildTimestamp() string
+
+	Clone() Package
 }
 
 type Tree interface {
@@ -132,6 +147,60 @@ func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) {
 	return unescaped, nil
 }
 
+type rawPackages []map[string]interface{}
+
+func (r rawPackages) Find(name, category, version string) map[string]interface{} {
+	for _, v := range r {
+		if v["name"] == name && v["category"] == category && v["version"] == version {
+			return v
+		}
+	}
+	return map[string]interface{}{}
+}
+
+func GetRawPackages(yml []byte) (rawPackages, error) {
+	var rawPackages struct {
+		Packages []map[string]interface{} `yaml:"packages"`
+	}
+	source, err := yaml.YAMLToJSON(yml)
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+	err = json.Unmarshal(bytes, &rawPackages)
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+	return rawPackages.Packages, nil
+
+}
+func DefaultPackagesFromYaml(yml []byte) ([]DefaultPackage, error) {
+
+	var unescaped struct {
+		Packages []DefaultPackage `json:"packages"`
+	}
+	source, err := yaml.YAMLToJSON(yml)
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+	err = json.Unmarshal(bytes, &unescaped)
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+	return unescaped.Packages, nil
+}
+
 // Major and minor gets escaped when marshalling in JSON, making compiler fails recognizing selectors for expansion
 func (t *DefaultPackage) JSON() ([]byte, error) {
 	buffer := &bytes.Buffer{}
@@ -151,19 +220,21 @@ type DefaultPackage struct {
 	State            State             `json:"state,omitempty"`
 	PackageRequires  []*DefaultPackage `json:"requires"`           // Affects YAML field names too.
 	PackageConflicts []*DefaultPackage `json:"conflicts"`          // Affects YAML field names too.
-	IsSet            bool              `json:"set,omitempty"`      // Affects YAML field names too.
 	Provides         []*DefaultPackage `json:"provides,omitempty"` // Affects YAML field names too.
+	Hidden           bool              `json:"hidden,omitempty"`   // Affects YAML field names too.
 
-	// TODO: Annotations?
+	// Annotations are used for core features/options
+	Annotations map[string]string `json:"annotations,omitempty"` // Affects YAML field names too
 
 	// Path is set only internally when tree is loaded from disk
 	Path string `json:"path,omitempty"`
 
-	Description string   `json:"description,omitempty"`
-	Uri         []string `json:"uri,omitempty"`
-	License     string   `json:"license,omitempty"`
+	Description    string   `json:"description,omitempty"`
+	Uri            []string `json:"uri,omitempty"`
+	License        string   `json:"license,omitempty"`
+	BuildTimestamp string   `json:"buildtimestamp,omitempty"`
 
-	Labels map[string]string `json:labels,omitempty`
+	Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
 }
 
 // State represent the package state
@@ -176,7 +247,7 @@ func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*D
 		Version:          version,
 		PackageRequires:  requires,
 		PackageConflicts: conflicts,
-		Labels:           make(map[string]string, 0),
+		Labels:           nil,
 	}
 }
 
@@ -194,9 +265,9 @@ func (p *DefaultPackage) GetFingerPrint() string {
 	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version)
 }
 
-func (p *DefaultPackage) HashFingerprint() string {
+func (p *DefaultPackage) HashFingerprint(salt string) string {
 	h := md5.New()
-	io.WriteString(h, p.GetFingerPrint())
+	io.WriteString(h, fmt.Sprintf("%s-%s", p.GetFingerPrint(), salt))
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
@@ -218,6 +289,20 @@ func (p *DefaultPackage) GetPackageName() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.Category)
 }
 
+func (p *DefaultPackage) GetPackageImageName() string {
+	return fmt.Sprintf("%s-%s:%s", p.Name, p.Category, p.Version)
+}
+
+// GetBuildTimestamp returns the package build timestamp
+func (p *DefaultPackage) GetBuildTimestamp() string {
+	return p.BuildTimestamp
+}
+
+// SetBuildTimestamp sets the package Build timestamp
+func (p *DefaultPackage) SetBuildTimestamp(s string) {
+	p.BuildTimestamp = s
+}
+
 // GetPath returns the path where the definition file was found
 func (p *DefaultPackage) GetPath() string {
 	return p.Path
@@ -235,26 +320,24 @@ func (p *DefaultPackage) IsSelector() bool {
 	return strings.ContainsAny(p.GetVersion(), "<>=")
 }
 
+func (p *DefaultPackage) IsHidden() bool {
+	return p.Hidden
+}
+
 func (p *DefaultPackage) HasLabel(label string) bool {
-	ans := false
-	for k, _ := range p.Labels {
-		if k == label {
-			ans = true
-			break
-		}
-	}
-	return ans
+	return helpers.MapHasKey(&p.Labels, label)
 }
 
 func (p *DefaultPackage) MatchLabel(r *regexp.Regexp) bool {
-	ans := false
-	for k, v := range p.Labels {
-		if r.MatchString(k + "=" + v) {
-			ans = true
-			break
-		}
-	}
-	return ans
+	return helpers.MapMatchRegex(&p.Labels, r)
+}
+
+func (p *DefaultPackage) HasAnnotation(label string) bool {
+	return helpers.MapHasKey(&p.Annotations, label)
+}
+
+func (p *DefaultPackage) MatchAnnotation(r *regexp.Regexp) bool {
+	return helpers.MapMatchRegex(&p.Annotations, r)
 }
 
 // AddUse adds a use to a package
@@ -287,7 +370,6 @@ func (p *DefaultPackage) Encode(db PackageDatabase) (string, error) {
 func (p *DefaultPackage) Yaml() ([]byte, error) {
 	j, err := p.JSON()
 	if err != nil {
-
 		return []byte{}, err
 	}
 	y, err := yaml.JSONToYAML(j)
@@ -298,15 +380,6 @@ func (p *DefaultPackage) Yaml() ([]byte, error) {
 	return y, nil
 }
 
-func (p *DefaultPackage) IsFlagged(b bool) Package {
-	p.IsSet = b
-	return p
-}
-
-func (p *DefaultPackage) Flagged() bool {
-	return p.IsSet
-}
-
 func (p *DefaultPackage) GetName() string {
 	return p.Name
 }
@@ -314,6 +387,9 @@ func (p *DefaultPackage) GetName() string {
 func (p *DefaultPackage) GetVersion() string {
 	return p.Version
 }
+func (p *DefaultPackage) SetVersion(v string) {
+	p.Version = v
+}
 func (p *DefaultPackage) GetDescription() string {
 	return p.Description
 }
@@ -338,15 +414,32 @@ func (p *DefaultPackage) GetCategory() string {
 func (p *DefaultPackage) SetCategory(s string) {
 	p.Category = s
 }
+
+func (p *DefaultPackage) SetName(s string) {
+	p.Name = s
+}
+
 func (p *DefaultPackage) GetUses() []string {
 	return p.UseFlags
 }
 func (p *DefaultPackage) AddLabel(k, v string) {
+	if p.Labels == nil {
+		p.Labels = make(map[string]string, 0)
+	}
 	p.Labels[k] = v
 }
+func (p *DefaultPackage) AddAnnotation(k, v string) {
+	if p.Annotations == nil {
+		p.Annotations = make(map[string]string, 0)
+	}
+	p.Annotations[k] = v
+}
 func (p *DefaultPackage) GetLabels() map[string]string {
 	return p.Labels
 }
+func (p *DefaultPackage) GetAnnotations() map[string]string {
+	return p.Annotations
+}
 func (p *DefaultPackage) GetProvides() []*DefaultPackage {
 	return p.Provides
 }
@@ -417,52 +510,47 @@ func (p *DefaultPackage) Revdeps(definitiondb PackageDatabase) Packages {
 	return versionsInWorld
 }
 
-// ExpandedRevdeps returns the package reverse dependencies,
-// matching also selectors in versions (>, <, >=, <=)
-func (p *DefaultPackage) ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages {
+func walkPackage(p Package, definitiondb PackageDatabase, visited map[string]interface{}) Packages {
 	var versionsInWorld Packages
 	if _, ok := visited[p.HumanReadableString()]; ok {
 		return versionsInWorld
 	}
 	visited[p.HumanReadableString()] = true
 
-	for _, w := range definitiondb.World() {
-		if w.Matches(p) {
-			continue
+	revdeps, _ := definitiondb.GetRevdeps(p)
+	for _, r := range revdeps {
+		versionsInWorld = append(versionsInWorld, r)
+	}
+
+	if !p.IsSelector() {
+		versionsInWorld = append(versionsInWorld, p)
+	}
+
+	for _, re := range p.GetRequires() {
+		versions, _ := re.Expand(definitiondb)
+		for _, r := range versions {
+
+			versionsInWorld = append(versionsInWorld, r)
+			versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...)
 		}
-		match := false
-
-		for _, re := range w.GetRequires() {
-			if re.Matches(p) {
-				match = true
-			}
-			if !match {
-
-				packages, _ := re.Expand(definitiondb)
-				for _, pa := range packages {
-					if pa.Matches(p) {
-						match = true
-					}
-				}
-			}
-
-			//	if ok, _ := w.RequiresContains(definitiondb, p); ok {
-
-		}
-
-		if match {
-			versionsInWorld = append(versionsInWorld, w)
-
-			versionsInWorld = append(versionsInWorld, w.ExpandedRevdeps(definitiondb, visited).Unique()...)
-
-		}
-		//	}
 
 	}
-	//visited[p.HumanReadableString()] = true
+	for _, re := range p.GetConflicts() {
+		versions, _ := re.Expand(definitiondb)
+		for _, r := range versions {
+
+			versionsInWorld = append(versionsInWorld, r)
+			versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...)
+
+		}
+	}
 	return versionsInWorld.Unique()
 }
 
+func (p *DefaultPackage) Related(definitiondb PackageDatabase) Packages {
+	return walkPackage(p, definitiondb, map[string]interface{}{})
+}
+
 func (p *DefaultPackage) LabelDeps(definitiondb PackageDatabase, labelKey string) Packages {
 	var pkgsWithLabelInWorld Packages
 	// TODO: check if integrate some index to improve
@@ -622,7 +710,7 @@ func (pack *DefaultPackage) buildFormula(definitiondb PackageDatabase, db Packag
 					C = bf.Var(encodedC)
 					// Or the Candidate is true, or all the others might be not true
 					// This forces the CDCL sat implementation to look first at a solution with C=true
-					formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Or(C, bf.Or(priorityConstraints...)), bf.Or(bf.Not(C), bf.Or(priorityALO...)))))
+					formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.And(C, bf.Or(priorityConstraints...)), bf.And(bf.Not(C), bf.Or(priorityALO...)))))
 				}
 
 				// AMO - At most one
@@ -728,7 +816,6 @@ func (p *DefaultPackage) Explain() {
 	fmt.Println("Name: ", p.GetName())
 	fmt.Println("Category: ", p.GetCategory())
 	fmt.Println("Version: ", p.GetVersion())
-	fmt.Println("Installed: ", p.IsSet)
 
 	for _, req := range p.GetRequires() {
 		fmt.Println("\t-> ", req)

pkg/package/package_test.go

@@ -36,8 +36,8 @@ var _ = Describe("Package", func() {
 		})
 
 		It("Generates packages fingerprint's hashes", func() {
-			Expect(a.HashFingerprint()).ToNot(Equal(a1.HashFingerprint()))
-			Expect(a.HashFingerprint()).To(Equal("c64caa391b79adb598ad98e261aa79a0"))
+			Expect(a.HashFingerprint("")).ToNot(Equal(a1.HashFingerprint("")))
+			Expect(a.HashFingerprint("")).To(Equal("76972ef6991ec6102f33b401105c1351"))
 		})
 	})
 
@@ -46,6 +46,7 @@ var _ = Describe("Package", func() {
 		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
 		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+		re := regexp.MustCompile("project[0-9][=].*")
 		It("Expands correctly", func() {
 			definitions := NewInMemoryDatabase(false)
 			for _, p := range []Package{a1, a11, a01} {
@@ -60,6 +61,8 @@ var _ = Describe("Package", func() {
 			Expect(len(lst)).To(Equal(2))
 			p := lst.Best(nil)
 			Expect(p).To(Equal(a11))
+			// Test annotation with null map
+			Expect(a.MatchAnnotation(re)).To(Equal(false))
 		})
 	})
 
@@ -91,6 +94,34 @@ var _ = Describe("Package", func() {
 		})
 	})
 
+	Context("Find annotations on packages", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.AddAnnotation("project1", "test1")
+		a.AddAnnotation("label2", "value1")
+		b := NewPackage("B", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b.AddAnnotation("project2", "test2")
+		b.AddAnnotation("label2", "value1")
+		It("Expands correctly", func() {
+			var err error
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b} {
+				_, err = definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			re := regexp.MustCompile("project[0-9][=].*")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(re).ToNot(BeNil())
+			Expect(a.HasAnnotation("label2")).To(Equal(true))
+			Expect(a.HasAnnotation("label3")).To(Equal(false))
+			Expect(a.HasAnnotation("project1")).To(Equal(true))
+			Expect(b.HasAnnotation("project2")).To(Equal(true))
+			Expect(b.HasAnnotation("label2")).To(Equal(true))
+			Expect(b.MatchAnnotation(re)).To(Equal(true))
+			Expect(a.MatchAnnotation(re)).To(Equal(true))
+
+		})
+	})
+
 	Context("Check description", func() {
 		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a.SetDescription("Description A")
@@ -189,8 +220,8 @@ var _ = Describe("Package", func() {
 				_, err := definitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			visited := make(map[string]interface{})
-			lst := a.ExpandedRevdeps(definitions, visited)
+			lst, err := definitions.GetRevdeps(a)
+			Expect(err).ToNot(HaveOccurred())
 			Expect(lst).To(ContainElement(c))
 			Expect(lst).To(ContainElement(d))
 			Expect(lst).To(ContainElement(e))
@@ -211,9 +242,9 @@ var _ = Describe("Package", func() {
 				_, err := definitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			visited := make(map[string]interface{})
 
-			lst := a.ExpandedRevdeps(definitions, visited)
+			lst, err := definitions.GetRevdeps(a)
+			Expect(err).ToNot(HaveOccurred())
 			Expect(lst).To(ContainElement(b))
 			Expect(lst).To(ContainElement(c))
 			Expect(lst).To(ContainElement(d))
@@ -235,9 +266,8 @@ var _ = Describe("Package", func() {
 				_, err := definitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			visited := make(map[string]interface{})
-
-			lst := a.ExpandedRevdeps(definitions, visited)
+			lst, err := definitions.GetRevdeps(a)
+			Expect(err).ToNot(HaveOccurred())
 			Expect(lst).To(ContainElement(b))
 			Expect(lst).To(ContainElement(c))
 			Expect(lst).To(ContainElement(d))
@@ -283,7 +313,6 @@ var _ = Describe("Package", func() {
 
 			Expect(p.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(p.GetName()).To(Equal(a.GetName()))
-			Expect(p.Flagged()).To(Equal(a.Flagged()))
 			Expect(p.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(p.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(p.GetRequires())).To(Equal(len(a.GetRequires())))
@@ -343,7 +372,6 @@ var _ = Describe("Package", func() {
 			a2 := a.Clone()
 			Expect(a2.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(a2.GetName()).To(Equal(a.GetName()))
-			Expect(a2.Flagged()).To(Equal(a.Flagged()))
 			Expect(a2.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(a2.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(a2.GetRequires())).To(Equal(len(a.GetRequires())))

pkg/repository/loader.go

@@ -19,6 +19,7 @@ package repository
 import (
 	"io/ioutil"
 	"path"
+	"path/filepath"
 	"regexp"
 
 	"github.com/ghodss/yaml"
@@ -28,9 +29,22 @@ import (
 )
 
 func LoadRepositories(c *LuetConfig) error {
-	var regexRepo = regexp.MustCompile(`.yml$`)
+	var regexRepo = regexp.MustCompile(`.yml$|.yaml$`)
+	var err error
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
 
 	for _, rdir := range c.RepositoriesConfDir {
+
+		rdir = filepath.Join(rootfs, rdir)
+
 		Debug("Parsing Repository Directory", rdir, "...")
 
 		files, err := ioutil.ReadDir(rdir)

pkg/repository/repository_test.go

@@ -35,12 +35,22 @@ var _ = Describe("Repository", func() {
 
 		It("Chec Load Repository 1", func() {
 			Expect(err).Should(BeNil())
-			Expect(len(cfg.SystemRepositories)).Should(Equal(1))
+			Expect(len(cfg.SystemRepositories)).Should(Equal(2))
 			Expect(cfg.SystemRepositories[0].Name).Should(Equal("test1"))
 			Expect(cfg.SystemRepositories[0].Priority).Should(Equal(999))
 			Expect(cfg.SystemRepositories[0].Type).Should(Equal("disk"))
 			Expect(len(cfg.SystemRepositories[0].Urls)).Should(Equal(1))
 			Expect(cfg.SystemRepositories[0].Urls[0]).Should(Equal("tests/repos/test1"))
 		})
+
+		It("Chec Load Repository 2", func() {
+			Expect(err).Should(BeNil())
+			Expect(len(cfg.SystemRepositories)).Should(Equal(2))
+			Expect(cfg.SystemRepositories[1].Name).Should(Equal("test2"))
+			Expect(cfg.SystemRepositories[1].Priority).Should(Equal(1000))
+			Expect(cfg.SystemRepositories[1].Type).Should(Equal("disk"))
+			Expect(len(cfg.SystemRepositories[1].Urls)).Should(Equal(1))
+			Expect(cfg.SystemRepositories[1].Urls[0]).Should(Equal("tests/repos/test2"))
+		})
 	})
 })

pkg/solver/benchmark_test.go

@@ -0,0 +1,298 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package solver_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/tests/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "github.com/mudler/luet/pkg/solver"
+)
+
+var _ = Describe("Solver Benchmarks", func() {
+	db := pkg.NewInMemoryDatabase(false)
+	dbInstalled := pkg.NewInMemoryDatabase(false)
+	dbDefinitions := pkg.NewInMemoryDatabase(false)
+	var s PackageSolver
+
+	Context("Complex data sets", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+		Measure("it should be fast in resolution from a 50000 dataset", func(b Benchmarker) {
+
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 50000; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					_, err := dbInstalled.CreatePackage(C)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for i := 0; i < 1; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+					solution, err := s.Install([]pkg.Package{A})
+					Expect(err).ToNot(HaveOccurred())
+
+					Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: H, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: G, Value: true}))
+				}
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 120), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+	Context("Complex data sets - Parallel", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: ParallelSimple, Concurrency: 10}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+		Measure("it should be fast in resolution from a 50000 dataset", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 50000; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					_, err := dbInstalled.CreatePackage(C)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				for i := 0; i < 1; i++ {
+					C := pkg.NewPackage("C"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G"+strconv.Itoa(i), "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H"+strconv.Itoa(i), "", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D"+strconv.Itoa(i), "", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B"+strconv.Itoa(i), "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A"+strconv.Itoa(i), "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+					solution, err := s.Install([]pkg.Package{A})
+					Expect(err).ToNot(HaveOccurred())
+
+					Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: H, Value: true}))
+					Expect(solution).To(ContainElement(PackageAssert{Package: G, Value: true}))
+
+					//	Expect(len(solution)).To(Equal(6))
+				}
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+	Context("Complex data sets - Parallel Upgrades", func() {
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			//	dbInstalled = pkg.NewInMemoryDatabase(false)
+			dbDefinitions = pkg.NewInMemoryDatabase(false)
+			s = NewSolver(Options{Type: ParallelSimple, Concurrency: 100}, dbInstalled, dbDefinitions, db)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+			tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			dbInstalled = pkg.NewBoltDatabase(tmpfile.Name())
+		})
+
+		Measure("it should be fast in resolution from a 10000*8 dataset", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 2; i < 10000; i++ {
+					C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G, H} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+				}
+
+				//C := pkg.NewPackage("C", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				G := pkg.NewPackage("G", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H := pkg.NewPackage("H", "1", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "1", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "1", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				_, err := dbInstalled.CreatePackage(A)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(B)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(D)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(H)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = dbInstalled.CreatePackage(G)
+				Expect(err).ToNot(HaveOccurred())
+				fmt.Println("Upgrade starts")
+
+				packages, ass, err := s.Upgrade(false, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(packages).To(ContainElement(A))
+
+				G = pkg.NewPackage("G", "9999", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H = pkg.NewPackage("H", "9999", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D = pkg.NewPackage("D", "9999", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B = pkg.NewPackage("B", "9999", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A = pkg.NewPackage("A", "9999", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				Expect(ass).To(ContainElement(PackageAssert{Package: A, Value: true}))
+
+				Expect(len(packages)).To(Equal(5))
+				//	Expect(len(solution)).To(Equal(6))
+
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+
+		Measure("it should be fast in installation with 12000 packages installed and 2000*8 available", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 2000; i++ {
+					C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					fmt.Println("Creating package, run", i)
+				}
+
+				for i := 0; i < 12000; i++ {
+					x := helpers.RandomPackage()
+					_, err := dbInstalled.CreatePackage(x)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				G := pkg.NewPackage("G", strconv.Itoa(50000), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H := pkg.NewPackage("H", strconv.Itoa(50000), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", strconv.Itoa(50000), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", strconv.Itoa(50000), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", strconv.Itoa(50000), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				ass, err := s.Install([]pkg.Package{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(ass).To(ContainElement(PackageAssert{Package: pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}), Value: true}))
+				//Expect(ass).To(Equal(5))
+				//	Expect(len(solution)).To(Equal(6))
+
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+
+		PMeasure("it should be fast in resolution from a 50000 dataset with upgrade universe", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 2; i++ {
+					C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					fmt.Println("Creating package, run", i)
+				}
+
+				G := pkg.NewPackage("G", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H := pkg.NewPackage("H", "1", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "1", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "1", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				_, err := dbInstalled.CreatePackage(A)
+				Expect(err).ToNot(HaveOccurred())
+				fmt.Println("Upgrade starts")
+
+				packages, ass, err := s.UpgradeUniverse(true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(ass).To(ContainElement(PackageAssert{Package: pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}), Value: true}))
+				Expect(packages).To(ContainElement(pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})))
+				Expect(packages).To(Equal(5))
+				//	Expect(len(solution)).To(Equal(6))
+
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+	})
+
+})

pkg/solver/decoder.go

@@ -22,9 +22,9 @@ import (
 	"unicode"
 
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/topsort"
 	toposort "github.com/philopon/go-toposort"
 	"github.com/pkg/errors"
-	"github.com/stevenle/topsort"
 )
 
 type PackagesAssertions []PackageAssert
@@ -150,30 +150,21 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 
 	orderedAssertions := PackagesAssertions{}
 	unorderedAssertions := PackagesAssertions{}
-	fingerprints := []string{}
 
 	tmpMap := map[string]PackageAssert{}
 	graph := topsort.NewGraph()
-
 	for _, a := range assertions {
 		graph.AddNode(a.Package.GetFingerPrint())
 		tmpMap[a.Package.GetFingerPrint()] = a
-		fingerprints = append(fingerprints, a.Package.GetFingerPrint())
 		unorderedAssertions = append(unorderedAssertions, a) // Build a list of the ones that must be ordered
-
-		if a.Value {
-			unorderedAssertions = append(unorderedAssertions, a) // Build a list of the ones that must be ordered
-		} else {
-			orderedAssertions = append(orderedAssertions, a) // Keep last the ones which are not meant to be installed
-		}
 	}
 
 	sort.Sort(unorderedAssertions)
 	// Build a topological graph
-	//graph := toposort.NewGraph(len(unorderedAssertions))
-	//	graph.AddNodes(fingerprints...)
 	for _, a := range unorderedAssertions {
 		currentPkg := a.Package
+		added := map[string]interface{}{}
+	REQUIRES:
 		for _, requiredDef := range currentPkg.GetRequires() {
 			if def, err := definitiondb.FindPackage(requiredDef); err == nil { // Provides: Get a chance of being override here
 				requiredDef = def.(*pkg.DefaultPackage)
@@ -185,9 +176,12 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 			if req != nil {
 				requiredDef = req.Package
 			}
-
+			if _, ok := added[requiredDef.GetFingerPrint()]; ok {
+				continue REQUIRES
+			}
 			// Expand also here, as we need to order them (or instead the solver should give back the dep correctly?)
 			graph.AddEdge(currentPkg.GetFingerPrint(), requiredDef.GetFingerPrint())
+			added[requiredDef.GetFingerPrint()] = true
 		}
 	}
 	result, err := graph.TopSort(fingerprint)
@@ -197,8 +191,11 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 	for _, res := range result {
 		a, ok := tmpMap[res]
 		if !ok {
-			return nil, errors.New("fail looking for " + res)
-			//	continue
+			//return nil, errors.New("fail looking for " + res)
+			// Since now we don't return the entire world as part of assertions
+			// if we don't find any reference must be because fingerprint we are analyzing (which is the one we are ordering against)
+			// is not part of the assertions, thus we can omit it from the result
+			continue
 		}
 		orderedAssertions = append(orderedAssertions, a)
 		//	orderedAssertions = append(PackagesAssertions{a}, orderedAssertions...) // push upfront
@@ -248,6 +245,34 @@ func (a PackagesAssertions) Less(i, j int) bool {
 
 }
 
+func (a PackagesAssertions) TrueLen() int {
+	count := 0
+	for _, ass := range a {
+		if ass.Value {
+			count++
+		}
+	}
+
+	return count
+}
+
+// HashFrom computes the assertion hash From a given package. It drops it from the assertions
+// and checks it's not the only one. if it's unique it marks it specially - so the hash
+// which is generated is unique for the selected package
+func (assertions PackagesAssertions) HashFrom(p pkg.Package) string {
+
+	var assertionhash string
+
+	// When we don't have any solution to hash for, we need to generate an UUID by ourselves
+	latestsolution := assertions.Drop(p)
+	if latestsolution.TrueLen() == 0 {
+		assertionhash = assertions.Mark(p).AssertionHash()
+	} else {
+		assertionhash = latestsolution.AssertionHash()
+	}
+	return assertionhash
+}
+
 func (assertions PackagesAssertions) AssertionHash() string {
 	var fingerprint string
 	for _, assertion := range assertions { // Note: Always order them first!
@@ -284,3 +309,18 @@ func (assertions PackagesAssertions) Cut(p pkg.Package) PackagesAssertions {
 	}
 	return ass
 }
+
+// Mark returns a new assertion with the package marked
+func (assertions PackagesAssertions) Mark(p pkg.Package) PackagesAssertions {
+	ass := PackagesAssertions{}
+
+	for _, a := range assertions {
+		if a.Package.Matches(p) {
+			marked := a.Package.Clone()
+			marked.SetName("@@" + marked.GetName())
+			a = PackageAssert{Package: marked.(*pkg.DefaultPackage), Value: a.Value, Hash: a.Hash}
+		}
+		ass = append(ass, a)
+	}
+	return ass
+}

pkg/solver/decoder_test.go

@@ -19,6 +19,7 @@ import (
 	"strconv"
 
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -29,13 +30,13 @@ var _ = Describe("Decoder", func() {
 	db := pkg.NewInMemoryDatabase(false)
 	dbInstalled := pkg.NewInMemoryDatabase(false)
 	dbDefinitions := pkg.NewInMemoryDatabase(false)
-	s := NewSolver(dbInstalled, dbDefinitions, db)
+	s := NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 
 	BeforeEach(func() {
 		db = pkg.NewInMemoryDatabase(false)
 		dbInstalled = pkg.NewInMemoryDatabase(false)
 		dbDefinitions = pkg.NewInMemoryDatabase(false)
-		s = NewSolver(dbInstalled, dbDefinitions, db)
+		s = NewSolver(Options{Type: SingleCoreSimple}, dbInstalled, dbDefinitions, db)
 	})
 
 	Context("Assertion ordering", func() {
@@ -213,12 +214,10 @@ var _ = Describe("Decoder", func() {
 			hash2 := solution.AssertionHash()
 
 			//	Expect(len(solution)).To(Equal(6))
-			Expect(solution[0].Package.GetName()).To(Equal("A"))
-			Expect(solution[1].Package.GetName()).To(Equal("G"))
-			Expect(solution[2].Package.GetName()).To(Equal("H"))
-			Expect(solution[3].Package.GetName()).To(Equal("D"))
-			Expect(solution[4].Package.GetName()).To(Equal("B"))
-			Expect(solution[0].Value).ToNot(BeTrue())
+			Expect(solution[0].Package.GetName()).To(Equal("G"))
+			Expect(solution[1].Package.GetName()).To(Equal("H"))
+			Expect(solution[2].Package.GetName()).To(Equal("D"))
+			Expect(solution[3].Package.GetName()).To(Equal("B"))
 
 			Expect(hash).ToNot(Equal(""))
 			Expect(hash2).ToNot(Equal(""))
@@ -283,5 +282,143 @@ var _ = Describe("Decoder", func() {
 			Expect(orderY.Cut(Y).Drop(Y).AssertionHash()).To(Equal(orderZ.Cut(Z).Drop(Z).AssertionHash()))
 		})
 
+		It("HashFrom can be used equally", func() {
+
+			X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+			Z := pkg.NewPackage("Z", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{X, Y, Z} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			solution, err := s.Install([]pkg.Package{Y})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution2, err := s.Install([]pkg.Package{Z})
+			Expect(err).ToNot(HaveOccurred())
+			orderY, err := solution.Order(dbDefinitions, Y.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
+			orderZ, err := solution2.Order(dbDefinitions, Z.GetFingerPrint())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(orderY.Cut(Y).Drop(Y)).To(Equal(orderZ.Cut(Z).Drop(Z)))
+
+			Expect(orderY.Cut(Y).HashFrom(Y)).To(Equal(orderZ.Cut(Z).HashFrom(Z)))
+		})
+
+		It("Unique hashes for single packages", func() {
+
+			X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+			for _, p := range []pkg.Package{X, F, D} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			solution, err := s.Install([]pkg.Package{X})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution2, err := s.Install([]pkg.Package{F})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution3, err := s.Install([]pkg.Package{D})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(solution.AssertionHash()).ToNot(Equal(solution2.AssertionHash()))
+			Expect(solution3.AssertionHash()).To(Equal(solution.AssertionHash()))
+
+		})
+
+		It("Unique hashes for empty assertions", func() {
+			empty := solver.PackagesAssertions{}
+			empty2 := solver.PackagesAssertions{}
+
+			Expect(empty.AssertionHash()).To(Equal(empty2.AssertionHash()))
+		})
+
+		It("Unique hashes for single packages with HashFrom", func() {
+
+			X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			D := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+
+			empty := solver.PackagesAssertions{}
+
+			for _, p := range []pkg.Package{X, F, D, Y} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			solution, err := s.Install([]pkg.Package{X})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution2, err := s.Install([]pkg.Package{F})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution3, err := s.Install([]pkg.Package{D})
+			Expect(err).ToNot(HaveOccurred())
+
+			solution4, err := s.Install([]pkg.Package{Y})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(solution.HashFrom(X)).ToNot(Equal(solution2.HashFrom(F)))
+			Expect(solution3.HashFrom(D)).To(Equal(solution.HashFrom(X)))
+
+			Expect(empty.AssertionHash()).ToNot(Equal(solution3.HashFrom(D)))
+			Expect(empty.AssertionHash()).ToNot(Equal(solution2.HashFrom(F)))
+
+			Expect(solution4.Drop(Y).AssertionHash()).To(Equal(solution4.HashFrom(Y)))
+		})
+		for index := 0; index < 300; index++ { // Just to make sure we don't have false positives
+
+			It("Always same solution", func() {
+
+				X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+				Z := pkg.NewPackage("Z", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+				W := pkg.NewPackage("W", "", []*pkg.DefaultPackage{Z, Y}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{X, Y, Z} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{W})
+				Expect(err).ToNot(HaveOccurred())
+
+				orderW, err := solution.Order(dbDefinitions, W.GetFingerPrint())
+				Expect(err).ToNot(HaveOccurred())
+				Expect(orderW[0].Package.GetName()).To(Equal("X"))
+				Expect(orderW[1].Package.GetName()).To(Equal("Y"))
+				Expect(orderW[2].Package.GetName()).To(Equal("Z"))
+				Expect(orderW[3].Package.GetName()).To(Equal("W"))
+			})
+
+		}
+
 	})
 })