Tag 0.9.1

Respect rootfs path for configs and url

.github/workflows/release.yml

@@ -0,0 +1,23 @@
+on: push
+name: Build and release on push
+jobs:
+  release:
+    name: Test and Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Tests
+        run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage
+      - name: Build
+        run: sudo -E env "PATH=$PATH" make multiarch-build && sudo chmod -R 777 release/
+      - name: Release
+        uses: fnkr/github-action-ghr@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GHR_PATH: release/
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+
+on: pull_request
+name: Build and Test
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage

.gitignore

@@ -1,2 +1,4 @@
 *.swp
 luet
+tests/integration/shunit2
+tests/integration/bin

.travis.yml

@@ -1,18 +1,35 @@
+dist: bionic
 language: go
-services:
-  - docker
 go:
-  - "1.12"
+  - "1.14"
+
 env:
-  - "GO15VENDOREXPERIMENT=1"
+  global:
+    - "GO15VENDOREXPERIMENT=1"
+  jobs:
+    - "DOCKER_BUILDKIT=0"
+    - "DOCKER_BUILDKIT=1"
 before_install:
-  - make deps
-  - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && mkdir -p $HOME/bin && export PATH=$PATH:$HOME/bin && mv container-diff-linux-amd64 $HOME/bin/container-diff
+  - sudo rm -rf /var/lib/apt/lists/*
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) edge"
+  - sudo apt-get update
+  - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
+  - export DOCKER_CLI_EXPERIMENTAL=enabled
+  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  - mkdir -vp ~/.docker/cli-plugins/
+  - curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+  - chmod a+x ~/.docker/cli-plugins/docker-buildx
+  - docker buildx version
+  - sudo -E env "PATH=$PATH" apt-get install -y libcap2-bin
+  - sudo -E env "PATH=$PATH" make deps
 script:
-  - make multiarch-build test
-after_success:
-  - make coverage
-  - bash <(curl -s https://codecov.io/bash)
-  - git config --global user.name "Deployer" && git config --global user.email foo@bar.com
-  - go get github.com/tcnksm/ghr
-  - ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+  - sudo -E env "PATH=$PATH" make multiarch-build test-integration test-coverage
+
+#after_success:
+#  - |
+#      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+#        sudo -E env "PATH=$PATH" git config --global user.name "Deployer" && git config --global user.email foo@bar.com
+#        sudo -E env "PATH=$PATH" go get github.com/tcnksm/ghr
+#        sudo -E env "PATH=$PATH" ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+#      fi

Dockerfile

@@ -0,0 +1,11 @@
+FROM golang as builder
+RUN apt-get update && apt-get install -y upx
+ADD . /luet
+RUN cd /luet && make build-small
+
+FROM scratch
+ENV LUET_NOLOCK=true
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /luet/luet /usr/bin/luet
+
+ENTRYPOINT ["/usr/bin/luet"]

Makefile

@@ -1,10 +1,14 @@
+
+# go tool nm ./luet | grep Commit
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S %Z')"
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildCommit=$(shell git rev-parse HEAD)"
+
 NAME ?= luet
 PACKAGE_NAME ?= $(NAME)
 PACKAGE_CONFLICT ?= $(PACKAGE_NAME)-beta
 REVISION := $(shell git rev-parse --short HEAD || echo dev)
 VERSION := $(shell git describe --tags || echo $(REVISION))
 VERSION := $(shell echo $(VERSION) | sed -e 's/^v//g')
-ITTERATION := $(shell date +%s)
 BUILD_PLATFORMS ?= -osarch="linux/amd64" -osarch="linux/386" -osarch="linux/arm"
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
@@ -19,12 +23,20 @@ fmt:
 test:
 	GO111MODULE=off go get github.com/onsi/ginkgo/ginkgo
 	GO111MODULE=off go get github.com/onsi/gomega/...
-	ginkgo -race -r ./...
+	ginkgo -race -r -flakeAttempts 3 ./...
+
+.PHONY: test-integration
+test-integration:
+	tests/integration/run.sh
 
 .PHONY: coverage
 coverage:
 	go test ./... -race -coverprofile=coverage.txt -covermode=atomic
 
+.PHONY: test-coverage
+test-coverage:
+	scripts/ginkgo.coverage.sh --codecov
+
 .PHONY: help
 help:
 	# make all => deps test lint build
@@ -36,6 +48,8 @@ help:
 .PHONY: clean
 clean:
 	rm -rf release/
+	rm -rf tests/integration/shunit2
+	rm -rf tests/integration/bin
 
 .PHONY: deps
 deps:
@@ -49,16 +63,19 @@ deps:
 
 .PHONY: build
 build:
-	go build
+	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)'
 
-.PHONY: gox-build
-gox-build:
-	# Building gitlab-ci-multi-runner for $(BUILD_PLATFORMS)
-	gox $(BUILD_PLATFORMS) -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+.PHONY: build-small
+build-small:
+	@$(MAKE) LDFLAGS+="-s -w" build
+	upx --brute -1 $(NAME)
+
+.PHONY: image
+image:
+	docker build --rm -t luet/base .
 
 .PHONY: lint
 lint:
-	# Checking project code style...
 	golint ./... | grep -v "be unexported"
 
 .PHONY: vendor
@@ -73,4 +90,4 @@ test-docker:
 
 .PHONY: multiarch-build
 multiarch-build:
-	gox $(BUILD_PLATFORMS) -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"

README.md

@@ -1,9 +1,64 @@
-# luet - Package manager
+# luet - Container-based Package manager
+[![Docker Repository on Quay](https://quay.io/repository/luet/base/status "Docker Repository on Quay")](https://quay.io/repository/luet/base)
 [![Go Report Card](https://goreportcard.com/badge/github.com/mudler/luet)](https://goreportcard.com/report/github.com/mudler/luet)
 [![Build Status](https://travis-ci.org/mudler/luet.svg?branch=master)](https://travis-ci.org/mudler/luet)
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
 [![codecov](https://codecov.io/gh/mudler/luet/branch/master/graph/badge.svg)](https://codecov.io/gh/mudler/luet)
 
-No dep hell for your own package manager !
+Luet is a multi-platform Package Manager based off from containers - it uses Docker (and other tech) to sandbox your builds and generate packages from them. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
 
-Package manager that reuses Gentoo's Portage output binaries, but which doesn't depend on it.
+It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/specfile/) in YAML notation to define both packages and rootfs. As it is based on containers, it can be used to build seed stages for Linux From Scratch installations and it can build and track updates for those systems.
+
+It is written entirely in Golang and where used as package manager, it can run in from scratch environment, with zero dependencies.
+
+## In a glance
+
+- Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.
+- It builds, installs, uninstalls and perform upgrades on machines
+- Installer doesn't depend on anything ( 0 dep installer !), statically built
+- Support for packages as "layers"
+- It uses SAT solving techniques to solve the deptree ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
+
+## Install
+
+To install luet, you can grab a release on the [Release page](https://github.com/mudler/luet/releases) or compile it in your machine (requires Golang installed):
+
+    $ git clone https://github.com/mudler/luet.git
+    $ cd luet
+    $ make build
+
+## Status
+
+Luet is not feature-complete yet, it can build, install/uninstall/upgrade packages - but it doesn't support yet all the features you would normally expect from a Package Manager nowadays.
+
+# Dependency solving
+
+Luet uses SAT and Reinforcement learning engine for dependency solving.
+It encodes the package requirements into a SAT problem, using gophersat to solve the dependency tree and give a concrete model as result.
+
+## SAT encoding
+
+Each package and its constraints are encoded and built around [OPIUM](https://ranjitjhala.github.io/static/opium.pdf). Additionally, Luet treats
+also selectors seamlessly while building the model, adding *ALO* ( *At least one* ) and *AMO* ( *At most one* ) rules to guarantee coherence within the installed system.
+
+## Reinforcement learning
+
+Luet also implements a small and portable qlearning agent that will try to solve conflict on your behalf
+when they arises while trying to validate your queries against the system model.
+
+To leverage it, simply pass ```--solver-type qlearning``` to the subcommands that supports it ( you can check out by invoking ```--help``` ).
+
+## Documentation
+
+[Documentation](https://luet-lab.github.io/docs) is available, or
+run `luet --help`,  any subcommand is documented as well, try e.g.: `luet build --help`.
+
+## Authors
+
+Luet is here thanks to our amazing [contributors](https://github.com/mudler/luet/graphs/contributors)!.
+
+Luet was originally created by Ettore Di Giacinto, mudler@sabayon.org, mudler@gentoo.org.
+
+## License
+
+Luet is distributed under the terms of GPLv3, check out the LICENSE file.

cmd/box.go

@@ -0,0 +1,36 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/box"
+
+	"github.com/spf13/cobra"
+)
+
+var boxGroupCmd = &cobra.Command{
+	Use:   "box [command] [OPTIONS]",
+	Short: "Manage luet boxes",
+}
+
+func init() {
+	RootCmd.AddCommand(boxGroupCmd)
+
+	boxGroupCmd.AddCommand(
+		NewBoxExecCommand(),
+	)
+}

cmd/box/exec.go

@@ -0,0 +1,81 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_box
+
+import (
+	"os"
+
+	b64 "encoding/base64"
+
+	"github.com/mudler/luet/pkg/box"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+func NewBoxExecCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "exec [OPTIONS]",
+		Short: "Execute a binary in a box",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			stdin, _ := cmd.Flags().GetBool("stdin")
+			stdout, _ := cmd.Flags().GetBool("stdout")
+			stderr, _ := cmd.Flags().GetBool("stderr")
+			rootfs, _ := cmd.Flags().GetString("rootfs")
+			base, _ := cmd.Flags().GetBool("decode")
+			entrypoint, _ := cmd.Flags().GetString("entrypoint")
+			envs, _ := cmd.Flags().GetStringArray("env")
+			mounts, _ := cmd.Flags().GetStringArray("mount")
+
+			if base {
+				var ss []string
+				for _, a := range args {
+					sDec, _ := b64.StdEncoding.DecodeString(a)
+					ss = append(ss, string(sDec))
+				}
+				//If the command to run is complex,using base64 to avoid bad input
+
+				args = ss
+			}
+			Info("Executing", args, "in", rootfs)
+
+			b := box.NewBox(entrypoint, args, mounts, envs, rootfs, stdin, stdout, stderr)
+			err := b.Run()
+			if err != nil {
+				Fatal(err)
+			}
+		},
+	}
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	ans.Flags().String("rootfs", path, "Rootfs path")
+	ans.Flags().Bool("stdin", false, "Attach to stdin")
+	ans.Flags().Bool("stdout", true, "Attach to stdout")
+	ans.Flags().Bool("stderr", true, "Attach to stderr")
+	ans.Flags().Bool("decode", false, "Base64 decode")
+	ans.Flags().StringArrayP("env", "e", []string{}, "Environment settings")
+	ans.Flags().StringArrayP("mount", "m", []string{}, "List of paths to bind-mount from the host")
+
+	ans.Flags().String("entrypoint", "/bin/sh", "Entrypoint command (/bin/sh)")
+
+	return ans
+}

cmd/build.go

@@ -15,15 +15,18 @@
 package cmd
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
-	"regexp"
-	"runtime"
 
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/spf13/cobra"
@@ -34,17 +37,61 @@ var buildCmd = &cobra.Command{
 	Use:   "build <package name> <package name> <package name> ...",
 	Short: "build a package or a tree",
 	Long:  `build packages or trees from luet tree definitions. Packages are in [category]/[name]-[version] form`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("clean", cmd.Flags().Lookup("clean"))
+		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
+		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
+		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
+		viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged"))
+		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
+		viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
+		viper.BindPFlag("all", cmd.Flags().Lookup("all"))
+		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+
+		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
+		viper.BindPFlag("push", cmd.Flags().Lookup("push"))
+		viper.BindPFlag("pull", cmd.Flags().Lookup("pull"))
+		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
+
+		LuetCfg.Viper.BindPFlag("keep-exported-images", cmd.Flags().Lookup("keep-exported-images"))
+
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+	},
 	Run: func(cmd *cobra.Command, args []string) {
 
-		src := viper.GetString("tree")
-		dst := viper.GetString("output")
-		concurrency := viper.GetInt("concurrency")
+		clean := viper.GetBool("clean")
+		treePaths := viper.GetStringSlice("tree")
+		dst := viper.GetString("destination")
+		concurrency := LuetCfg.GetGeneral().Concurrency
 		backendType := viper.GetString("backend")
 		privileged := viper.GetBool("privileged")
 		revdeps := viper.GetBool("revdeps")
 		all := viper.GetBool("all")
 		databaseType := viper.GetString("database")
+		compressionType := viper.GetString("compression")
+		imageRepository := viper.GetString("image-repository")
+		push := viper.GetBool("push")
+		pull := viper.GetBool("pull")
+		keepImages := viper.GetBool("keep-images")
+		nodeps := viper.GetBool("nodeps")
+		onlydeps := viper.GetBool("onlydeps")
+		keepExportedImages := viper.GetBool("keep-exported-images")
+		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
+		full, _ := cmd.Flags().GetBool("full")
+		skip, _ := cmd.Flags().GetBool("skip-if-metadata-exists")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		var results Results
 
+		out, _ := cmd.Flags().GetString("output")
+		if out != "terminal" {
+			LuetCfg.GetLogging().SetLogLevel("error")
+		}
+		pretend, _ := cmd.Flags().GetBool("pretend")
 		compilerSpecs := compiler.NewLuetCompilationspecs()
 		var compilerBackend compiler.CompilerBackend
 		var db pkg.PackageDatabase
@@ -58,40 +105,88 @@ var buildCmd = &cobra.Command{
 		switch databaseType {
 		case "memory":
 			db = pkg.NewInMemoryDatabase(false)
+
 		case "boltdb":
 			tmpdir, err := ioutil.TempDir("", "package")
 			if err != nil {
 				Fatal(err)
 			}
 			db = pkg.NewBoltDatabase(tmpdir)
+
 		}
 		defer db.Clean()
 
 		generalRecipe := tree.NewCompilerRecipe(db)
 
-		Info("Loading", src)
-		err := generalRecipe.Load(src)
-		if err != nil {
-			Fatal("Error: " + err.Error())
+		if len(treePaths) <= 0 {
+			Fatal("No tree path supplied!")
 		}
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
 
-		err = luetCompiler.Prepare(concurrency)
-		if err != nil {
-			Fatal("Error: " + err.Error())
+		for _, src := range treePaths {
+			Info("Loading tree", src)
+
+			err := generalRecipe.Load(src)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
 		}
-		if !all {
+
+		Info("Building in", dst)
+
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		opts := compiler.NewDefaultCompilerOptions()
+		opts.SolverOptions = *LuetCfg.GetSolverOptions()
+		opts.ImageRepository = imageRepository
+		opts.Clean = clean
+		opts.PullFirst = pull
+		opts.KeepImg = keepImages
+		opts.Push = push
+		opts.OnlyDeps = onlydeps
+		opts.NoDeps = nodeps
+		opts.KeepImageExport = keepExportedImages
+		opts.SkipIfMetadataExists = skip
+		opts.PackageTargetOnly = onlyTarget
+
+		var solverOpts solver.Options
+		if concurrent {
+			solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
+		} else {
+			solverOpts = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+		}
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts, solverOpts)
+		luetCompiler.SetConcurrency(concurrency)
+		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
+		if full {
+			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
+			if err != nil {
+				Fatal(err.Error())
+			}
+			for _, spec := range specs {
+				Info(":package: Selecting ", spec.GetPackage().GetName(), spec.GetPackage().GetVersion())
+
+				compilerSpecs.Add(spec)
+			}
+		} else if !all {
 			for _, a := range args {
-				decodepackage, err := regexp.Compile(`^([<>]?\~?=?)((([^\/]+)\/)?(?U)(\S+))(-(\d+(\.\d+)*[a-z]?(_(alpha|beta|pre|rc|p)\d*)*(-r\d+)?))?$`)
-				if err != nil {
-					Fatal("Error: " + err.Error())
-				}
-				packageInfo := decodepackage.FindAllStringSubmatch(a, -1)
 
-				category := packageInfo[0][4]
-				name := packageInfo[0][5]
-				version := packageInfo[0][7]
-				spec, err := luetCompiler.FromPackage(&pkg.DefaultPackage{Name: name, Category: category, Version: version})
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					Fatal("Invalid package string ", a, ": ", err.Error())
+				}
+
+				spec, err := luetCompiler.FromPackage(pack)
 				if err != nil {
 					Fatal("Error: " + err.Error())
 				}
@@ -100,16 +195,15 @@ var buildCmd = &cobra.Command{
 				compilerSpecs.Add(spec)
 			}
 		} else {
-			w, e := generalRecipe.Tree().World()
-			if e != nil {
-				Fatal("Error: " + err.Error())
-			}
+			w := generalRecipe.GetDatabase().World()
+
 			for _, p := range w {
 				spec, err := luetCompiler.FromPackage(p)
 				if err != nil {
 					Fatal("Error: " + err.Error())
 				}
 				Info(":package: Selecting ", p.GetName(), p.GetVersion())
+				spec.SetOutputPath(dst)
 				compilerSpecs.Add(spec)
 			}
 		}
@@ -117,11 +211,60 @@ var buildCmd = &cobra.Command{
 		var artifact []compiler.Artifact
 		var errs []error
 		if revdeps {
-			artifact, errs = luetCompiler.CompileWithReverseDeps(concurrency, privileged, compilerSpecs)
+			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
+		} else if pretend {
+			toCalculate := []compiler.CompilationSpec{}
+			if full {
+				var err error
+				toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...)
+				if err != nil {
+					errs = append(errs, err)
+				}
+			} else {
+				toCalculate = compilerSpecs.All()
+			}
+
+			for _, sp := range toCalculate {
+				packs, err := luetCompiler.ComputeDepTree(sp)
+				if err != nil {
+					errs = append(errs, err)
+				}
+				for _, p := range packs {
+					results.Packages = append(results.Packages,
+						PackageResult{
+							Name:       p.Package.GetName(),
+							Version:    p.Package.GetVersion(),
+							Category:   p.Package.GetCategory(),
+							Repository: "",
+							Hidden:     p.Package.IsHidden(),
+							Target:     sp.GetPackage().HumanReadableString(),
+						})
+				}
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			case "terminal":
+				for _, p := range results.Packages {
+					Info(p.String())
+				}
+			}
 		} else {
-			artifact, errs = luetCompiler.CompileParallel(concurrency, privileged, compilerSpecs)
-
+			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
 		}
 		if len(errs) != 0 {
 			for _, e := range errs {
@@ -140,22 +283,35 @@ func init() {
 	if err != nil {
 		Fatal(err)
 	}
-	buildCmd.Flags().String("tree", path, "Source luet tree")
-	viper.BindPFlag("tree", buildCmd.Flags().Lookup("tree"))
-	buildCmd.Flags().String("output", path, "Destination folder")
-	viper.BindPFlag("output", buildCmd.Flags().Lookup("output"))
+	buildCmd.Flags().Bool("clean", true, "Build all packages without considering the packages present in the build directory")
+	buildCmd.Flags().StringSliceP("tree", "t", []string{}, "Path of the tree to use.")
 	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
-	viper.BindPFlag("backend", buildCmd.Flags().Lookup("backend"))
-	buildCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
-	viper.BindPFlag("concurrency", buildCmd.Flags().Lookup("concurrency"))
 	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
-	viper.BindPFlag("privileged", buildCmd.Flags().Lookup("privileged"))
 	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
-	viper.BindPFlag("database", buildCmd.Flags().Lookup("database"))
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
-	viper.BindPFlag("revdeps", buildCmd.Flags().Lookup("revdeps"))
+	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
+	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
+
+	buildCmd.Flags().String("destination", path, "Destination folder")
+	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
+	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+	buildCmd.Flags().Bool("push", false, "Push images to a hub")
+	buildCmd.Flags().Bool("pull", false, "Pull images from a hub")
+	buildCmd.Flags().Bool("keep-images", true, "Keep built docker images in the host")
+	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
+	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
+	buildCmd.Flags().Bool("keep-exported-images", false, "Keep exported images used during building")
+	buildCmd.Flags().Bool("skip-if-metadata-exists", false, "Skip package if metadata exists")
+	buildCmd.Flags().Bool("only-target-package", false, "Build packages of only the required target. Otherwise builds all the necessary ones not present in the destination")
+	buildCmd.Flags().String("solver-type", "", "Solver strategy")
+	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+
+	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
+
+	buildCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 
-	buildCmd.Flags().Bool("all", false, "Build all packages in the tree")
-	viper.BindPFlag("all", buildCmd.Flags().Lookup("all"))
 	RootCmd.AddCommand(buildCmd)
 }

cmd/cleanup.go

@@ -0,0 +1,70 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	config "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+var cleanupCmd = &cobra.Command{
+	Use:   "cleanup",
+	Short: "Clean packages cache.",
+	Long:  `remove downloaded packages tarballs and clean cache directory`,
+	Run: func(cmd *cobra.Command, args []string) {
+		var cleaned int = 0
+
+		// Check if cache dir exists
+		if helpers.Exists(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
+
+			files, err := ioutil.ReadDir(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
+			if err != nil {
+				Fatal("Error on read cachedir ", err.Error())
+			}
+
+			for _, file := range files {
+				if file.IsDir() {
+					continue
+				}
+
+				if config.LuetCfg.GetGeneral().Debug {
+					Info("Removing ", file.Name())
+				}
+
+				err := os.RemoveAll(
+					filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
+				if err != nil {
+					Fatal("Error on removing", file.Name())
+				}
+				cleaned++
+			}
+		}
+
+		Info("Cleaned: ", cleaned, "packages.")
+
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(cleanupCmd)
+}

cmd/config.go

@@ -0,0 +1,78 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+
+	config "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+
+	"github.com/spf13/cobra"
+)
+
+var configCmd = &cobra.Command{
+	Use:     "config",
+	Short:   "Print config",
+	Long:    `Show luet configuration`,
+	Aliases: []string{"c"},
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println(config.LuetCfg.GetLogging())
+		fmt.Println(config.LuetCfg.GetGeneral())
+		fmt.Println(config.LuetCfg.GetSystem())
+		if len(config.LuetCfg.CacheRepositories) > 0 {
+			fmt.Println("repetitors:")
+			for _, r := range config.LuetCfg.CacheRepositories {
+				fmt.Println("  - ", r.String())
+			}
+		}
+		if len(config.LuetCfg.SystemRepositories) > 0 {
+			fmt.Println("repositories:")
+			for _, r := range config.LuetCfg.SystemRepositories {
+				fmt.Println("  - ", r.String())
+			}
+		}
+
+		if len(config.LuetCfg.RepositoriesConfDir) > 0 {
+			fmt.Println("repos_confdir:")
+			for _, dir := range config.LuetCfg.RepositoriesConfDir {
+				fmt.Println("  - ", dir)
+			}
+		}
+
+		if len(config.LuetCfg.ConfigProtectConfDir) > 0 {
+
+			// Load config protect configs
+			installer.LoadConfigProtectConfs(config.LuetCfg)
+
+			fmt.Println("config_protect_confdir:")
+			for _, dir := range config.LuetCfg.ConfigProtectConfDir {
+				fmt.Println("  - ", dir)
+			}
+
+			if len(config.LuetCfg.GetConfigProtectConfFiles()) > 0 {
+				fmt.Println("protect_conf_files:")
+				for _, file := range config.LuetCfg.GetConfigProtectConfFiles() {
+					fmt.Println("  - ", file.String())
+				}
+			}
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(configCmd)
+}

cmd/convert.go

@@ -16,8 +16,8 @@ package cmd
 
 import (
 	"io/ioutil"
-	"runtime"
 
+	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	tree "github.com/mudler/luet/pkg/tree"
@@ -28,13 +28,16 @@ import (
 )
 
 var convertCmd = &cobra.Command{
-	Use:   "convert",
+	Use:   "convert [portage-tree] [luet-tree]",
 	Short: "convert other package manager tree into luet",
 	Long:  `Parses external PM and produces a luet parsable tree`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
+		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
+	},
 	Run: func(cmd *cobra.Command, args []string) {
 
 		t := viper.GetString("type")
-		c := viper.GetInt("concurrency")
 		databaseType := viper.GetString("database")
 		var db pkg.PackageDatabase
 
@@ -49,9 +52,15 @@ var convertCmd = &cobra.Command{
 		var builder tree.Parser
 		switch t {
 		case "gentoo":
-			builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c, gentoo.InMemory)
+			builder = gentoo.NewGentooBuilder(
+				&gentoo.SimpleEbuildParser{},
+				LuetCfg.GetGeneral().Concurrency,
+				gentoo.InMemory)
 		default: // dup
-			builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c, gentoo.InMemory)
+			builder = gentoo.NewGentooBuilder(
+				&gentoo.SimpleEbuildParser{},
+				LuetCfg.GetGeneral().Concurrency,
+				gentoo.InMemory)
 		}
 
 		switch databaseType {
@@ -71,13 +80,12 @@ var convertCmd = &cobra.Command{
 			Fatal("Error: " + err.Error())
 		}
 
-		defer packageTree.GetPackageSet().Clean()
+		defer packageTree.Clean()
 		Info("Tree generated")
 
-		generalRecipe := tree.NewGeneralRecipe(db)
+		generalRecipe := tree.NewGeneralRecipe(packageTree)
 		Info("Saving generated tree to " + output)
 
-		generalRecipe.WithTree(packageTree)
 		err = generalRecipe.Save(output)
 		if err != nil {
 			Fatal("Error: " + err.Error())
@@ -87,11 +95,7 @@ var convertCmd = &cobra.Command{
 
 func init() {
 	convertCmd.Flags().String("type", "gentoo", "source type")
-	viper.BindPFlag("type", convertCmd.Flags().Lookup("type"))
-	convertCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
-	viper.BindPFlag("concurrency", convertCmd.Flags().Lookup("concurrency"))
 	convertCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
-	viper.BindPFlag("database", convertCmd.Flags().Lookup("database"))
 
 	RootCmd.AddCommand(convertCmd)
 }

cmd/create-repo.go

@@ -0,0 +1,149 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var createrepoCmd = &cobra.Command{
+	Use:   "create-repo",
+	Short: "Create a luet repository from a build",
+	Long:  `Generate and renew repository metadata`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("packages", cmd.Flags().Lookup("packages"))
+		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
+		viper.BindPFlag("output", cmd.Flags().Lookup("output"))
+		viper.BindPFlag("name", cmd.Flags().Lookup("name"))
+		viper.BindPFlag("descr", cmd.Flags().Lookup("descr"))
+		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
+		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
+		viper.BindPFlag("tree-compression", cmd.Flags().Lookup("tree-compression"))
+		viper.BindPFlag("tree-filename", cmd.Flags().Lookup("tree-filename"))
+		viper.BindPFlag("meta-compression", cmd.Flags().Lookup("meta-compression"))
+		viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename"))
+		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
+		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var err error
+		var repo installer.Repository
+
+		treePaths := viper.GetStringSlice("tree")
+		dst := viper.GetString("output")
+		packages := viper.GetString("packages")
+		name := viper.GetString("name")
+		descr := viper.GetString("descr")
+		urls := viper.GetStringSlice("urls")
+		t := viper.GetString("type")
+		reset := viper.GetBool("reset-revision")
+		treetype := viper.GetString("tree-compression")
+		treeName := viper.GetString("tree-filename")
+		metatype := viper.GetString("meta-compression")
+		metaName := viper.GetString("meta-filename")
+		source_repo := viper.GetString("repo")
+
+		treeFile := installer.NewDefaultTreeRepositoryFile()
+		metaFile := installer.NewDefaultMetaRepositoryFile()
+
+		if source_repo != "" {
+			// Search for system repository
+			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
+			if len(treePaths) <= 0 {
+				treePaths = []string{lrepo.TreePath}
+			}
+
+			if t == "" {
+				t = lrepo.Type
+			}
+
+			repo, err = installer.GenerateRepository(lrepo.Name,
+				lrepo.Description, t,
+				lrepo.Urls,
+				lrepo.Priority,
+				packages,
+				treePaths,
+				pkg.NewInMemoryDatabase(false))
+
+		} else {
+			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
+				treePaths, pkg.NewInMemoryDatabase(false))
+		}
+
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+
+		if treetype != "" {
+			treeFile.SetCompressionType(compiler.CompressionImplementation(treetype))
+		}
+
+		if treeName != "" {
+			treeFile.SetFileName(treeName)
+		}
+
+		if metatype != "" {
+			metaFile.SetCompressionType(compiler.CompressionImplementation(metatype))
+		}
+
+		if metaName != "" {
+			metaFile.SetFileName(metaName)
+		}
+
+		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
+		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
+
+		err = repo.Write(dst, reset)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	createrepoCmd.Flags().String("packages", path, "Packages folder (output from build)")
+	createrepoCmd.Flags().StringSliceP("tree", "t", []string{}, "Path of the source trees to use.")
+	createrepoCmd.Flags().String("output", path, "Destination folder")
+	createrepoCmd.Flags().String("name", "luet", "Repository name")
+	createrepoCmd.Flags().String("descr", "luet", "Repository description")
+	createrepoCmd.Flags().StringSlice("urls", []string{}, "Repository URLs")
+	createrepoCmd.Flags().String("type", "disk", "Repository type (disk)")
+	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
+	createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
+
+	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
+	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
+
+	RootCmd.AddCommand(createrepoCmd)
+}

cmd/database.go

@@ -0,0 +1,37 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/database"
+
+	"github.com/spf13/cobra"
+)
+
+var databaseGroupCmd = &cobra.Command{
+	Use:   "database [command] [OPTIONS]",
+	Short: "Manage system database (dangerous commands ahead!)",
+}
+
+func init() {
+	RootCmd.AddCommand(databaseGroupCmd)
+
+	databaseGroupCmd.AddCommand(
+		NewDatabaseCreateCommand(),
+		NewDatabaseRemoveCommand(),
+	)
+}

cmd/database/create.go

@@ -0,0 +1,78 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseCreateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "create <artifact_metadata1.yaml> <artifact_metadata1.yaml>",
+		Short: "Insert a package in the system DB",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			var systemDB pkg.PackageDatabase
+
+			for _, a := range args {
+				dat, err := ioutil.ReadFile(a)
+				if err != nil {
+					Fatal("Failed reading ", a, ": ", err.Error())
+				}
+				art, err := compiler.NewPackageArtifactFromYaml(dat)
+				if err != nil {
+					Fatal("Failed reading yaml ", a, ": ", err.Error())
+				}
+
+				if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+					systemDB = pkg.NewBoltDatabase(
+						filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+				} else {
+					systemDB = pkg.NewInMemoryDatabase(true)
+				}
+
+				files := art.GetFiles()
+
+				if _, err := systemDB.CreatePackage(art.GetCompileSpec().GetPackage()); err != nil {
+					Fatal("Failed to create ", a, ": ", err.Error())
+				}
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.GetCompileSpec().GetPackage().GetFingerPrint(), Files: files}); err != nil {
+					Fatal("Failed setting package files for ", a, ": ", err.Error())
+				}
+
+				Info(art.GetCompileSpec().GetPackage().HumanReadableString(), " created")
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/database/remove.go

@@ -0,0 +1,69 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"path/filepath"
+
+	. "github.com/mudler/luet/pkg/logger"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseRemoveCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "remove [package1] [package2] ...",
+		Short: "Remove a package from the system DB (forcefully - you normally don't want to do that)",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var systemDB pkg.PackageDatabase
+
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					Fatal("Invalid package string ", a, ": ", err.Error())
+				}
+
+				if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+					systemDB = pkg.NewBoltDatabase(
+						filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+				} else {
+					systemDB = pkg.NewInMemoryDatabase(true)
+				}
+
+				if err := systemDB.RemovePackage(pack); err != nil {
+					Fatal("Failed removing ", a, ": ", err.Error())
+				}
+
+				if err := systemDB.RemovePackageFiles(pack); err != nil {
+					Fatal("Failed removing files for ", a, ": ", err.Error())
+				}
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/exec.go

@@ -0,0 +1,86 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	b64 "encoding/base64"
+
+	"github.com/mudler/luet/pkg/box"
+	. "github.com/mudler/luet/pkg/logger"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+var execCmd = &cobra.Command{
+	Use:   "exec --rootfs /path [command]",
+	Short: "Execute a command in the rootfs context",
+	Long:  `Uses unshare technique and pivot root to execute a command inside a folder containing a valid rootfs`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+	},
+	// If you change this, look at pkg/box/exec that runs this command and adapt
+	Run: func(cmd *cobra.Command, args []string) {
+
+		stdin, _ := cmd.Flags().GetBool("stdin")
+		stdout, _ := cmd.Flags().GetBool("stdout")
+		stderr, _ := cmd.Flags().GetBool("stderr")
+		rootfs, _ := cmd.Flags().GetString("rootfs")
+		base, _ := cmd.Flags().GetBool("decode")
+
+		entrypoint, _ := cmd.Flags().GetString("entrypoint")
+		envs, _ := cmd.Flags().GetStringArray("env")
+		mounts, _ := cmd.Flags().GetStringArray("mount")
+
+		if base {
+			var ss []string
+			for _, a := range args {
+				sDec, _ := b64.StdEncoding.DecodeString(a)
+				ss = append(ss, string(sDec))
+			}
+			//If the command to run is complex,using base64 to avoid bad input
+
+			args = ss
+		}
+		Info("Executing", args, "in", rootfs)
+
+		b := box.NewBox(entrypoint, args, mounts, envs, rootfs, stdin, stdout, stderr)
+		err := b.Exec()
+		if err != nil {
+			Fatal(errors.Wrap(err, fmt.Sprintf("entrypoint: %s rootfs: %s", entrypoint, rootfs)))
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	execCmd.Hidden = true
+	execCmd.Flags().String("rootfs", path, "Rootfs path")
+	execCmd.Flags().Bool("stdin", false, "Attach to stdin")
+	execCmd.Flags().Bool("stdout", false, "Attach to stdout")
+	execCmd.Flags().Bool("stderr", false, "Attach to stderr")
+	execCmd.Flags().Bool("decode", false, "Base64 decode")
+
+	execCmd.Flags().StringArrayP("env", "e", []string{}, "Environment settings")
+	execCmd.Flags().StringArrayP("mount", "m", []string{}, "List of paths to bind-mount from the host")
+
+	execCmd.Flags().String("entrypoint", "/bin/sh", "Entrypoint command (/bin/sh)")
+
+	RootCmd.AddCommand(execCmd)
+}

cmd/helpers/cli.go

@@ -0,0 +1,78 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_helpers
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+
+	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
+	pkg "github.com/mudler/luet/pkg/package"
+	version "github.com/mudler/luet/pkg/versioner"
+)
+
+func CreateRegexArray(rgx []string) ([]*regexp.Regexp, error) {
+	ans := make([]*regexp.Regexp, len(rgx))
+	if len(rgx) > 0 {
+		for idx, reg := range rgx {
+			re := regexp.MustCompile(reg)
+			if re == nil {
+				return nil, errors.New("Invalid regex " + reg + "!")
+			}
+			ans[idx] = re
+		}
+	}
+
+	return ans, nil
+}
+
+func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
+	gp, err := _gentoo.ParsePackageStr(p)
+	if err != nil {
+		return nil, err
+	}
+	if gp.Version == "" {
+		gp.Version = "0"
+		gp.Condition = _gentoo.PkgCondGreaterEqual
+	}
+
+	pkgVersion := ""
+	if gp.VersionBuild != "" {
+		pkgVersion = fmt.Sprintf("%s%s%s+%s",
+			version.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+			gp.Version,
+			gp.VersionSuffix,
+			gp.VersionBuild,
+		)
+	} else {
+		pkgVersion = fmt.Sprintf("%s%s%s",
+			version.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+			gp.Version,
+			gp.VersionSuffix,
+		)
+	}
+
+	pack := &pkg.DefaultPackage{
+		Name:     gp.Name,
+		Category: gp.Category,
+		Version:  pkgVersion,
+		Uri:      make([]string, 0),
+	}
+
+	return pack, nil
+}

cmd/install.go

@@ -0,0 +1,136 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+var installCmd = &cobra.Command{
+	Use:     "install <pkg1> <pkg2> ...",
+	Short:   "Install a package",
+	Aliases: []string{"i"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+	},
+	Long: `Install packages in parallel`,
+	Run: func(cmd *cobra.Command, args []string) {
+		var toInstall pkg.Packages
+		var systemDB pkg.PackageDatabase
+
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toInstall = append(toInstall, pack)
+		}
+
+		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps := LuetCfg.Viper.GetBool("nodeps")
+		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+		})
+		inst.Repositories(repos)
+
+		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+			systemDB = pkg.NewBoltDatabase(
+				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+		} else {
+			systemDB = pkg.NewInMemoryDatabase(true)
+		}
+		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Install(toInstall, system)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	installCmd.Flags().String("system-dbpath", path, "System db path")
+	installCmd.Flags().String("system-target", path, "System rootpath")
+	installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	installCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	installCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	installCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+
+	RootCmd.AddCommand(installCmd)
+}

cmd/pack.go

@@ -0,0 +1,89 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var packCmd = &cobra.Command{
+	Use:   "pack <package name>",
+	Short: "pack a custom package",
+	Long:  `pack and creates metadata directly from a source path`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
+		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+		viper.BindPFlag("source", cmd.Flags().Lookup("source"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		sourcePath := viper.GetString("source")
+
+		dst := viper.GetString("destination")
+		compressionType := viper.GetString("compression")
+		concurrency := LuetCfg.GetGeneral().Concurrency
+
+		if len(args) != 1 {
+			Fatal("You must specify a package name")
+		}
+
+		packageName := args[0]
+
+		p, err := helpers.ParsePackageStr(packageName)
+		if err != nil {
+			Fatal("Invalid package string ", packageName, ": ", err.Error())
+		}
+
+		spec := &compiler.LuetCompilationSpec{Package: p}
+		artifact := compiler.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
+		artifact.SetCompressionType(compiler.CompressionImplementation(compressionType))
+		err = artifact.Compress(sourcePath, concurrency)
+		if err != nil {
+			Fatal("failed compressing ", packageName, ": ", err.Error())
+		}
+		artifact.SetCompileSpec(spec)
+		filelist, err := artifact.FileList()
+		if err != nil {
+			Fatal("failed generating file list for ", packageName, ": ", err.Error())
+		}
+		artifact.SetFiles(filelist)
+		artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
+		err = artifact.WriteYaml(dst)
+		if err != nil {
+			Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	packCmd.Flags().String("source", path, "Source folder")
+	packCmd.Flags().String("destination", path, "Destination folder")
+	packCmd.Flags().String("compression", "gzip", "Compression alg: none, gzip")
+
+	RootCmd.AddCommand(packCmd)
+}

cmd/query.go

@@ -1,112 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-package cmd
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-
-	. "github.com/mudler/luet/pkg/logger"
-
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
-	tree "github.com/mudler/luet/pkg/tree"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-)
-
-var queryCmd = &cobra.Command{
-	Use:   "query install <pkg>",
-	Short: "query other package manager tree into luet",
-	Long:  `Parses external PM and produces a luet parsable tree`,
-	Run: func(cmd *cobra.Command, args []string) {
-
-		input := viper.GetString("input")
-
-		if len(args) != 4 {
-			log.Fatalln("Incorrect number of arguments")
-		}
-		databaseType := viper.GetString("database")
-		var db pkg.PackageDatabase
-
-		switch databaseType {
-		case "memory":
-			db = pkg.NewInMemoryDatabase(false)
-		case "boltdb":
-			tmpdir, err := ioutil.TempDir("", "package")
-			if err != nil {
-				Fatal(err)
-			}
-			db = pkg.NewBoltDatabase(tmpdir)
-		}
-		defer db.Clean()
-
-		generalRecipe := tree.NewGeneralRecipe(db)
-		fmt.Println("Loading generated tree from " + input)
-
-		err := generalRecipe.Load(input)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-
-		defer generalRecipe.Tree().GetPackageSet().Clean()
-
-		t := args[0]
-		v := args[1]
-		version := args[2]
-		cat := args[3]
-		switch t {
-		case "install":
-			// XXX: pack needs to be the same which is present in world.
-			// Tree caches generated world when using FindPackage
-			pack, err := generalRecipe.Tree().FindPackage(&pkg.DefaultPackage{Category: cat, Name: v, Version: version})
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-
-			fmt.Println("Install query from " + input + " [" + v + "]")
-			world, err := generalRecipe.Tree().World()
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-			fmt.Println(">>> World")
-			for _, packss := range world {
-				packss.Explain()
-			}
-			s := solver.NewSolver([]pkg.Package{}, world, generalRecipe.Tree().GetPackageSet())
-			solution, err := s.Install([]pkg.Package{pack})
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-			fmt.Println(">>> Solution")
-
-			for _, assertion := range solution {
-				assertion.Explain()
-			}
-		}
-
-	},
-}
-
-func init() {
-	queryCmd.Flags().String("input", "", "source folder")
-	viper.BindPFlag("input", queryCmd.Flags().Lookup("input"))
-	queryCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
-	viper.BindPFlag("database", queryCmd.Flags().Lookup("database"))
-
-	RootCmd.AddCommand(queryCmd)
-}

cmd/reclaim.go

@@ -0,0 +1,88 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	installer "github.com/mudler/luet/pkg/installer"
+
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+var reclaimCmd = &cobra.Command{
+	Use:   "reclaim",
+	Short: "Reclaim packages to Luet database from available repositories",
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+	},
+	Long: `Add packages to the systemdb if files belonging to packages 
+in available repositories exists in the target root.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		var systemDB pkg.PackageDatabase
+
+		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		force := LuetCfg.Viper.GetBool("force")
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			Force:                       force,
+			PreserveSystemEssentialData: true,
+		})
+		inst.Repositories(repos)
+
+		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+			systemDB = pkg.NewBoltDatabase(
+				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+		} else {
+			systemDB = pkg.NewInMemoryDatabase(true)
+		}
+		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Reclaim(system)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	reclaimCmd.Flags().String("system-dbpath", path, "System db path")
+	reclaimCmd.Flags().String("system-target", path, "System rootpath")
+	reclaimCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+
+	RootCmd.AddCommand(reclaimCmd)
+}

cmd/repo.go

@@ -0,0 +1,37 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/repo"
+
+	"github.com/spf13/cobra"
+)
+
+var repoGroupCmd = &cobra.Command{
+	Use:   "repo [command] [OPTIONS]",
+	Short: "Manage repositories",
+}
+
+func init() {
+	RootCmd.AddCommand(repoGroupCmd)
+
+	repoGroupCmd.AddCommand(
+		NewRepoListCommand(),
+		NewRepoUpdateCommand(),
+	)
+}

cmd/repo/list.go

@@ -0,0 +1,101 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_repo
+
+import (
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	. "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+
+	. "github.com/logrusorgru/aurora"
+	"github.com/spf13/cobra"
+)
+
+func NewRepoListCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "list [OPTIONS]",
+		Short: "List of the configured repositories.",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var repoColor, repoText, repoRevision string
+
+			enable, _ := cmd.Flags().GetBool("enabled")
+			quiet, _ := cmd.Flags().GetBool("quiet")
+			repoType, _ := cmd.Flags().GetString("type")
+
+			for _, repo := range LuetCfg.SystemRepositories {
+				if enable && !repo.Enable {
+					continue
+				}
+
+				if repoType != "" && repo.Type != repoType {
+					continue
+				}
+
+				repoRevision = ""
+
+				if quiet {
+					fmt.Println(repo.Name)
+				} else {
+					if repo.Enable {
+						repoColor = Bold(BrightGreen(repo.Name)).String()
+					} else {
+						repoColor = Bold(BrightRed(repo.Name)).String()
+					}
+					if repo.Description != "" {
+						repoText = Yellow(repo.Description).String()
+					} else {
+						repoText = Yellow(repo.Urls[0]).String()
+					}
+
+					repobasedir := LuetCfg.GetSystem().GetRepoDatabaseDirPath(repo.Name)
+					if repo.Cached {
+
+						r := installer.NewSystemRepository(repo)
+						localRepo, _ := r.(*installer.LuetSystemRepository).ReadSpecFile(filepath.Join(repobasedir,
+							installer.REPOSITORY_SPECFILE), false)
+						if localRepo != nil {
+							tsec, _ := strconv.ParseInt(localRepo.GetLastUpdate(), 10, 64)
+							repoRevision = Bold(Red(localRepo.GetRevision())).String() +
+								" - " + Bold(Green(time.Unix(tsec, 0).String())).String()
+						}
+					}
+
+					if repoRevision != "" {
+						fmt.Println(
+							fmt.Sprintf("%s\n  %s\n  Revision %s", repoColor, repoText, repoRevision))
+					} else {
+						fmt.Println(
+							fmt.Sprintf("%s\n  %s", repoColor, repoText))
+					}
+				}
+			}
+		},
+	}
+
+	ans.Flags().Bool("enabled", false, "Show only enable repositories.")
+	ans.Flags().BoolP("quiet", "q", false, "Show only name of the repositories.")
+	ans.Flags().StringP("type", "t", "", "Filter repositories of a specific type")
+
+	return ans
+}

cmd/repo/update.go

@@ -0,0 +1,84 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_repo
+
+import (
+	. "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+func NewRepoUpdateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "update [repo1] [repo2] [OPTIONS]",
+		Short: "Update a specific cached repository or all cached repositories.",
+		Example: `
+# Update all cached repositories:
+$> luet repo update
+
+# Update only repo1 and repo2
+$> luet repo update repo1 repo2
+`,
+		Aliases: []string{"up"},
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			ignore, _ := cmd.Flags().GetBool("ignore-errors")
+			force, _ := cmd.Flags().GetBool("force")
+
+			if len(args) > 0 {
+				for _, rname := range args {
+					repo, err := LuetCfg.GetSystemRepository(rname)
+					if err != nil && !ignore {
+						Fatal(err.Error())
+					} else if err != nil {
+						continue
+					}
+
+					r := installer.NewSystemRepository(*repo)
+					Spinner(32)
+					_, err = r.Sync(force)
+					if err != nil && !ignore {
+						Fatal("Error on sync repository " + rname + ": " + err.Error())
+					}
+					SpinnerStop()
+				}
+
+			} else {
+				for _, repo := range LuetCfg.SystemRepositories {
+					if repo.Cached && repo.Enable {
+						r := installer.NewSystemRepository(repo)
+						Spinner(32)
+						_, err := r.Sync(force)
+						if err != nil && !ignore {
+							Fatal("Error on sync repository " + r.GetName() + ": " + err.Error())
+						}
+						SpinnerStop()
+					}
+				}
+			}
+		},
+	}
+
+	ans.Flags().BoolP("ignore-errors", "i", false, "Ignore errors on sync repositories.")
+	ans.Flags().BoolP("force", "f", false, "Force resync.")
+
+	return ans
+}

cmd/root.go

@@ -16,72 +16,219 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
-	"path"
+	"os/user"
 	"path/filepath"
+	"runtime"
+	"strings"
 
+	"github.com/marcsauter/single"
+	extensions "github.com/mudler/cobra-extensions"
+	config "github.com/mudler/luet/pkg/config"
+	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
-
+	repo "github.com/mudler/luet/pkg/repository"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 var cfgFile string
 var Verbose bool
+var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
-const LuetCLIVersion = "0.1"
+const (
+	LuetCLIVersion = "0.9.1"
+	LuetEnvPrefix  = "LUET"
+)
+
+// Build time and commit information.
+//
+// ⚠️ WARNING: should only be set by "-ldflags".
+var (
+	BuildTime   string
+	BuildCommit string
+)
 
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
 	Use:     "luet",
 	Short:   "Package manager for the XXth century!",
 	Long:    `Package manager which uses containers to build packages`,
-	Version: LuetCLIVersion,
+	Version: fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime),
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		err := LoadConfig(config.LuetCfg)
+		if err != nil {
+			Fatal("failed to load configuration:", err.Error())
+		}
+		// Initialize tmpdir prefix. TODO: Move this with LoadConfig
+		// directly on sub command to ensure the creation only when it's
+		// needed.
+		err = config.LuetCfg.GetSystem().InitTmpDir()
+		if err != nil {
+			Fatal("failed on init tmp basedir:", err.Error())
+		}
+	},
+	PersistentPostRun: func(cmd *cobra.Command, args []string) {
+		// Cleanup all tmp directories used by luet
+		err := config.LuetCfg.GetSystem().CleanupTmpDir()
+		if err != nil {
+			Warning("failed on cleanup tmpdir:", err.Error())
+		}
+	},
+	SilenceErrors: true,
+}
+
+func LoadConfig(c *config.LuetConfig) error {
+	// If a config file is found, read it in.
+	c.Viper.ReadInConfig()
+
+	err := c.Viper.Unmarshal(&config.LuetCfg)
+	if err != nil {
+		return err
+	}
+
+	noSpinner := c.Viper.GetBool("no_spinner")
+
+	InitAurora()
+	if !noSpinner {
+		NewSpinner()
+	}
+
+	Debug("Using config file:", c.Viper.ConfigFileUsed())
+
+	if c.GetLogging().EnableLogFile && c.GetLogging().Path != "" {
+		// Init zap logger
+		err = ZapLogger()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Load repositories
+	err = repo.LoadRepositories(c)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
+
+	if os.Getenv("LUET_NOLOCK") != "true" {
+		if len(os.Args) > 1 {
+			for _, lockedCmd := range LockedCommands {
+				if os.Args[1] == lockedCmd {
+					s := single.New("luet")
+					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
+						Fatal("another instance of the app is already running, exiting")
+					} else if err != nil {
+						// Another error occurred, might be worth handling it as well
+						Fatal("failed to acquire exclusive app lock:", err.Error())
+					}
+					defer s.TryUnlock()
+					break
+				}
+			}
+		}
+	}
+
 	if err := RootCmd.Execute(); err != nil {
-		Error(err)
+		fmt.Println(err)
 		os.Exit(-1)
 	}
 }
 
 func init() {
 	cobra.OnInitialize(initConfig)
-	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
-	RootCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
+	pflags := RootCmd.PersistentFlags()
+	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
+	pflags.BoolP("debug", "d", false, "verbose output")
+	pflags.Bool("fatal", false, "Enables Warnings to exit")
+	pflags.Bool("enable-logfile", false, "Enable log to file")
+	pflags.Bool("no-spinner", false, "Disable spinner.")
+	pflags.Bool("color", config.LuetCfg.GetLogging().Color, "Enable/Disable color.")
+	pflags.Bool("emoji", config.LuetCfg.GetLogging().EnableEmoji, "Enable/Disable emoji.")
+	pflags.Bool("skip-config-protect", config.LuetCfg.ConfigProtectSkip,
+		"Disable config protect analysis.")
+	pflags.StringP("logfile", "l", config.LuetCfg.GetLogging().Path,
+		"Logfile path. Empty value disable log to file.")
+
+	// os/user doesn't work in from scratch environments.
+	// Check if i can retrieve user informations.
+	_, err := user.Current()
+	if err != nil {
+		Warning("failed to retrieve user identity:", err.Error())
+	}
+	pflags.Bool("same-owner", config.LuetCfg.GetGeneral().SameOwner, "Maintain same owner on uncompress.")
+	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
+
+	config.LuetCfg.Viper.BindPFlag("logging.color", pflags.Lookup("color"))
+	config.LuetCfg.Viper.BindPFlag("logging.enable_emoji", pflags.Lookup("emoji"))
+	config.LuetCfg.Viper.BindPFlag("logging.enable_logfile", pflags.Lookup("enable-logfile"))
+	config.LuetCfg.Viper.BindPFlag("logging.path", pflags.Lookup("logfile"))
+
+	config.LuetCfg.Viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
+	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("debug"))
+	config.LuetCfg.Viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
+	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
+	// Currently I maintain this only from cli.
+	config.LuetCfg.Viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))
+	config.LuetCfg.Viper.BindPFlag("config_protect_skip", pflags.Lookup("skip-config-protect"))
+
+	// Extensions must be binary with the "luet-" prefix to be able to be shown in the help.
+	// we also accept extensions in the relative path where luet is being started, "extensions/"
+	exts := extensions.Discover("luet", "extensions")
+	for _, ex := range exts {
+		cobraCmd := ex.CobraCommand()
+		RootCmd.AddCommand(cobraCmd)
+	}
+
 }
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
-	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
-	if err != nil {
-		Error(err)
-		os.Exit(1)
-	}
+	// Luet support these priorities on read configuration file:
+	// - command line option (if available)
+	// - $PWD/.luet.yaml
+	// - $HOME/.luet.yaml
+	// - /etc/luet/luet.yaml
+	//
+	// Note: currently a single viper instance support only one config name.
 
-	viper.SetConfigName(".luet") // name of config file (without extension)
-	if cfgFile != "" {           // enable ability to specify config file via flag
-		Info(">>> cfgFile: ", cfgFile)
+	viper.SetEnvPrefix(LuetEnvPrefix)
+	viper.SetConfigType("yaml")
+
+	if cfgFile != "" { // enable ability to specify config file via flag
 		viper.SetConfigFile(cfgFile)
-		configDir := path.Dir(cfgFile)
-		if configDir != "." && configDir != dir {
-			viper.AddConfigPath(configDir)
+	} else {
+		// Retrieve pwd directory
+		pwdDir, err := os.Getwd()
+		if err != nil {
+			Error(err)
+			os.Exit(1)
+		}
+		homeDir := helpers.GetHomeDir()
+
+		if helpers.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && helpers.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
+			viper.AddConfigPath(".")
+			if homeDir != "" {
+				viper.AddConfigPath(homeDir)
+			}
+			viper.SetConfigName(".luet")
+		} else {
+			viper.SetConfigName("luet")
+			viper.AddConfigPath("/etc/luet")
 		}
 	}
 
-	viper.AddConfigPath(dir)
-	viper.AddConfigPath(".")
-	viper.AddConfigPath("$HOME")
 	viper.AutomaticEnv() // read in environment variables that match
 
-	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err == nil {
-		Info("Using config file:", viper.ConfigFileUsed())
-	} else {
-		Warning(err)
-	}
-
+	// Create EnvKey Replacer for handle complex structure
+	replacer := strings.NewReplacer(".", "__")
+	viper.SetEnvKeyReplacer(replacer)
+	viper.SetTypeByDefaultValue(true)
 }

cmd/search.go

@@ -0,0 +1,250 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/ghodss/yaml"
+	. "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/spf13/cobra"
+)
+
+type PackageResult struct {
+	Name       string `json:"name"`
+	Category   string `json:"category"`
+	Version    string `json:"version"`
+	Repository string `json:"repository"`
+	Target     string `json:"target"`
+	Hidden     bool   `json:"hidden"`
+}
+
+type Results struct {
+	Packages []PackageResult `json:"packages"`
+}
+
+func (r PackageResult) String() string {
+	return fmt.Sprintf("%s/%s-%s required for %s", r.Category, r.Name, r.Version, r.Target)
+}
+
+var searchCmd = &cobra.Command{
+	Use:     "search <term>",
+	Short:   "Search packages",
+	Long:    `Search for installed and available packages`,
+	Aliases: []string{"s"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var systemDB pkg.PackageDatabase
+		var results Results
+		if len(args) != 1 {
+			Fatal("Wrong number of arguments (expected 1)")
+		}
+
+		hidden, _ := cmd.Flags().GetBool("hidden")
+
+		installed := LuetCfg.Viper.GetBool("installed")
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		searchWithLabel, _ := cmd.Flags().GetBool("by-label")
+		searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex")
+		revdeps, _ := cmd.Flags().GetBool("revdeps")
+
+		out, _ := cmd.Flags().GetString("output")
+		if out != "terminal" {
+			LuetCfg.GetLogging().SetLogLevel("error")
+		}
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		if !installed {
+
+			repos := installer.Repositories{}
+			for _, repo := range LuetCfg.SystemRepositories {
+				if !repo.Enable {
+					continue
+				}
+				r := installer.NewSystemRepository(repo)
+				repos = append(repos, r)
+			}
+
+			inst := installer.NewLuetInstaller(
+				installer.LuetInstallerOptions{
+					Concurrency:   LuetCfg.GetGeneral().Concurrency,
+					SolverOptions: *LuetCfg.GetSolverOptions(),
+				},
+			)
+			inst.Repositories(repos)
+			synced, err := inst.SyncRepositories(false)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
+			Info("--- Search results (" + args[0] + "): ---")
+
+			matches := []installer.PackageMatch{}
+			if searchWithLabel {
+				matches = synced.SearchLabel(args[0])
+			} else if searchWithLabelMatch {
+				matches = synced.SearchLabelMatch(args[0])
+			} else {
+				matches = synced.Search(args[0])
+			}
+			for _, m := range matches {
+				if !revdeps {
+					if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
+						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
+						results.Packages = append(results.Packages,
+							PackageResult{
+								Name:       m.Package.GetName(),
+								Version:    m.Package.GetVersion(),
+								Category:   m.Package.GetCategory(),
+								Repository: m.Repo.GetName(),
+								Hidden:     m.Package.IsHidden(),
+							})
+					}
+				} else {
+					visited := make(map[string]interface{})
+					for _, revdep := range m.Package.ExpandedRevdeps(m.Repo.GetTree().GetDatabase(), visited) {
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: m.Repo.GetName(),
+									Hidden:     revdep.IsHidden(),
+								})
+						}
+					}
+				}
+			}
+		} else {
+
+			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+				systemDB = pkg.NewBoltDatabase(
+					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+			} else {
+				systemDB = pkg.NewInMemoryDatabase(true)
+			}
+			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+
+			var err error
+			iMatches := pkg.Packages{}
+			if searchWithLabel {
+				iMatches, err = system.Database.FindPackageLabel(args[0])
+			} else if searchWithLabelMatch {
+				iMatches, err = system.Database.FindPackageLabelMatch(args[0])
+			} else {
+				iMatches, err = system.Database.FindPackageMatch(args[0])
+			}
+
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
+			for _, pack := range iMatches {
+				if !revdeps {
+					if !pack.IsHidden() || pack.IsHidden() && hidden {
+						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+						results.Packages = append(results.Packages,
+							PackageResult{
+								Name:       pack.GetName(),
+								Version:    pack.GetVersion(),
+								Category:   pack.GetCategory(),
+								Repository: "system",
+								Hidden:     pack.IsHidden(),
+							})
+					}
+				} else {
+					visited := make(map[string]interface{})
+
+					for _, revdep := range pack.ExpandedRevdeps(system.Database, visited) {
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: "system",
+									Hidden:     revdep.IsHidden(),
+								})
+						}
+					}
+				}
+			}
+		}
+
+		y, err := yaml.Marshal(results)
+		if err != nil {
+			fmt.Printf("err: %v\n", err)
+			return
+		}
+		switch out {
+		case "yaml":
+			fmt.Println(string(y))
+		case "json":
+			j2, err := yaml.YAMLToJSON(y)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			fmt.Println(string(j2))
+		}
+
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	searchCmd.Flags().String("system-dbpath", path, "System db path")
+	searchCmd.Flags().String("system-target", path, "System rootpath")
+	searchCmd.Flags().Bool("installed", false, "Search between system packages")
+	searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	searchCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+	searchCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	searchCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	searchCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	searchCmd.Flags().Bool("by-label", false, "Search packages through label")
+	searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex")
+	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
+
+	RootCmd.AddCommand(searchCmd)
+}

cmd/serve-repo.go

@@ -0,0 +1,59 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"net/http"
+	"os"
+
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var serverepoCmd = &cobra.Command{
+	Use:   "serve-repo",
+	Short: "Embedded micro-http server",
+	Long:  `Embedded mini http server for serving local repositories`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("dir", cmd.Flags().Lookup("dir"))
+		viper.BindPFlag("address", cmd.Flags().Lookup("address"))
+		viper.BindPFlag("port", cmd.Flags().Lookup("port"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+
+		dir := viper.GetString("dir")
+		port := viper.GetString("port")
+		address := viper.GetString("address")
+
+		http.Handle("/", http.FileServer(http.Dir(dir)))
+
+		Info("Serving ", dir, " on HTTP port: ", port)
+		Fatal(http.ListenAndServe(address+":"+port, nil))
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	serverepoCmd.Flags().String("dir", path, "Packages folder (output from build)")
+	serverepoCmd.Flags().String("port", "9090", "Listening port")
+	serverepoCmd.Flags().String("address", "0.0.0.0", "Listening address")
+
+	RootCmd.AddCommand(serverepoCmd)
+}

cmd/tree.go

@@ -0,0 +1,38 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/tree"
+
+	"github.com/spf13/cobra"
+)
+
+var treeGroupCmd = &cobra.Command{
+	Use:   "tree [command] [OPTIONS]",
+	Short: "Tree operations",
+}
+
+func init() {
+	RootCmd.AddCommand(treeGroupCmd)
+
+	treeGroupCmd.AddCommand(
+		NewTreePkglistCommand(),
+		NewTreeValidateCommand(),
+		NewTreeBumpCommand(),
+	)
+}

cmd/tree/bump.go

@@ -0,0 +1,88 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"fmt"
+
+	. "github.com/mudler/luet/pkg/logger"
+	spectooling "github.com/mudler/luet/pkg/spectooling"
+	tree "github.com/mudler/luet/pkg/tree"
+	version "github.com/mudler/luet/pkg/versioner"
+
+	"github.com/spf13/cobra"
+)
+
+func NewTreeBumpCommand() *cobra.Command {
+
+	var ans = &cobra.Command{
+		Use:   "bump [OPTIONS]",
+		Short: "Bump a new package build version.",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			df, _ := cmd.Flags().GetString("definition-file")
+			if df == "" {
+				Fatal("Mandatory definition.yaml path missing.")
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			spec, _ := cmd.Flags().GetString("definition-file")
+			toStdout, _ := cmd.Flags().GetBool("to-stdout")
+			pkgVersion, _ := cmd.Flags().GetString("pkg-version")
+			pack, err := tree.ReadDefinitionFile(spec)
+			if err != nil {
+				Fatal(err.Error())
+			}
+
+			if pkgVersion != "" {
+				validator := &version.WrappedVersioner{}
+				err := validator.Validate(pkgVersion)
+				if err != nil {
+					Fatal("Invalid version string: " + err.Error())
+				}
+				pack.SetVersion(pkgVersion)
+			} else {
+				// Retrieve version build section with Gentoo parser
+				err = pack.BumpBuildVersion()
+				if err != nil {
+					Fatal("Error on increment build version: " + err.Error())
+				}
+			}
+			if toStdout {
+				data, err := spectooling.NewDefaultPackageSanitized(&pack).Yaml()
+				if err != nil {
+					Fatal("Error on yaml conversion: " + err.Error())
+				}
+				fmt.Println(string(data))
+			} else {
+
+				err = tree.WriteDefinitionFile(&pack, spec)
+				if err != nil {
+					Fatal("Error on write definition file: " + err.Error())
+				}
+
+				fmt.Printf("Bumped package %s/%s-%s.\n", pack.Category, pack.Name, pack.Version)
+			}
+		},
+	}
+
+	ans.Flags().StringP("pkg-version", "p", "", "Set a specific package version")
+	ans.Flags().StringP("definition-file", "f", "", "Path of the definition to bump.")
+	ans.Flags().BoolP("to-stdout", "o", false, "Bump package to output.")
+
+	return ans
+}

cmd/tree/pkglist.go

@@ -0,0 +1,284 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"fmt"
+	"sort"
+
+	//. "github.com/mudler/luet/pkg/config"
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/spf13/cobra"
+)
+
+type TreePackageResult struct {
+	Name     string `json:"name"`
+	Category string `json:"category"`
+	Version  string `json:"version"`
+	Path     string `json:"path"`
+}
+
+type TreeResults struct {
+	Packages []TreePackageResult `json:"packages"`
+}
+
+func pkgDetail(pkg pkg.Package) string {
+	ans := fmt.Sprintf(`
+  @@ Package: %s/%s-%s
+     Description: %s
+     License:     %s`,
+		pkg.GetCategory(), pkg.GetName(), pkg.GetVersion(),
+		pkg.GetDescription(), pkg.GetLicense())
+
+	for idx, u := range pkg.GetURI() {
+		if idx == 0 {
+			ans += fmt.Sprintf("     URLs:        %s", u)
+		} else {
+			ans += fmt.Sprintf("                  %s", u)
+		}
+	}
+
+	return ans
+}
+
+func NewTreePkglistCommand() *cobra.Command {
+	var excludes []string
+	var matches []string
+
+	var ans = &cobra.Command{
+		Use:   "pkglist [OPTIONS]",
+		Short: "List of the packages found in tree.",
+		Args:  cobra.NoArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			t, _ := cmd.Flags().GetStringArray("tree")
+			if len(t) == 0 {
+				Fatal("Mandatory tree param missing.")
+			}
+
+			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
+			if revdeps && deps {
+				Fatal("Both revdeps and deps option used. Choice only one.")
+			}
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var results TreeResults
+			var depSolver solver.PackageSolver
+
+			treePath, _ := cmd.Flags().GetStringArray("tree")
+			verbose, _ := cmd.Flags().GetBool("verbose")
+			buildtime, _ := cmd.Flags().GetBool("buildtime")
+			full, _ := cmd.Flags().GetBool("full")
+			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
+
+			out, _ := cmd.Flags().GetString("output")
+			if out != "terminal" {
+				LuetCfg.GetLogging().SetLogLevel("error")
+			}
+
+			var reciper tree.Builder
+			if buildtime {
+				reciper = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			} else {
+				reciper = tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
+			}
+
+			for _, t := range treePath {
+				err := reciper.Load(t)
+				if err != nil {
+					Fatal("Error on load tree ", err)
+				}
+			}
+
+			if deps {
+				emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+
+				depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
+					reciper.GetDatabase(),
+					emptyInstallationDb)
+
+			}
+
+			regExcludes, err := helpers.CreateRegexArray(excludes)
+			if err != nil {
+				Fatal(err.Error())
+			}
+			regMatches, err := helpers.CreateRegexArray(matches)
+			if err != nil {
+				Fatal(err.Error())
+			}
+
+			plist := make([]string, 0)
+			for _, p := range reciper.GetDatabase().World() {
+				pkgstr := ""
+				addPkg := true
+				if full {
+					pkgstr = pkgDetail(p)
+				} else if verbose {
+					pkgstr = p.HumanReadableString()
+				} else {
+					pkgstr = fmt.Sprintf("%s/%s", p.GetCategory(), p.GetName())
+				}
+
+				if len(matches) > 0 {
+					matched := false
+					for _, rgx := range regMatches {
+						if rgx.MatchString(pkgstr) {
+							matched = true
+							break
+						}
+					}
+					if !matched {
+						addPkg = false
+					}
+				}
+
+				if len(excludes) > 0 && addPkg {
+					for _, rgx := range regExcludes {
+						if rgx.MatchString(pkgstr) {
+							addPkg = false
+							break
+						}
+					}
+				}
+
+				if addPkg {
+					if revdeps {
+
+						visited := make(map[string]interface{})
+						for _, revdep := range p.ExpandedRevdeps(reciper.GetDatabase(), visited) {
+							if full {
+								pkgstr = pkgDetail(revdep)
+							} else if verbose {
+								pkgstr = revdep.HumanReadableString()
+							} else {
+								pkgstr = fmt.Sprintf("%s/%s", revdep.GetCategory(), revdep.GetName())
+							}
+							plist = append(plist, pkgstr)
+							results.Packages = append(results.Packages, TreePackageResult{
+								Name:     revdep.GetName(),
+								Version:  revdep.GetVersion(),
+								Category: revdep.GetCategory(),
+								Path:     revdep.GetPath(),
+							})
+						}
+					} else if deps {
+
+						Spinner(32)
+						solution, err := depSolver.Install(pkg.Packages{p})
+						if err != nil {
+							Fatal(err.Error())
+						}
+						ass := solution.SearchByName(p.GetPackageName())
+						solution, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+						if err != nil {
+							Fatal(err.Error())
+						}
+						SpinnerStop()
+
+						for _, pa := range solution {
+
+							if pa.Value {
+								// Exclude itself
+								if pa.Package.GetName() == p.GetName() && pa.Package.GetCategory() == p.GetCategory() {
+									continue
+								}
+
+								if full {
+									pkgstr = pkgDetail(pa.Package)
+								} else if verbose {
+									pkgstr = pa.Package.HumanReadableString()
+								} else {
+									pkgstr = fmt.Sprintf("%s/%s", pa.Package.GetCategory(), pa.Package.GetName())
+								}
+								plist = append(plist, pkgstr)
+								results.Packages = append(results.Packages, TreePackageResult{
+									Name:     pa.Package.GetName(),
+									Version:  pa.Package.GetVersion(),
+									Category: pa.Package.GetCategory(),
+									Path:     pa.Package.GetPath(),
+								})
+							}
+
+						}
+
+					} else {
+
+						plist = append(plist, pkgstr)
+						results.Packages = append(results.Packages, TreePackageResult{
+							Name:     p.GetName(),
+							Version:  p.GetVersion(),
+							Category: p.GetCategory(),
+							Path:     p.GetPath(),
+						})
+
+					}
+
+				}
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			default:
+				if !deps {
+					sort.Strings(plist)
+				}
+				for _, p := range plist {
+					fmt.Println(p)
+				}
+			}
+
+		},
+	}
+
+	ans.Flags().BoolP("buildtime", "b", false, "Build time match")
+	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+	ans.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	ans.Flags().Bool("deps", false, "Search package dependencies")
+
+	ans.Flags().BoolP("verbose", "v", false, "Add package version")
+	ans.Flags().BoolP("full", "f", false, "Show package detail")
+	ans.Flags().StringArrayP("tree", "t", []string{}, "Path of the tree to use.")
+	ans.Flags().StringSliceVarP(&matches, "matches", "m", []string{},
+		"Include only matched packages from list. (Use string as regex).")
+	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},
+		"Exclude matched packages from list. (Use string as regex).")
+
+	return ans
+}

cmd/tree/validate.go

@@ -0,0 +1,474 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"sync"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/spf13/cobra"
+)
+
+type ValidateOpts struct {
+	WithSolver    bool
+	OnlyRuntime   bool
+	OnlyBuildtime bool
+	RegExcludes   []*regexp.Regexp
+	RegMatches    []*regexp.Regexp
+	Excludes      []string
+	Matches       []string
+
+	// Runtime validate stuff
+	RuntimeCacheDeps *pkg.InMemoryDatabase
+	RuntimeReciper   *tree.InstallerRecipe
+
+	// Buildtime validate stuff
+	BuildtimeCacheDeps *pkg.InMemoryDatabase
+	BuildtimeReciper   *tree.CompilerRecipe
+
+	Mutex      sync.Mutex
+	BrokenPkgs int
+	BrokenDeps int
+
+	Errors []error
+}
+
+func (o *ValidateOpts) IncrBrokenPkgs() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenPkgs++
+}
+
+func (o *ValidateOpts) IncrBrokenDeps() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenDeps++
+}
+
+func (o *ValidateOpts) AddError(err error) {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.Errors = append(o.Errors, err)
+}
+
+func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, reciper tree.Builder, cacheDeps *pkg.InMemoryDatabase) error {
+	var errstr string
+	var ans error
+
+	var depSolver solver.PackageSolver
+
+	if opts.WithSolver {
+		emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+		depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
+			reciper.GetDatabase(),
+			emptyInstallationDb)
+	}
+
+	found, err := reciper.GetDatabase().FindPackages(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  ">=0",
+		},
+	)
+
+	if err != nil || len(found) < 1 {
+		if err != nil {
+			errstr = err.Error()
+		} else {
+			errstr = "No packages"
+		}
+		Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			errstr,
+		))
+
+		opts.IncrBrokenDeps()
+
+		return errors.New(
+			fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+				checkType,
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				errstr,
+			))
+	}
+
+	// Ensure that we use the right package from right recipier for deps
+	pReciper, err := reciper.GetDatabase().FindPackage(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  p.GetVersion(),
+		},
+	)
+	if err != nil {
+		errstr = fmt.Sprintf("[%9s] %s/%s-%s: Error on retrieve package - %s.",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			err.Error(),
+		)
+		Error(errstr)
+
+		return errors.New(errstr)
+	}
+	p = pReciper
+
+	pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
+		p.GetVersion())
+
+	validpkg := true
+
+	if len(opts.Matches) > 0 {
+		matched := false
+		for _, rgx := range opts.RegMatches {
+			if rgx.MatchString(pkgstr) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return nil
+		}
+	}
+
+	if len(opts.Excludes) > 0 {
+		excluded := false
+		for _, rgx := range opts.RegExcludes {
+			if rgx.MatchString(pkgstr) {
+				excluded = true
+				break
+			}
+		}
+
+		if excluded {
+			return nil
+		}
+	}
+
+	Info(fmt.Sprintf("[%9s] Checking package ", checkType)+
+		fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
+		"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
+
+	all := p.GetRequires()
+	all = append(all, p.GetConflicts()...)
+	for idx, r := range all {
+
+		var deps pkg.Packages
+		var err error
+		if r.IsSelector() {
+			deps, err = reciper.GetDatabase().FindPackages(
+				&pkg.DefaultPackage{
+					Name:     r.GetName(),
+					Category: r.GetCategory(),
+					Version:  r.GetVersion(),
+				},
+			)
+		} else {
+			deps = append(deps, r)
+		}
+
+		if err != nil || len(deps) < 1 {
+			if err != nil {
+				errstr = err.Error()
+			} else {
+				errstr = "No packages"
+			}
+			Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+				checkType,
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				r.GetCategory(), r.GetName(), r.GetVersion(),
+				errstr,
+			))
+
+			opts.IncrBrokenDeps()
+
+			ans = errors.New(
+				fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+					checkType,
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					r.GetCategory(), r.GetName(), r.GetVersion(),
+					errstr))
+
+			validpkg = false
+
+		} else {
+
+			Debug(fmt.Sprintf("[%9s] Find packages for dep", checkType),
+				fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+
+			if opts.WithSolver {
+
+				Info(fmt.Sprintf("[%9s]  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
+					checkType,
+					idx+1, len(all),
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					r.GetCategory(), r.GetName(), r.GetVersion(),
+				))
+
+				// Check if the solver is already been done for the deep
+				_, err := cacheDeps.Get(r.HashFingerprint(""))
+				if err == nil {
+					Debug(fmt.Sprintf("[%9s]  :direct_hit: Cache Hit for dep", checkType),
+						fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+					continue
+				}
+
+				Spinner(32)
+				solution, err := depSolver.Install(pkg.Packages{r})
+				ass := solution.SearchByName(r.GetPackageName())
+				if err == nil {
+					_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+				}
+				SpinnerStop()
+
+				if err != nil {
+
+					Error(fmt.Sprintf("[%9s] %s/%s-%s: solver broken for dep %s/%s-%s - %s",
+						checkType,
+						p.GetCategory(), p.GetName(), p.GetVersion(),
+						r.GetCategory(), r.GetName(), r.GetVersion(),
+						err.Error(),
+					))
+
+					ans = errors.New(
+						fmt.Sprintf("[%9s] %s/%s-%s: solver broken for Dep %s/%s-%s - %s",
+							checkType,
+							p.GetCategory(), p.GetName(), p.GetVersion(),
+							r.GetCategory(), r.GetName(), r.GetVersion(),
+							err.Error()))
+
+					opts.IncrBrokenDeps()
+					validpkg = false
+				}
+
+				// Register the key
+				cacheDeps.Set(r.HashFingerprint(""), "1")
+
+			}
+		}
+
+	}
+
+	if !validpkg {
+		opts.IncrBrokenPkgs()
+	}
+
+	return ans
+}
+
+func validateWorker(i int,
+	wg *sync.WaitGroup,
+	c <-chan pkg.Package,
+	opts *ValidateOpts) {
+
+	defer wg.Done()
+
+	for p := range c {
+
+		if opts.OnlyBuildtime {
+			// Check buildtime compiler/deps
+			err := validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+		} else if opts.OnlyRuntime {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+		} else {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+			// Check buildtime compiler/deps
+			err = validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+			}
+
+		}
+
+	}
+}
+
+func initOpts(opts *ValidateOpts, onlyRuntime, onlyBuildtime, withSolver bool, treePaths []string) {
+	var err error
+
+	opts.OnlyBuildtime = onlyBuildtime
+	opts.OnlyRuntime = onlyRuntime
+	opts.WithSolver = withSolver
+	opts.RuntimeReciper = nil
+	opts.BuildtimeReciper = nil
+	opts.BrokenPkgs = 0
+	opts.BrokenDeps = 0
+
+	if onlyBuildtime {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+	} else if onlyRuntime {
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	} else {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	}
+
+	opts.RuntimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+	opts.BuildtimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+
+	for _, treePath := range treePaths {
+		Info(fmt.Sprintf("Loading :deciduous_tree: %s...", treePath))
+		if opts.BuildtimeReciper != nil {
+			err = opts.BuildtimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
+		}
+		if opts.RuntimeReciper != nil {
+			err = opts.RuntimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
+		}
+	}
+
+	opts.RegExcludes, err = helpers.CreateRegexArray(opts.Excludes)
+	if err != nil {
+		Fatal(err.Error())
+	}
+	opts.RegMatches, err = helpers.CreateRegexArray(opts.Matches)
+	if err != nil {
+		Fatal(err.Error())
+	}
+
+}
+
+func NewTreeValidateCommand() *cobra.Command {
+	var excludes []string
+	var matches []string
+	var treePaths []string
+	var opts ValidateOpts
+
+	var ans = &cobra.Command{
+		Use:   "validate [OPTIONS]",
+		Short: "Validate a tree or a list of packages",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
+			if len(treePaths) < 1 {
+				Fatal("Mandatory tree param missing.")
+			}
+			if onlyRuntime && onlyBuildtime {
+				Fatal("Both --only-runtime and --only-buildtime options are not possibile.")
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var reciper tree.Builder
+
+			concurrency := LuetCfg.GetGeneral().Concurrency
+
+			withSolver, _ := cmd.Flags().GetBool("with-solver")
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
+			opts.Excludes = excludes
+			opts.Matches = matches
+			initOpts(&opts, onlyRuntime, onlyBuildtime, withSolver, treePaths)
+
+			// We need at least one valid reciper for get list of the packages.
+			if onlyBuildtime {
+				reciper = opts.BuildtimeReciper
+			} else {
+				reciper = opts.RuntimeReciper
+			}
+
+			all := make(chan pkg.Package)
+
+			var wg = new(sync.WaitGroup)
+
+			for i := 0; i < concurrency; i++ {
+				wg.Add(1)
+				go validateWorker(i, wg, all, &opts)
+			}
+			for _, p := range reciper.GetDatabase().World() {
+				all <- p
+			}
+			close(all)
+
+			// Wait separately and once done close the channel
+			go func() {
+				wg.Wait()
+			}()
+
+			stringerrs := []string{}
+			for _, e := range opts.Errors {
+				stringerrs = append(stringerrs, e.Error())
+			}
+			sort.Strings(stringerrs)
+			for _, e := range stringerrs {
+				fmt.Println(e)
+			}
+
+			// fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
+			if len(stringerrs) != 0 {
+				Error(fmt.Sprintf("Found %d broken packages and %d broken deps.",
+					opts.BrokenPkgs, opts.BrokenDeps))
+				Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
+			} else {
+				Info("All good! :white_check_mark:")
+				os.Exit(0)
+			}
+		},
+	}
+
+	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
+	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")
+	ans.Flags().BoolP("with-solver", "s", false,
+		"Enable check of requires also with solver.")
+	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{},
+		"Path of the tree to use.")
+	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},
+		"Exclude matched packages from analysis. (Use string as regex).")
+	ans.Flags().StringSliceVarP(&matches, "matches", "m", []string{},
+		"Analyze only matched packages. (Use string as regex).")
+
+	return ans
+}

cmd/uninstall.go

@@ -0,0 +1,125 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/spf13/cobra"
+)
+
+var uninstallCmd = &cobra.Command{
+	Use:     "uninstall <pkg> <pkg2> ...",
+	Short:   "Uninstall a package or a list of packages",
+	Long:    `Uninstall packages`,
+	Aliases: []string{"rm", "un"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var systemDB pkg.PackageDatabase
+
+		for _, a := range args {
+
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+
+			stype := LuetCfg.Viper.GetString("solver.type")
+			discount := LuetCfg.Viper.GetFloat64("solver.discount")
+			rate := LuetCfg.Viper.GetFloat64("solver.rate")
+			attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+			force := LuetCfg.Viper.GetBool("force")
+			nodeps, _ := cmd.Flags().GetBool("nodeps")
+			full, _ := cmd.Flags().GetBool("full")
+			checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
+			fullClean, _ := cmd.Flags().GetBool("full-clean")
+			concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+
+			LuetCfg.GetSolverOptions().Type = stype
+			LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+			LuetCfg.GetSolverOptions().Discount = float32(discount)
+			LuetCfg.GetSolverOptions().MaxAttempts = attempts
+			if concurrent {
+				LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+			} else {
+				LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+			}
+			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+			// Load config protect configs
+			installer.LoadConfigProtectConfs(LuetCfg)
+
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+				Concurrency:        LuetCfg.GetGeneral().Concurrency,
+				SolverOptions:      *LuetCfg.GetSolverOptions(),
+				NoDeps:             nodeps,
+				Force:              force,
+				FullUninstall:      full,
+				FullCleanUninstall: fullClean,
+				CheckConflicts:     checkconflicts,
+			})
+
+			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+				systemDB = pkg.NewBoltDatabase(
+					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+			} else {
+				systemDB = pkg.NewInMemoryDatabase(true)
+			}
+			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+			err = inst.Uninstall(pack, system)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	uninstallCmd.Flags().String("system-dbpath", path, "System db path")
+	uninstallCmd.Flags().String("system-target", path, "System rootpath")
+	uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	uninstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	uninstallCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
+	uninstallCmd.Flags().Bool("force", false, "Force uninstall")
+	uninstallCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
+	uninstallCmd.Flags().Bool("conflictscheck", true, "Check if the package marked for deletion is required by other packages")
+	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
+	uninstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+
+	RootCmd.AddCommand(uninstallCmd)
+}

cmd/upgrade.go

@@ -0,0 +1,134 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	. "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/spf13/cobra"
+)
+
+var upgradeCmd = &cobra.Command{
+	Use:     "upgrade",
+	Short:   "Upgrades the system",
+	Aliases: []string{"u"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", installCmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", installCmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+	},
+	Long: `Upgrades packages in parallel`,
+	Run: func(cmd *cobra.Command, args []string) {
+		var systemDB pkg.PackageDatabase
+
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps, _ := cmd.Flags().GetBool("nodeps")
+		full, _ := cmd.Flags().GetBool("full")
+		universe, _ := cmd.Flags().GetBool("universe")
+		clean, _ := cmd.Flags().GetBool("clean")
+		sync, _ := cmd.Flags().GetBool("sync")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
+		Debug("Solver", LuetCfg.GetSolverOptions().String())
+
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:              *LuetCfg.GetSolverOptions(),
+			Force:                      force,
+			FullUninstall:              full,
+			NoDeps:                     nodeps,
+			SolverUpgrade:              universe,
+			RemoveUnavailableOnUpgrade: clean,
+			UpgradeNewRevisions:        sync,
+		})
+		inst.Repositories(repos)
+		_, err := inst.SyncRepositories(false)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+
+		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+			systemDB = pkg.NewBoltDatabase(
+				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+		} else {
+			systemDB = pkg.NewInMemoryDatabase(true)
+		}
+		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		err = inst.Upgrade(system)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	upgradeCmd.Flags().String("system-dbpath", path, "System db path")
+	upgradeCmd.Flags().String("system-target", path, "System rootpath")
+	upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
+	upgradeCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
+	upgradeCmd.Flags().Bool("full", true, "Attempts to remove as much packages as possible which aren't required (slow)")
+	upgradeCmd.Flags().Bool("universe", false, "Use ONLY the SAT solver to compute upgrades (experimental)")
+	upgradeCmd.Flags().Bool("clean", false, "Try to drop removed packages (experimental, only when --universe is enabled)")
+	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
+	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+
+	RootCmd.AddCommand(upgradeCmd)
+}

contrib/config/config.protect.d/01_etc.yml.example

@@ -0,0 +1,3 @@
+name: "etc_conf"
+dirs:
+  - "/etc/"

contrib/config/get_luet_root.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -ex
+export LUET_NOLOCK=true
+
+LUET_VERSION=0.8.6
+LUET_ROOTFS=${LUET_ROOTFS:-/}
+LUET_DATABASE_PATH=${LUET_DATABASE_PATH:-/var/luet/db}
+LUET_DATABASE_ENGINE=${LUET_DATABASE_ENGINE:-boltdb}
+LUET_CONFIG_PROTECT=${LUET_CONFIG_PROTECT:-1}
+
+wget -q https://github.com/mudler/luet/releases/download/0.8.6/luet-0.8.6-linux-amd64 -O luet
+chmod +x luet
+
+mkdir -p /etc/luet/repos.conf.d || true
+mkdir -p $LUET_DATABASE_PATH || true
+mkdir -p /var/tmp/luet || true
+
+if [ "${LUET_CONFIG_PROTECT}" = "1" ] ; then
+  mkdir -p /etc/luet/config.protect.d || true
+  wget -q  https://raw.githubusercontent.com/mudler/luet/master/contrib/config/config.protect.d/01_etc.yml.example -O /etc/luet/config.protect.d/01_etc.yml
+fi
+wget -q https://raw.githubusercontent.com/mocaccinoOS/repository-index/master/packages/mocaccino-repository-index/mocaccino-repository-index.yml -O /etc/luet/repos.conf.d/mocaccino-repository-index.yml
+
+cat > /etc/luet/luet.yaml <<EOF
+general:
+  debug: false
+system:
+  rootfs: ${LUET_ROOTFS}
+  database_path: "${LUET_DATABASE_PATH}"
+  database_engine: "${LUET_DATABASE_ENGINE}"
+  tmpdir_base: "/var/tmp/luet"
+EOF
+
+./luet install repository/luet repository/mocaccino-repository-index
+./luet install system/luet system/luet-extensions
+
+rm -rf luet

contrib/config/luet.yaml

@@ -0,0 +1,163 @@
+# Luet Configuration File
+#
+# ---------------------------------------------
+# Logging configuration section:
+# ---------------------------------------------
+# logging:
+#   # Enable loggging to file (if path is not empty)
+#   enable_logfile: false
+#
+#   Leave empty to skip logging to file.
+#   path: "/var/log/luet.log"
+#
+#   Set logging level: error|warning|info|debug
+#   level: "info"
+#
+#   Enable JSON log format instead of console mode.
+#   json_format: false.
+#
+#   Disable/Enable color
+#   color: true
+#
+#   Enable/Disable emoji
+#   enable_emoji: true
+#
+# ---------------------------------------------
+# General configuration section:
+# ---------------------------------------------
+# general:
+#   Define max concurrency processes. Default is based of arch: runtime.NumCPU()
+#   concurrency: 1
+#
+#   Enable Debug. If debug is active spinner is disabled.
+#   debug: false
+#
+#   Show output of build execution (docker, img, etc.)
+#   show_build_output: false
+#
+#   Define spinner ms
+#   spinner_ms: 200
+#
+#   Define spinner charset. See https://github.com/briandowns/spinner
+#   spinner_charset: 22
+#
+#   Enable warnings to exit
+#   fatal_warnings: false
+#
+#   Try extracting tree/packages with the same ownership as exists in the archive (default for superuser).
+#   same_owner: false
+#
+# ---------------------------------------------
+# System configuration section:
+# ---------------------------------------------
+# system:
+#
+#   Rootfs path of the luet system. Default is /.
+#   A specific path could be used for test installation to
+#   a chroot environment.
+#   rootfs: "/"
+#
+#   Choice database engine used for luet database.
+#   Supported values: boltdb|memory
+#   database_engine: boltdb
+#
+#   Database path directory where store luet database.
+#   The path is append to rootfs option path.
+#   database_path: "/var/cache/luet"
+#
+#   Define the tmpdir base directory where luet store temporary files.
+#   Default $TMPDIR/tmpluet
+#   tmpdir_base: "/tmp/tmpluet"
+#
+#
+# ---------------------------------------------
+# Repositories configurations directories.
+# ---------------------------------------------
+# Define the list of directories where luet
+# try for files with .yml extension that define
+# luet repository.
+# repos_confdir:
+#   - /etc/luet/repos.conf.d
+#
+#
+# ------------------------------------------------
+# Config protect configuration files directories.
+# -----------------------------------------------
+# Define the list of directories where load
+# configuration files with the list of config
+# protect paths.
+# config_protect_confdir:
+#   - /etc/luet/config.protect.d
+#
+# Permit to ignore rules defined on
+# config protect confdir and packages
+# annotation.
+# config_protect_skip: false
+#
+# The paths used for load repositories and config
+# protects are based on host rootfs.
+# If set to false rootfs path is used as prefix.
+# config_from_host: true
+#
+# System repositories
+# ---------------------------------------------
+# In alternative to define repositories files
+# through repos_confdir option is possible
+# define directly the list of the repositories.
+#
+# repositories:
+#
+#     Name of the repository. It's better that this name is unique. Mandatory.
+#   - name: "repo1"
+#
+#     A user-friendly description of the repository
+#     description: "My luet repo"
+#
+#     Type of the repository. Supported types are: dir|http. Mandatory.
+#     type: "dir"
+#
+#     Define the priority of the repository on research packages. Default is 9999.
+#     priority: 9999
+#
+#     Enable/Disable of the repository.
+#     enable: false
+#
+#     Cached repository. If true a local cache of the remote repository tree is maintained
+#     locally in the $tree_path else it is used a temporary directory that is removed when
+#     installation of a package is completed. A cached repository reduce time on search/install
+#     packages. By default caching is disable.
+#     cached: false
+#
+#     Path where store tree of the specifications. Default path is $database_path/repos/$repo_name/treefs
+#     tree_path: "/var/cache/luet/repos/local/treefs"
+#
+#     Path where store repository metadata. Default path is $database_path/repos/$repo_name/meta
+#     meta_path: "/var/cache/luet/repos/local/meta"
+#
+#     Define the list of the URL where retrieve tree and packages.
+#     urls:
+#        - https://mydomain.local/luet/repo1
+#
+#     auth:
+#        Define Basic authentication header
+#        basic: "mybasicauth"
+#        Define token authentication header
+#        token: "mytoken"
+# ---------------------------------------------
+# Solver parameter configuration:
+# ---------------------------------------------
+# solver:
+#
+#   Solver strategy to solve possible conflicts during depedency
+#   solving. Defaults to empty (none). Available: qlearning
+#   type: ""
+#
+#   Solver agent learning rate. 0.1 to 1.0
+#   rate: 0.7
+#
+#   Learning discount factor.
+#   discount: 1.0
+#
+#   Number of overall attempts that the solver has available before bailing out.
+#   max_attempts: 9000
+#

go.mod

@@ -4,33 +4,46 @@ go 1.12
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56 // indirect
-	github.com/Sabayon/pkgs-checker v0.4.1
+	github.com/Sabayon/pkgs-checker v0.7.2
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/briandowns/spinner v1.7.0
-	github.com/crillab/gophersat v1.1.7
-	github.com/docker/docker v0.7.3-0.20180827131323-0c5f8d2b9b23
+	github.com/cavaliercoder/grab v2.0.0+incompatible
+	github.com/crillab/gophersat v1.3.2-0.20201023142334-3fc2ac466765
+	github.com/docker/docker v17.12.0-ce-rc1.0.20200417035958-130b0bc6032c+incompatible
+	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
+	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
 	github.com/ghodss/yaml v1.0.0
-	github.com/go-yaml/yaml v2.1.0+incompatible // indirect
 	github.com/hashicorp/go-version v1.2.0
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
+	github.com/klauspost/pgzip v1.2.1
+	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
+	github.com/kr/pretty v0.2.0 // indirect
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
+	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
 	github.com/mattn/go-isatty v0.0.10 // indirect
-	github.com/mudler/docker-companion v0.4.6-0.20191110154655-b8b364100616
-	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.0
+	github.com/moby/sys/mount v0.1.1-0.20200320164225-6154f11e6840 // indirect
+	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
+	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
+	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
+	github.com/onsi/ginkgo v1.12.1
+	github.com/onsi/gomega v1.10.0
+	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
 	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
-	github.com/pkg/errors v0.8.1
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.5.0
-	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
-	go.etcd.io/bbolt v1.3.3
-	golang.org/x/crypto v0.0.0-20191028145041-f83a4685e152 // indirect
-	golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4 // indirect
-	gopkg.in/yaml.v2 v2.2.5
-	mvdan.cc/sh v2.6.4+incompatible
+	github.com/pkg/errors v0.9.1
+	github.com/spf13/cobra v1.0.0
+	github.com/spf13/viper v1.6.3
+	go.etcd.io/bbolt v1.3.4
+	go.uber.org/atomic v1.5.1 // indirect
+	go.uber.org/multierr v1.4.0 // indirect
+	go.uber.org/zap v1.13.0
+	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
+	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
+	gopkg.in/yaml.v2 v2.2.8
+	gotest.tools/v3 v3.0.2 // indirect
+	helm.sh/helm/v3 v3.3.4
+	mvdan.cc/sh/v3 v3.0.0-beta1
 )
+
+replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200605210607-749178b8f80d+incompatible

pkg/box/exec.go

@@ -0,0 +1,185 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package box
+
+import (
+	b64 "encoding/base64"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/errors"
+
+	helpers "github.com/mudler/luet/pkg/helpers"
+)
+
+type Box interface {
+	Run() error
+	Exec() error
+}
+
+type DefaultBox struct {
+	Name                  string
+	Root                  string
+	Env                   []string
+	Cmd                   string
+	Args                  []string
+	HostMounts            []string
+	Stdin, Stdout, Stderr bool
+}
+
+func NewBox(cmd string, args, hostmounts, env []string, rootfs string, stdin, stdout, stderr bool) Box {
+	return &DefaultBox{
+		Stdin:      stdin,
+		Stdout:     stdout,
+		Stderr:     stderr,
+		Cmd:        cmd,
+		Args:       args,
+		Root:       rootfs,
+		HostMounts: hostmounts,
+		Env:        env,
+	}
+}
+
+func (b *DefaultBox) Exec() error {
+
+	if err := mountProc(b.Root); err != nil {
+		return errors.Wrap(err, "Failed mounting proc on rootfs")
+	}
+	if err := mountDev(b.Root); err != nil {
+		return errors.Wrap(err, "Failed mounting dev on rootfs")
+	}
+
+	for _, hostMount := range b.HostMounts {
+		target := hostMount
+		if strings.Contains(hostMount, ":") {
+			dest := strings.Split(hostMount, ":")
+			if len(dest) != 2 {
+				return errors.New("Invalid arguments for mount, it can be: fullpath, or source:target")
+			}
+			hostMount = dest[0]
+			target = dest[1]
+		}
+		if err := mountBind(hostMount, b.Root, target); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed mounting %s on rootfs", hostMount))
+		}
+	}
+
+	if err := PivotRoot(b.Root); err != nil {
+		return errors.Wrap(err, "Failed switching pivot on rootfs")
+	}
+	cmd := exec.Command(b.Cmd, b.Args...)
+
+	if b.Stdin {
+		cmd.Stdin = os.Stdin
+	}
+
+	if b.Stderr {
+		cmd.Stderr = os.Stderr
+	}
+
+	if b.Stdout {
+		cmd.Stdout = os.Stdout
+	}
+
+	cmd.Env = b.Env
+
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Error running the %s command in box.Exec", b.Cmd))
+	}
+	return nil
+}
+
+func (b *DefaultBox) Run() error {
+
+	if !helpers.Exists(b.Root) {
+		return errors.New(b.Root + " does not exist")
+	}
+
+	// This matches with exec CLI command in luet
+	// TODO: Pass by env var as well
+	execCmd := []string{"exec", "--rootfs", b.Root, "--entrypoint", b.Cmd}
+
+	if b.Stdin {
+		execCmd = append(execCmd, "--stdin")
+	}
+
+	if b.Stderr {
+		execCmd = append(execCmd, "--stderr")
+	}
+
+	if b.Stdout {
+		execCmd = append(execCmd, "--stdout")
+	}
+	// Encode the command in base64 to avoid bad input from the args given
+	execCmd = append(execCmd, "--decode")
+
+	for _, m := range b.HostMounts {
+		execCmd = append(execCmd, "--mount")
+		execCmd = append(execCmd, m)
+	}
+
+	for _, e := range b.Env {
+		execCmd = append(execCmd, "--env")
+		execCmd = append(execCmd, e)
+	}
+
+	for _, a := range b.Args {
+		execCmd = append(execCmd, b64.StdEncoding.EncodeToString([]byte(a)))
+	}
+
+	cmd := exec.Command("/proc/self/exe", execCmd...)
+	if b.Stdin {
+		cmd.Stdin = os.Stdin
+	}
+
+	if b.Stderr {
+		cmd.Stderr = os.Stderr
+	}
+
+	if b.Stdout {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Cloneflags: syscall.CLONE_NEWNS |
+			syscall.CLONE_NEWUTS |
+			syscall.CLONE_NEWIPC |
+			syscall.CLONE_NEWPID |
+			syscall.CLONE_NEWNET |
+			syscall.CLONE_NEWUSER,
+		UidMappings: []syscall.SysProcIDMap{
+			{
+				ContainerID: 0,
+				HostID:      os.Getuid(),
+				Size:        1,
+			},
+		},
+		GidMappings: []syscall.SysProcIDMap{
+			{
+				ContainerID: 0,
+				HostID:      os.Getgid(),
+				Size:        1,
+			},
+		},
+	}
+
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "Failed running Box command in box.Run")
+	}
+	return nil
+}

pkg/box/rootfs.go

@@ -0,0 +1,94 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package box
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+func PivotRoot(newroot string) error {
+	putold := filepath.Join(newroot, "/.pivot_root")
+
+	// bind mount newroot to itself - this is a slight hack needed to satisfy the
+	// pivot_root requirement that newroot and putold must not be on the same
+	// filesystem as the current root
+	if err := syscall.Mount(newroot, newroot, "", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
+		return err
+	}
+
+	// create putold directory
+	if err := os.MkdirAll(putold, 0700); err != nil {
+		return err
+	}
+
+	// call pivot_root
+	if err := syscall.PivotRoot(newroot, putold); err != nil {
+		return err
+	}
+
+	// ensure current working directory is set to new root
+	if err := os.Chdir("/"); err != nil {
+		return err
+	}
+
+	// umount putold, which now lives at /.pivot_root
+	putold = "/.pivot_root"
+	if err := syscall.Unmount(putold, syscall.MNT_DETACH); err != nil {
+		return err
+	}
+
+	// remove putold
+	if err := os.RemoveAll(putold); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountProc(newroot string) error {
+	source := "proc"
+	target := filepath.Join(newroot, "/proc")
+	fstype := "proc"
+	flags := 0
+	data := ""
+
+	os.MkdirAll(target, 0755)
+	if err := syscall.Mount(source, target, fstype, uintptr(flags), data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountBind(hostfolder, newroot, dst string) error {
+	source := hostfolder
+	target := filepath.Join(newroot, dst)
+	fstype := "bind"
+	data := ""
+
+	os.MkdirAll(target, 0755)
+	if err := syscall.Mount(source, target, fstype, syscall.MS_BIND|syscall.MS_REC, data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountDev(newroot string) error {
+	return mountBind("/dev", newroot, "/dev")
+}

pkg/compiler/artifact.go

@@ -16,31 +16,183 @@
 package compiler
 
 import (
+	"archive/tar"
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
-	"strconv"
+
+	system "github.com/docker/docker/pkg/system"
+	gzip "github.com/klauspost/pgzip"
+
+	//"strconv"
 	"strings"
 	"sync"
 
-	. "github.com/mudler/luet/pkg/logger"
-	"github.com/mudler/luet/pkg/solver"
-
+	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/pkg/errors"
+	yaml "gopkg.in/yaml.v2"
 )
 
-type PackageArtifact struct {
-	Path         string
-	Dependencies []Artifact
-	CompileSpec  CompilationSpec
+type CompressionImplementation string
 
-	SourceAssertion solver.PackagesAssertions
+const (
+	None CompressionImplementation = "none" // e.g. tar for standard packages
+	GZip CompressionImplementation = "gzip"
+)
+
+type ArtifactIndex []Artifact
+
+func (i ArtifactIndex) CleanPath() ArtifactIndex {
+	newIndex := ArtifactIndex{}
+	for _, n := range i {
+		art := n.(*PackageArtifact)
+		// FIXME: This is a dup and makes difficult to add attributes to artifacts
+		newIndex = append(newIndex, &PackageArtifact{
+			Path:            path.Base(n.GetPath()),
+			SourceAssertion: art.SourceAssertion,
+			CompileSpec:     art.CompileSpec,
+			Dependencies:    art.Dependencies,
+			CompressionType: art.CompressionType,
+			Checksums:       art.Checksums,
+			Files:           art.Files,
+		})
+	}
+	return newIndex
+	//Update if exists, otherwise just create
+}
+
+//  When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository
+// which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact.
+// In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages.
+type PackageArtifact struct {
+	Path string `json:"path"`
+
+	Dependencies    []*PackageArtifact        `json:"dependencies"`
+	CompileSpec     *LuetCompilationSpec      `json:"compilationspec"`
+	Checksums       Checksums                 `json:"checksums"`
+	SourceAssertion solver.PackagesAssertions `json:"-"`
+	CompressionType CompressionImplementation `json:"compressiontype"`
+	Files           []string                  `json:"files"`
 }
 
 func NewPackageArtifact(path string) Artifact {
-	return &PackageArtifact{Path: path, Dependencies: []Artifact{}}
+	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: None}
+}
+
+func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
+	p := &PackageArtifact{Checksums: Checksums{}}
+	err := yaml.Unmarshal(data, &p)
+	if err != nil {
+		return p, err
+	}
+
+	return p, err
+}
+
+func LoadArtifactFromYaml(spec CompilationSpec) (Artifact, error) {
+
+	metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml"
+	dat, err := ioutil.ReadFile(spec.Rel(metaFile))
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading file "+metaFile)
+	}
+	art, err := NewPackageArtifactFromYaml(dat)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error writing file "+metaFile)
+	}
+	// It is relative, set it back to abs
+	art.SetPath(spec.Rel(art.GetPath()))
+	return art, nil
+}
+
+func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) {
+	a.CompressionType = t
+}
+
+func (a *PackageArtifact) GetChecksums() Checksums {
+	return a.Checksums
+}
+
+func (a *PackageArtifact) SetChecksums(c Checksums) {
+	a.Checksums = c
+}
+
+func (a *PackageArtifact) SetFiles(f []string) {
+	a.Files = f
+}
+
+func (a *PackageArtifact) GetFiles() []string {
+	return a.Files
+}
+
+func (a *PackageArtifact) Hash() error {
+	return a.Checksums.Generate(a)
+}
+
+func (a *PackageArtifact) Verify() error {
+	sum := Checksums{}
+	err := sum.Generate(a)
+	if err != nil {
+		return err
+	}
+	err = sum.Compare(a.Checksums)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (a *PackageArtifact) WriteYaml(dst string) error {
+	// First compute checksum of artifact. When we write the yaml we want to write up-to-date informations.
+	err := a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for artifact")
+	}
+
+	//p := a.CompileSpec.GetPackage().GetPath()
+
+	//a.CompileSpec.GetPackage().SetPath("")
+	//	for _, ass := range a.CompileSpec.GetSourceAssertion() {
+	//		ass.Package.SetPath("")
+	//	}
+	data, err := yaml.Marshal(a)
+	if err != nil {
+		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
+	}
+
+	mangle, err := NewPackageArtifactFromYaml(data)
+	if err != nil {
+		return errors.Wrap(err, "Generated invalid artifact")
+	}
+	//p := a.CompileSpec.GetPackage().GetPath()
+
+	mangle.GetCompileSpec().GetPackage().SetPath("")
+	for _, ass := range mangle.GetCompileSpec().GetSourceAssertion() {
+		ass.Package.SetPath("")
+	}
+
+	data, err = yaml.Marshal(mangle)
+	if err != nil {
+		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
+	}
+
+	err = ioutil.WriteFile(filepath.Join(dst, a.GetCompileSpec().GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm)
+	if err != nil {
+		return errors.Wrap(err, "While writing PackageArtifact YAML")
+	}
+	//a.CompileSpec.GetPackage().SetPath(p)
+
+	return nil
 }
 
 func (a *PackageArtifact) GetSourceAssertion() solver.PackagesAssertions {
@@ -48,7 +200,7 @@ func (a *PackageArtifact) GetSourceAssertion() solver.PackagesAssertions {
 }
 
 func (a *PackageArtifact) SetCompileSpec(as CompilationSpec) {
-	a.CompileSpec = as
+	a.CompileSpec = as.(*LuetCompilationSpec)
 }
 
 func (a *PackageArtifact) GetCompileSpec() CompilationSpec {
@@ -60,11 +212,19 @@ func (a *PackageArtifact) SetSourceAssertion(as solver.PackagesAssertions) {
 }
 
 func (a *PackageArtifact) GetDependencies() []Artifact {
-	return a.Dependencies
+	ret := []Artifact{}
+	for _, d := range a.Dependencies {
+		ret = append(ret, d)
+	}
+	return ret
 }
 
 func (a *PackageArtifact) SetDependencies(d []Artifact) {
-	a.Dependencies = d
+	ret := []*PackageArtifact{}
+	for _, dd := range d {
+		ret = append(ret, dd.(*PackageArtifact))
+	}
+	a.Dependencies = ret
 }
 
 func (a *PackageArtifact) GetPath() string {
@@ -75,43 +235,297 @@ func (a *PackageArtifact) SetPath(p string) {
 	a.Path = p
 }
 
+// Compress Archives and compress (TODO) to the artifact path
+func (a *PackageArtifact) Compress(src string, concurrency int) error {
+	switch a.CompressionType {
+	case GZip:
+		err := helpers.Tar(src, a.Path)
+		if err != nil {
+			return err
+		}
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return err
+		}
+		defer original.Close()
+
+		gzipfile := a.Path + ".gz"
+		bufferedReader := bufio.NewReader(original)
+
+		// Open a file for writing.
+		dst, err := os.Create(gzipfile)
+		if err != nil {
+			return err
+		}
+		// Create gzip writer.
+		w := gzip.NewWriter(dst)
+		w.SetConcurrency(concurrency, 10)
+		defer w.Close()
+		defer dst.Close()
+		_, err = io.Copy(w, bufferedReader)
+		if err != nil {
+			return err
+		}
+		w.Close()
+		os.RemoveAll(a.Path) // Remove original
+		//	a.CompressedPath = gzipfile
+		a.Path = gzipfile
+		return nil
+		//a.Path = gzipfile
+
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		return helpers.Tar(src, a.Path)
+
+	}
+	return errors.New("Compression type must be supplied")
+}
+
+func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	// If the destination path already exists I rename target file name with postfix.
+	var destPath string
+
+	// Read data. TODO: We need change archive callback to permit to return a Reader
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	// If file is not present on archive but is defined on mods
+	// I receive the callback. Prevent nil exception.
+	if header != nil {
+		switch header.Typeflag {
+		case tar.TypeReg:
+			destPath = filepath.Join(dst, path)
+		default:
+			// Nothing to do. I return original reader
+			return header, buffer.Bytes(), nil
+		}
+
+		// Check if exists
+		if helpers.Exists(destPath) {
+			for i := 1; i < 1000; i++ {
+				name := filepath.Join(filepath.Join(filepath.Dir(path),
+					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
+
+				if helpers.Exists(name) {
+					continue
+				}
+				Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
+					filepath.Join(dst, name)))
+				return &tar.Header{
+					Mode:       header.Mode,
+					Typeflag:   header.Typeflag,
+					PAXRecords: header.PAXRecords,
+					Name:       name,
+				}, buffer.Bytes(), nil
+			}
+		}
+	}
+
+	return header, buffer.Bytes(), nil
+}
+
+func (a *PackageArtifact) GetProtectFiles() []string {
+	ans := []string{}
+	annotationDir := ""
+
+	if !LuetCfg.ConfigProtectSkip {
+
+		// a.CompileSpec could be nil when artifact.Unpack is used for tree tarball
+		if a.CompileSpec != nil &&
+			a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
+			}
+		}
+		// TODO: check if skip this if we have a.CompileSpec nil
+
+		cp := NewConfigProtect(annotationDir)
+		cp.Map(a.Files)
+
+		// NOTE: for unpack we need files path without initial /
+		ans = cp.GetProtectFiles(false)
+	}
+
+	return ans
+}
+
+// Unpack Untar and decompress (TODO) to the given path
+func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+
+	// Create
+	protectedFiles := a.GetProtectFiles()
+
+	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
+
+	switch a.CompressionType {
+	case GZip:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := gzip.NewReader(bufferedReader)
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+
+		_, err = io.Copy(archive, r)
+		if err != nil {
+			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
+		}
+
+		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
+		if err != nil {
+			return err
+		}
+		return nil
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		return helpers.UntarProtect(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner,
+			protectedFiles, tarModifier)
+	}
+	return errors.New("Compression type must be supplied")
+}
+
+// FileList generates the list of file of a package from the local archive
+func (a *PackageArtifact) FileList() ([]string, error) {
+	var tr *tar.Reader
+	switch a.CompressionType {
+	case GZip:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		if err != nil {
+			return []string{}, err
+		}
+		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := gzip.NewReader(bufferedReader)
+		if err != nil {
+			return []string{}, err
+		}
+		defer r.Close()
+		tr = tar.NewReader(r)
+
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		tarFile, err := os.Open(a.GetPath())
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Could not open package archive")
+		}
+		defer tarFile.Close()
+		tr = tar.NewReader(tarFile)
+
+	}
+
+	var files []string
+	// untar each segment
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return []string{}, err
+		}
+		// determine proper file path info
+		finfo := hdr.FileInfo()
+		fileName := hdr.Name
+		if finfo.Mode().IsDir() {
+			continue
+		}
+		files = append(files, fileName)
+
+		// if a dir, create it, then go to next segment
+	}
+	return files, nil
+}
+
 type CopyJob struct {
 	Src, Dst string
 	Artifact string
 }
 
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
 	for job := range s {
-		Info("#"+strconv.Itoa(i), "copying", job.Src, "to", job.Dst)
-		if dir, err := helpers.IsDirectory(job.Src); err == nil && dir {
-			err = helpers.CopyDir(job.Src, job.Dst)
-			if err != nil {
-				Fatal("Error copying dir", job, err)
-			}
-			continue
-		}
+		//Info("#"+strconv.Itoa(i), "copying", job.Src, "to", job.Dst)
+		// if dir, err := helpers.IsDirectory(job.Src); err == nil && dir {
+		// 	err = helpers.CopyDir(job.Src, job.Dst)
+		// 	if err != nil {
+		// 		Warning("Error copying dir", job, err)
+		// 	}
+		// 	continue
+		// }
 
-		if !helpers.Exists(job.Dst) {
+		_, err := os.Lstat(job.Dst)
+		if err != nil {
+			Debug("Copying ", job.Src)
 			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
-				Fatal("Error copying", job, err)
+				Warning("Error copying", job, err)
 			}
+			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
 
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string) (Artifact, error) {
+func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t CompressionImplementation) (Artifact, error) {
 
-	archive, err := ioutil.TempDir(os.TempDir(), "archive")
+	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for archive")
 	}
 	defer os.RemoveAll(archive) // clean up
 
 	if strings.HasSuffix(src, ".tar") {
-		rootfs, err := ioutil.TempDir(os.TempDir(), "rootfs")
+		rootfs, err := LuetCfg.GetSystem().TempDir("rootfs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 		}
@@ -132,7 +546,8 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 	}
 
 	// Handle includes in spec. If specified they filter what gets in the package
-	if len(includes) > 0 {
+
+	if len(includes) > 0 && len(excludes) == 0 {
 		var includeRegexp []*regexp.Regexp
 		for _, i := range includes {
 			r, e := regexp.Compile(i)
@@ -153,23 +568,148 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 					}
 				}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
+
+	} else if len(includes) == 0 && len(excludes) != 0 {
+		var excludeRegexp []*regexp.Regexp
+		for _, i := range excludes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			excludeRegexp = append(excludeRegexp, r)
+		}
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		ADD:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range excludeRegexp {
+					if i.MatchString(a.Name) {
+						continue ADD
+					}
+				}
+				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
+		}
+
+	} else if len(includes) != 0 && len(excludes) != 0 {
+
+		var includeRegexp []*regexp.Regexp
+		for _, i := range includes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			includeRegexp = append(includeRegexp, r)
+		}
+		var excludeRegexp []*regexp.Regexp
+		for _, i := range excludes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			excludeRegexp = append(excludeRegexp, r)
+		}
+
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		EXCLUDES:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) {
+						for _, e := range excludeRegexp {
+							if e.MatchString(a.Name) {
+								continue EXCLUDES
+							}
+						}
+						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+						continue EXCLUDES
+					}
+				}
+			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
+		}
+
 	} else {
 		// Otherwise just grab all
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 			for _, a := range l.Diffs.Additions {
+				Debug("File ", a.Name, " added")
 				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	}
 
 	close(toCopy)
 	wg.Wait()
 
-	err = helpers.Tar(archive, dst)
+	a := NewPackageArtifact(dst)
+	a.SetCompressionType(t)
+	err = a.Compress(archive, concurrency)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}
-	return NewPackageArtifact(dst), nil
+	return a, nil
+}
+
+func ComputeArtifactLayerSummary(diffs []ArtifactLayer) ArtifactLayersSummary {
+
+	ans := ArtifactLayersSummary{
+		Layers: make([]ArtifactLayerSummary, 0),
+	}
+
+	for _, layer := range diffs {
+		sum := ArtifactLayerSummary{
+			FromImage:   layer.FromImage,
+			ToImage:     layer.ToImage,
+			AddFiles:    0,
+			AddSizes:    0,
+			DelFiles:    0,
+			DelSizes:    0,
+			ChangeFiles: 0,
+			ChangeSizes: 0,
+		}
+		for _, a := range layer.Diffs.Additions {
+			sum.AddFiles++
+			sum.AddSizes += int64(a.Size)
+		}
+		for _, d := range layer.Diffs.Deletions {
+			sum.DelFiles++
+			sum.DelSizes += int64(d.Size)
+		}
+		for _, c := range layer.Diffs.Changes {
+			sum.ChangeFiles++
+			sum.ChangeSizes += int64(c.Size)
+		}
+		ans.Layers = append(ans.Layers, sum)
+	}
+
+	return ans
 }

pkg/compiler/artifact_test.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/solver"
 
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
@@ -32,17 +33,16 @@ import (
 
 var _ = Describe("Artifact", func() {
 	Context("Simple package build definition", func() {
-		It("Generates a delta", func() {
+		It("Generates a verified delta", func() {
 
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
 			err := generalRecipe.Load("../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -76,7 +76,9 @@ var _ = Describe("Artifact", func() {
 FROM alpine
 COPY . /luetbuild
 WORKDIR /luetbuild
-`))
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin`))
 			b := NewSimpleDockerBackend()
 			opts := CompilerBackendOptions{
 				ImageName:      "luet/base",
@@ -94,6 +96,9 @@ WORKDIR /luetbuild
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
 			opts = CompilerBackendOptions{
@@ -107,21 +112,25 @@ RUN echo bar > /test2`))
 			diffs, err := b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))
 			Expect(err).ToNot(HaveOccurred())
 
+			artifacts := []ArtifactNode{}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
 			Expect(diffs).To(Equal(
 				[]ArtifactLayer{{
 					FromImage: filepath.Join(tmpdir2, "output1.tar"),
 					ToImage:   filepath.Join(tmpdir, "output2.tar"),
 					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+						Additions: artifacts,
 					},
 				}}))
 			err = b.ExtractRootfs(CompilerBackendOptions{SourcePath: filepath.Join(tmpdir, "output2.tar"), Destination: rootfs}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{})
+			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, None)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
 			err = helpers.Untar(artifact.GetPath(), unpacked, false)
@@ -134,6 +143,15 @@ RUN echo bar > /test2`))
 			content2, err := helpers.Read(filepath.Join(unpacked, "test2"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content2).To(Equal("bar\n"))
+
+			err = artifact.Hash()
+			Expect(err).ToNot(HaveOccurred())
+			err = artifact.Verify()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
+
+			err = artifact.Verify()
+			Expect(err).To(HaveOccurred())
 		})
 
 	})

pkg/compiler/backend/backend_suite_test.go

@@ -18,11 +18,15 @@ package backend_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Backend Suite")
 }

pkg/compiler/backend/diff.go

@@ -0,0 +1,199 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/pkg/errors"
+)
+
+// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
+// example of json return: [
+//   {
+//     "Image1": "luet/base",
+//     "Image2": "alpine",
+//     "DiffType": "File",
+//     "Diff": {
+//       "Adds": null,
+//       "Dels": [
+//         {
+//           "Name": "/luetbuild",
+//           "Size": 5830706
+//         },
+//         {
+//           "Name": "/luetbuild/Dockerfile",
+//           "Size": 50
+//         },
+//         {
+//           "Name": "/luetbuild/output1",
+//           "Size": 5830656
+//         }
+//       ],
+//       "Mods": null
+//     }
+//   }
+// ]
+func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]compiler.ArtifactLayer, error) {
+
+	res := compiler.ArtifactLayer{FromImage: srcImage, ToImage: dstImage}
+
+	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(srcRootFS) // clean up
+
+	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(dstRootFS) // clean up
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(srcImage, "/") {
+		srcImageTar, err := ioutil.TempFile(tmpdiffs, "srctar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(srcImageTar.Name()) // clean up
+		srcImageExport := compiler.CompilerBackendOptions{
+			ImageName:   srcImage,
+			Destination: srcImageTar.Name(),
+		}
+		err = b.ExportImage(srcImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting src image "+srcImage)
+		}
+		srcImage = srcImageTar.Name()
+	}
+
+	srcImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  srcImage,
+		Destination: srcRootFS,
+	}
+	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+srcImage)
+	}
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(dstImage, "/") {
+		dstImageTar, err := ioutil.TempFile(tmpdiffs, "dsttar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(dstImageTar.Name()) // clean up
+		dstImageExport := compiler.CompilerBackendOptions{
+			ImageName:   dstImage,
+			Destination: dstImageTar.Name(),
+		}
+		err = b.ExportImage(dstImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting dst image "+dstImage)
+		}
+		dstImage = dstImageTar.Name()
+	}
+
+	dstImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  dstImage,
+		Destination: dstRootFS,
+	}
+	err = b.ExtractRootfs(dstImageExtract, false)
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+dstImage)
+	}
+
+	// Get Additions/Changes. dst -> src
+	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, dstRootFS, "", -1)
+		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
+		if err == nil {
+			var sizeA, sizeB int64
+			sizeA = fileInfo.Size()
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+
+			if sizeA != sizeB {
+				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
+				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
+					Name: filepath.Join("/", realpath),
+					Size: int(sizeB),
+				})
+			} else {
+				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
+			}
+		} else {
+			var sizeB int64
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+				Size: int(sizeB),
+			})
+
+			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+	}
+
+	// Get deletions. src -> dst
+	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, srcRootFS, "", -1)
+		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
+			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
+			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+			})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+	}
+
+	return []compiler.ArtifactLayer{res}, nil
+}

pkg/compiler/backend/diff_test.go

@@ -0,0 +1,73 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend_test
+
+import (
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Docker image diffs", func() {
+	var b compiler.CompilerBackend
+
+	BeforeEach(func() {
+		b = NewSimpleDockerBackend()
+	})
+
+	Context("Generate diffs from docker images", func() {
+		It("Detect no changes", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "alpine:latest",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, "alpine:latest", "alpine:latest")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+
+		It("Detects additions and changed files", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/micro",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			err = b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/extra",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, "quay.io/mocaccino/micro", "quay.io/mocaccino/extra")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+
+			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+	})
+})

pkg/compiler/backend/simpledocker.go

@@ -17,6 +17,7 @@ package backend
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -26,6 +27,7 @@ import (
 	capi "github.com/mudler/docker-companion/api"
 
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -54,7 +56,12 @@ func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
 	}
 	Info(":whale: Building image " + name + " done")
 
-	//Info(string(out))
+	if config.LuetCfg.GetGeneral().ShowBuildOutput {
+		Info(string(out))
+	} else {
+		Debug(string(out))
+	}
+
 	return nil
 }
 
@@ -82,6 +89,19 @@ func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
+func (*SimpleDocker) ImageExists(imagename string) bool {
+	buildarg := []string{"inspect", "--type=image", imagename}
+	Debug(":whale: Checking existance of docker image: " + imagename)
+	cmd := exec.Command("docker", buildarg...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		Warning("Image not present")
+		Debug(string(out))
+		return false
+	}
+	return true
+}
+
 func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
@@ -94,6 +114,18 @@ func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
+func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
+	name := opts.ImageName
+	pusharg := []string{"push", name}
+	out, err := exec.Command("docker", pusharg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed pushing image: "+string(out))
+	}
+	Info(":whale: Pushed image:", name)
+	//Info(string(out))
+	return nil
+}
+
 func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
@@ -162,11 +194,13 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 
 	layers_sha := []string{}
 
-	if len(manifestData) != 1 {
-		return errors.New("Manifest should have one entry")
-	}
-	for _, l := range manifestData[0].Layers {
-		layers_sha = append(layers_sha, strings.Replace(l, "/layer.tar", "", -1))
+	for _, data := range manifestData {
+
+		for _, l := range data.Layers {
+			if strings.Contains(l, "layer.tar") {
+				layers_sha = append(layers_sha, strings.Replace(l, "/layer.tar", "", -1))
+			}
+		}
 	}
 
 	export, err := capi.CreateExport(rootfs)
@@ -187,51 +221,20 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 	return nil
 }
 
-// 	container-diff diff daemon://luet/base alpine --type=file -j
-// [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	tmpdiffs, err := ioutil.TempDir(os.TempDir(), "tmpdiffs")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
+// Changes retrieves changes between image layers
+func (d *SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+	diffs, err := GenerateChanges(d, fromImage, toImage)
 
-	diffargs := []string{"diff", fromImage, toImage, "--type=file", "-j", "-n", "-c", tmpdiffs}
-	out, err := exec.Command("container-diff", diffargs...).CombinedOutput()
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
+	if config.LuetCfg.GetGeneral().Debug {
+		summary := compiler.ComputeArtifactLayerSummary(diffs)
+		for _, l := range summary.Layers {
+			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
+				l.FromImage, l.ToImage,
+				l.AddFiles, l.AddSizes,
+				l.DelFiles, l.DelSizes,
+				l.ChangeFiles, l.ChangeSizes))
+		}
 	}
 
-	var diffs []compiler.ArtifactLayer
-
-	err = json.Unmarshal(out, &diffs)
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
-	}
-	return diffs, nil
+	return diffs, err
 }

pkg/compiler/backend/simpledocker_test.go

@@ -18,6 +18,7 @@ package backend_test
 import (
 	. "github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/solver"
 
 	"io/ioutil"
 	"os"
@@ -37,11 +38,10 @@ var _ = Describe("Docker backend", func() {
 
 			err := generalRecipe.Load("../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -67,7 +67,9 @@ var _ = Describe("Docker backend", func() {
 FROM alpine
 COPY . /luetbuild
 WORKDIR /luetbuild
-`))
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin`))
 			b := NewSimpleDockerBackend()
 			opts := CompilerBackendOptions{
 				ImageName:      "luet/base",
@@ -85,6 +87,9 @@ WORKDIR /luetbuild
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
 			opts = CompilerBackendOptions{
@@ -96,15 +101,19 @@ RUN echo bar > /test2`))
 			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
+			artifacts := []ArtifactNode{}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
 			Expect(b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))).To(Equal(
 				[]ArtifactLayer{{
 					FromImage: filepath.Join(tmpdir2, "output1.tar"),
 					ToImage:   filepath.Join(tmpdir, "output2.tar"),
 					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+						Additions: artifacts,
 					},
 				}}))
 

pkg/compiler/backend/simpleimg.go

@@ -16,6 +16,7 @@
 package backend
 
 import (
+	"os"
 	"os/exec"
 
 	"github.com/mudler/luet/pkg/compiler"
@@ -36,15 +37,18 @@ func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error {
 	path := opts.SourcePath
 	dockerfileName := opts.DockerFileName
 
-	buildarg := []string{"build", "-t", name, path, "-f ", dockerfileName}
+	buildarg := []string{"build", "-f", dockerfileName, "-t", name, "."}
 	Spinner(22)
-	Debug("Building image "+name+" - running img with: ", buildarg)
-	out, err := exec.Command("img", buildarg...).CombinedOutput()
+	defer SpinnerStop()
+	Debug(":tea: Building image " + name)
+	cmd := exec.Command("img", buildarg...)
+	cmd.Dir = path
+	out, err := cmd.CombinedOutput()
+
 	if err != nil {
 		return errors.Wrap(err, "Failed building image: "+string(out))
 	}
-	SpinnerStop()
-	Info(out)
+	Info(":tea: Building image " + name + " done")
 	return nil
 }
 
@@ -52,12 +56,13 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	buildarg := []string{"rm", name}
 	Spinner(22)
+	defer SpinnerStop()
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed building image: "+string(out))
 	}
-	SpinnerStop()
-	Info(out)
+
+	Info(":tea: Image " + name + " removed")
 	return nil
 }
 
@@ -66,27 +71,39 @@ func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	buildarg := []string{"pull", name}
 
-	Debug("Downloading image "+name+" - running img with: ", buildarg)
+	Debug(":tea: Downloading image " + name)
 	cmd := exec.Command("img", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed building image: "+string(out))
 	}
+
+	Info(":tea: Image " + name + " downloaded")
+
 	return nil
 }
 func (*SimpleImg) CopyImage(src, dst string) error {
 	Spinner(22)
 	defer SpinnerStop()
 
-	Debug("Tagging image - running img with: ", src, dst)
+	Debug(":tea: Tagging image", src, dst)
 	cmd := exec.Command("img", "tag", src, dst)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed tagging image: "+string(out))
 	}
+	Info(":tea: Image " + dst + " tagged")
+
 	return nil
 }
 
+func (*SimpleImg) ImageExists(imagename string) bool {
+	// NOOP: not implemented
+	// TODO: Since img doesn't have an inspect command,
+	// we need to parse the ls output manually
+	return false
+}
+
 func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
@@ -103,22 +120,47 @@ func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) e
 func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	path := opts.Destination
-	buildarg := []string{"save", name, "-o", path}
-	Debug("Saving image "+name+" - running img with: ", buildarg)
+	buildarg := []string{"save", "-o", path, name}
+	Debug(":tea: Saving image " + name)
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed building image: "+string(out))
 	}
+	Info(":tea: Image " + name + " saved")
 	return nil
 }
 
 // TODO: Dup in docker, refactor common code in helpers for shared parts
 func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
-	return NewSimpleDockerBackend().ExtractRootfs(opts, keepPerms)
+	name := opts.ImageName
+	path := opts.Destination
+
+	os.RemoveAll(path)
+	buildarg := []string{"unpack", "-o", path, name}
+	Debug(":tea: Extracting image " + name)
+	out, err := exec.Command("img", buildarg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed extracting image: "+string(out))
+	}
+	Info(":tea: Image " + name + " extracted")
+	return nil
+	//return NewSimpleDockerBackend().ExtractRootfs(opts, keepPerms)
 }
 
 // TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
 // Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	return NewSimpleDockerBackend().Changes(fromImage, toImage)
+func (i *SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+	return GenerateChanges(i, fromImage, toImage)
+}
+
+func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {
+	name := opts.ImageName
+	pusharg := []string{"push", name}
+	out, err := exec.Command("img", pusharg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed pushing image: "+string(out))
+	}
+	Info(":tea: Pushed image:", name)
+	//Info(string(out))
+	return nil
 }

pkg/compiler/checksum.go

@@ -0,0 +1,78 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+
+	//"strconv"
+
+	"crypto/sha256"
+	"fmt"
+	"hash"
+	"io"
+	"os"
+
+	//	. "github.com/mudler/luet/pkg/logger"
+	"github.com/pkg/errors"
+)
+
+type HashImplementation string
+
+const (
+	SHA256 HashImplementation = "sha256"
+)
+
+type Checksums map[string]string
+
+type HashOptions struct {
+	Hasher hash.Hash
+	Type   HashImplementation
+}
+
+// Generate generates all Checksums supported for the artifact
+func (c *Checksums) Generate(a Artifact) error {
+	return c.generateSHA256(a)
+}
+
+func (c Checksums) Compare(d Checksums) error {
+	for t, sum := range d {
+		if v, ok := c[t]; ok && v != sum {
+			return errors.New("Checksum mismsatch")
+		}
+	}
+	return nil
+}
+
+func (c *Checksums) generateSHA256(a Artifact) error {
+	return c.generateSum(a, HashOptions{Hasher: sha256.New(), Type: SHA256})
+}
+
+func (c *Checksums) generateSum(a Artifact, opts HashOptions) error {
+
+	f, err := os.Open(a.GetPath())
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := io.Copy(opts.Hasher, f); err != nil {
+		return err
+	}
+
+	sum := fmt.Sprintf("%x", opts.Hasher.Sum(nil))
+
+	(*c)[string(opts.Type)] = sum
+	return nil
+}

pkg/compiler/checksum_test.go

@@ -0,0 +1,61 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler_test
+
+import (
+	"io/ioutil"
+	"os"
+
+	. "github.com/mudler/luet/pkg/compiler"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Checksum", func() {
+	Context("Generation", func() {
+		It("Compares successfully", func() {
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			buildsum := Checksums{}
+			definitionsum := Checksums{}
+			definitionsum2 := Checksums{}
+
+			Expect(len(buildsum)).To(Equal(0))
+			Expect(len(definitionsum)).To(Equal(0))
+			Expect(len(definitionsum2)).To(Equal(0))
+
+			err = buildsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/build.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = definitionsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = definitionsum2.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(buildsum)).To(Equal(1))
+			Expect(len(definitionsum)).To(Equal(1))
+			Expect(len(definitionsum2)).To(Equal(1))
+
+			Expect(definitionsum.Compare(buildsum)).To(HaveOccurred())
+			Expect(definitionsum.Compare(definitionsum2)).ToNot(HaveOccurred())
+		})
+	})
+
+})

pkg/compiler/compiler.go

@@ -20,11 +20,14 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"sync"
 
-	. "github.com/mudler/luet/pkg/logger"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
 
 	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
@@ -32,24 +35,47 @@ import (
 )
 
 const BuildFile = "build.yaml"
+const DefinitionFile = "definition.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
-	Backend  CompilerBackend
-	Database pkg.PackageDatabase
+	Backend                   CompilerBackend
+	Database                  pkg.PackageDatabase
+	ImageRepository           string
+	PullFirst, KeepImg, Clean bool
+	Concurrency               int
+	CompressionType           CompressionImplementation
+	Options                   CompilerOptions
+	SolverOptions             solver.Options
 }
 
-func NewLuetCompiler(backend CompilerBackend, t pkg.Tree, db pkg.PackageDatabase) Compiler {
+func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions, solvopts solver.Options) Compiler {
 	// The CompilerRecipe will gives us a tree with only build deps listed.
 	return &LuetCompiler{
 		Backend: backend,
 		CompilerRecipe: &tree.CompilerRecipe{
-			tree.Recipe{PackageTree: t},
+			tree.Recipe{Database: db},
 		},
-		Database: db,
+		Database:        db,
+		ImageRepository: opt.ImageRepository,
+		PullFirst:       opt.PullFirst,
+		CompressionType: opt.CompressionType,
+		KeepImg:         opt.KeepImg,
+		Concurrency:     opt.Concurrency,
+		Clean:           opt.Clean,
+		Options:         *opt,
+		SolverOptions:   solvopts,
 	}
 }
 
+func (cs *LuetCompiler) SetConcurrency(i int) {
+	cs.Concurrency = i
+}
+
+func (cs *LuetCompiler) SetCompressionType(t CompressionImplementation) {
+	cs.CompressionType = t
+}
+
 func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan CompilationSpec, a *[]Artifact, m *sync.Mutex, concurrency int, keepPermissions bool, errors chan error) {
 	defer wg.Done()
 
@@ -64,8 +90,9 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan Co
 		m.Unlock()
 	}
 }
-func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
-	artifacts, err := cs.CompileParallel(concurrency, keepPermissions, ps)
+
+func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
+	artifacts, err := cs.CompileParallel(keepPermissions, ps)
 	if len(err) != 0 {
 		return artifacts, err
 	}
@@ -73,11 +100,8 @@ func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions
 	Info(":ant: Resolving reverse dependencies")
 	toCompile := NewLuetCompilationspecs()
 	for _, a := range artifacts {
-		w, asserterr := cs.Tree().World()
-		if asserterr != nil {
-			return nil, append(err, asserterr)
-		}
-		revdeps := a.GetCompileSpec().GetPackage().Revdeps(&w)
+
+		revdeps := a.GetCompileSpec().GetPackage().Revdeps(cs.Database)
 		for _, r := range revdeps {
 			spec, asserterr := cs.FromPackage(r)
 			if err != nil {
@@ -116,11 +140,11 @@ func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions
 		Info(" :arrow_right_hook:", u.GetPackage().GetName(), ":leaves:", u.GetPackage().GetVersion(), "(", u.GetPackage().GetCategory(), ")")
 	}
 
-	artifacts2, err := cs.CompileParallel(concurrency, keepPermissions, uniques)
+	artifacts2, err := cs.CompileParallel(keepPermissions, uniques)
 	return append(artifacts, artifacts2...), err
 }
 
-func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
+func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
 	Spinner(22)
 	defer SpinnerStop()
 	all := make(chan CompilationSpec)
@@ -128,9 +152,9 @@ func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, p
 	mutex := &sync.Mutex{}
 	errors := make(chan error, ps.Len())
 	var wg = new(sync.WaitGroup)
-	for i := 0; i < concurrency; i++ {
+	for i := 0; i < cs.Concurrency; i++ {
 		wg.Add(1)
-		go cs.compilerWorker(i, wg, all, &artifacts, mutex, concurrency, keepPermissions, errors)
+		go cs.compilerWorker(i, wg, all, &artifacts, mutex, cs.Concurrency, keepPermissions, errors)
 	}
 
 	for _, p := range ps.All() {
@@ -155,11 +179,96 @@ func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, p
 	return artifacts, allErrors
 }
 
-func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
+func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, include bool) error {
+	var includeRegexp []*regexp.Regexp
+	for _, i := range includes {
+		r, e := regexp.Compile(i)
+		if e != nil {
+			return errors.Wrap(e, "Could not compile regex in the include of the package")
+		}
+		includeRegexp = append(includeRegexp, r)
+	}
+
+	toRemove := []string{}
+
+	// the function that handles each file or dir
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+
+		// if info.Name() != DefinitionFile {
+		// 	return nil // Skip with no errors
+		// }
+		if currentpath == rootfs {
+			return nil
+		}
+
+		abspath := strings.ReplaceAll(currentpath, rootfs, "")
+
+		match := false
+
+		for _, i := range includeRegexp {
+			if i.MatchString(abspath) {
+				match = true
+			}
+		}
+
+		if include && !match || !include && match {
+			toRemove = append(toRemove, currentpath)
+		}
+
+		return nil
+	}
+
+	err := filepath.Walk(rootfs, ff)
+	if err != nil {
+		return err
+	}
+
+	for _, s := range toRemove {
+		e := os.RemoveAll(s)
+		if e != nil {
+			Warning("Failed removing", s, e.Error())
+			return e
+		}
+	}
+	return nil
+}
+
+func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec, generateArtifact bool) (Artifact, error) {
+
+	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
+
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	// TODO: As the salt contains the packageImage ( in registry/organization/imagename:tag format)
+	// the images hashes are broken with registry mirrors.
+	// We should use the image tag, or pass by the package assertion hash which is unique
+	// and identifies the deptree of the package.
+
+	fp := p.GetPackage().HashFingerprint(packageImage)
+
+	if buildertaggedImage == "" {
+		buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder"
+		Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image)
+	}
+
+	// TODO:  Cleanup, not actually hit
+	if packageImage == "" {
+		packageImage = cs.ImageRepository + "-" + fp
+	}
+
+	if !cs.Clean {
+		exists := cs.Backend.ImageExists(buildertaggedImage) && cs.Backend.ImageExists(packageImage)
+		if art, err := LoadArtifactFromYaml(p); err == nil && (cs.Options.SkipIfMetadataExists || exists) {
+			Debug("Artifact reloaded. Skipping build")
+			return art, err
+		}
+	}
+
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
 	// and we build all the images first.
-	keepImg := true
-	keepPackageImg := true
+
 	err := os.MkdirAll(p.Rel("build"), os.ModePerm)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for building")
@@ -176,15 +285,17 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		return nil, errors.Wrap(err, "Could not copy package sources")
 
 	}
-	if buildertaggedImage == "" {
-		keepImg = false
-		buildertaggedImage = "luet/" + p.GetPackage().GetFingerPrint() + "-builder"
-	}
-	if packageImage == "" {
-		keepPackageImg = false
-		packageImage = "luet/" + p.GetPackage().GetFingerPrint()
+
+	// Copy file into the build context, the compilespec might have requested to do so.
+	if len(p.GetRetrieve()) > 0 {
+		err := p.CopyRetrieves(buildDir)
+		if err != nil {
+			Warning("Failed copying retrieves", err.Error())
+		}
 	}
 
+	Info(pkgTag, ":whale: Generating 'builder' image definition from", image)
+
 	// First we create the builder image
 	p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile"))
 	builderOpts := CompilerBackendOptions{
@@ -194,16 +305,32 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 	}
 
-	err = cs.Backend.BuildImage(builderOpts)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
+	buildBuilderImage := true
+	if cs.Options.PullFirst {
+		if err := cs.Backend.DownloadImage(builderOpts); err == nil {
+			buildBuilderImage = false
+		}
 	}
 
-	err = cs.Backend.ExportImage(builderOpts)
-	if err != nil {
+	if buildBuilderImage {
+		if err = cs.Backend.BuildImage(builderOpts); err != nil {
+			return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
+
+	if err = cs.Backend.ExportImage(builderOpts); err != nil {
 		return nil, errors.Wrap(err, "Could not export image")
 	}
 
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(builderOpts.Destination)
+	}
+
+	if cs.Options.Push && buildBuilderImage {
+		if err = cs.Backend.Push(builderOpts); err != nil {
+			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
 	// Then we write the step image, which uses the builder one
 	p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
 	runnerOpts := CompilerBackendOptions{
@@ -213,161 +340,199 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 	}
 
-	if !keepPackageImg {
-		err = cs.Backend.ImageDefinitionToTar(runnerOpts)
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not export image to tar")
-		}
-	} else {
-		if err := cs.Backend.BuildImage(runnerOpts); err != nil {
-			return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
-		}
-		if err := cs.Backend.ExportImage(runnerOpts); err != nil {
-			return nil, errors.Wrap(err, "Failed exporting image")
+	buildPackageImage := true
+	if cs.Options.PullFirst {
+		//Best effort pull
+		if err := cs.Backend.DownloadImage(runnerOpts); err == nil {
+			buildPackageImage = false
 		}
 	}
 
-	var diffs []ArtifactLayer
-	var artifact Artifact
-
-	if !p.ImageUnpack() {
-		// we have to get diffs only if spec is not unpacked
-		diffs, err = cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate changes from layers")
+	if buildPackageImage {
+		if err := cs.Backend.BuildImage(runnerOpts); err != nil {
+			return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
 		}
 	}
 
+	if err := cs.Backend.ExportImage(runnerOpts); err != nil {
+		return nil, errors.Wrap(err, "Failed exporting image")
+	}
+
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(runnerOpts.Destination)
+	}
+
+	if cs.Options.Push && buildPackageImage {
+		err = cs.Backend.Push(runnerOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
+
+	var artifact Artifact
+	unpack := p.ImageUnpack()
+
+	// If package_dir was specified in the spec, we want to treat the content of the directory
+	// as the root of our archive.  ImageUnpack is implied to be true. override it
+	if p.GetPackageDir() != "" {
+		unpack = true
+	}
+
+	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not create tempdir")
+	}
+	defer os.RemoveAll(rootfs) // clean up
+
+	// TODO: Compression and such
+	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
+		ImageName:  packageImage,
+		SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not extract rootfs")
+	}
+
 	if !keepImg {
 		// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
 		// TODO: Handle caching and optionally do not remove things
 		err = cs.Backend.RemoveImage(builderOpts)
 		if err != nil {
-			// TODO: Have a --fatal flag which enables Warnings to exit.
+			Warning("Could not remove image ", builderOpts.ImageName)
+			//	return nil, errors.Wrap(err, "Could not remove image")
+		}
+		err = cs.Backend.RemoveImage(runnerOpts)
+		if err != nil {
 			Warning("Could not remove image ", builderOpts.ImageName)
 			//	return nil, errors.Wrap(err, "Could not remove image")
 		}
 	}
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not create tempdir")
-	}
-	defer os.RemoveAll(rootfs) // clean up
 
-	// TODO: Compression and such
-	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not extract rootfs")
+	if !generateArtifact {
+		return &PackageArtifact{}, nil
 	}
 
-	if p.ImageUnpack() {
-		err = helpers.Tar(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"))
+	if unpack {
+		if p.GetPackageDir() != "" {
+			Info(":tophat: Packing from output dir", p.GetPackageDir())
+			rootfs = filepath.Join(rootfs, p.GetPackageDir())
+		}
+
+		if len(p.GetIncludes()) > 0 {
+			// strip from includes
+			cs.stripFromRootfs(p.GetIncludes(), rootfs, true)
+		}
+		if len(p.GetExcludes()) > 0 {
+			// strip from includes
+			cs.stripFromRootfs(p.GetExcludes(), rootfs, false)
+		}
+		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
+		artifact.SetCompressionType(cs.CompressionType)
+
+		err = artifact.Compress(rootfs, concurrency)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating package archive")
 		}
 
-		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 		artifact.SetCompileSpec(p)
 	} else {
-		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes())
+		Info(pkgTag, ":hammer: Generating delta")
+		diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not generate changes from layers")
+		}
+		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.CompressionType)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not generate deltas")
 		}
+
 		artifact.SetCompileSpec(p)
 	}
 
+	filelist, err := artifact.FileList()
+	if err != nil {
+		return artifact, errors.Wrap(err, "Failed getting package list")
+	}
+
+	artifact.SetFiles(filelist)
+	artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
+
+	err = artifact.WriteYaml(p.GetOutputPath())
+	if err != nil {
+		return artifact, errors.Wrap(err, "Failed while writing metadata file")
+	}
+	Info(pkgTag, "   :white_check_mark: Done")
+
 	return artifact, nil
 }
 
-func (cs *LuetCompiler) Prepare(concurrency int) error {
+func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) {
+	compilerSpecs := NewLuetCompilationspecs()
 
-	err := cs.Tree().ResolveDeps(concurrency) // FIXME: When done in parallel, this could be done on top before starting
-	if err != nil {
-		return errors.Wrap(err, "While resoolving tree world deps")
-	}
-	return nil
-}
-func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions bool) (Artifact, error) {
-	pkgTag := ":package:  " + p.GetPackage().GetName()
+	w := db.World()
 
-	Info(pkgTag, "   🍩 Build starts 🔨 🔨 🔨 ")
-
-	builderOpts := CompilerBackendOptions{
-		ImageName:   p.GetImage(),
-		Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
-	}
-	err := cs.Backend.DownloadImage(builderOpts)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not download image")
-	}
-
-	if tag != "" {
-		err = cs.Backend.CopyImage(p.GetImage(), tag)
+	for _, p := range w {
+		spec, err := cs.FromPackage(p)
 		if err != nil {
-			return nil, errors.Wrap(err, "Could not download image")
+			return nil, err
+		}
+		if dst != "" {
+			spec.SetOutputPath(dst)
+		}
+		compilerSpecs.Add(spec)
+	}
+
+	switch minimum {
+	case true:
+		return cs.ComputeMinimumCompilableSet(compilerSpecs.Unique().All()...)
+	default:
+		return compilerSpecs.Unique().All(), nil
+	}
+}
+
+// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
+func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error) {
+	// Generate a set with all the deps of the provided specs
+	// we will use that set to remove the deps from the list of provided compilation specs
+	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
+	result := []CompilationSpec{}
+	for _, spec := range p {
+		ass, err := cs.ComputeDepTree(spec)
+		if err != nil {
+			return result, errors.Wrap(err, "computin specs deptree")
+		}
+
+		allDependencies = append(allDependencies, ass.Drop(spec.GetPackage())...)
+	}
+
+	for _, spec := range p {
+		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
+			result = append(result, spec)
 		}
 	}
-	err = cs.Backend.ExportImage(builderOpts)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not export image")
-	}
-
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not create tempdir")
-	}
-	defer os.RemoveAll(rootfs) // clean up
-
-	// TODO: Compression and such
-	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: builderOpts.Destination, Destination: rootfs}, keepPermissions)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not extract rootfs")
-	}
-
-	err = helpers.Tar(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"))
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating package archive")
-	}
-
-	Info(pkgTag, "   :white_check_mark: Done")
-	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
-	artifact.SetCompileSpec(p)
-	return artifact, nil
+	return result, nil
 }
 
 func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
 
-	// Get build deps tree (ordered)
-	world, err := cs.Tree().World()
+	s := solver.NewResolver(cs.SolverOptions, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
+
+	solution, err := s.Install(pkg.Packages{p.GetPackage()})
 	if err != nil {
-		return nil, errors.Wrap(err, "While computing tree world")
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
 	}
-	s := solver.NewSolver([]pkg.Package{}, world, cs.Database)
-	pack, err := cs.Tree().FindPackage(p.GetPackage())
+
+	dependencies, err := solution.Order(cs.Database, p.GetPackage().GetFingerPrint())
 	if err != nil {
-		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+		return nil, errors.Wrap(err, "While order a solution for "+p.GetPackage().HumanReadableString())
 	}
-	solution, err := s.Install([]pkg.Package{pack})
-	if err != nil {
-		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
-	}
-	dependencies := solution.Order(p.GetPackage().GetFingerPrint())
+
 	assertions := solver.PackagesAssertions{}
-
 	for _, assertion := range dependencies { //highly dependent on the order
-		if assertion.Value && assertion.Package.Flagged() {
-			depPack, err := cs.Tree().FindPackage(assertion.Package)
-			if err != nil {
-				return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
-			}
-			nthsolution, err := s.Install([]pkg.Package{depPack})
-			if err != nil {
-				return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
-			}
-
+		if assertion.Value {
+			nthsolution := dependencies.Cut(assertion.Package)
 			assertion.Hash = solver.PackageHash{
-				BuildHash:   nthsolution.Order(depPack.GetFingerPrint()).Drop(depPack).AssertionHash(),
-				PackageHash: nthsolution.Order(depPack.GetFingerPrint()).AssertionHash(),
+				BuildHash:   nthsolution.HashFrom(assertion.Package),
+				PackageHash: nthsolution.AssertionHash(),
 			}
 			assertions = append(assertions, assertion)
 		}
@@ -377,31 +542,31 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
 }
 
 // Compile is non-parallel
-func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
+func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifact, error) {
 	asserts, err := cs.ComputeDepTree(p)
 	if err != nil {
 		panic(err)
 	}
 	p.SetSourceAssertion(asserts)
-	return cs.compile(concurrency, keepPermissions, p)
+	return cs.compile(cs.Concurrency, keepPermissions, p)
 }
 
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
-	Info(":package: Compiling", p.GetPackage().GetName(), "version", p.GetPackage().GetVersion(), ".... :coffee:")
+	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
 	if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" {
 		Error("Package with no deps and no seed image supplied, bailing out")
-		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() + "with no deps and no seed image supplied, bailing out")
+		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() +
+			" with no deps and no seed image supplied, bailing out")
 	}
 
+	targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
+	targetPackageHash := cs.ImageRepository + ":" + targetAssertion.Hash.PackageHash
+
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		if p.ImageUnpack() { // If it is just an entire image, create a package from it
-			return cs.packageFromImage(p, "", keepPermissions)
-		}
-
-		return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, p)
+		return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@@ -411,98 +576,91 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 
 	dependencies := p.GetSourceAssertion().Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
 	departifacts := []Artifact{}                                // TODO: Return this somehow
-	deperrs := []error{}
 	var lastHash string
 	depsN := 0
 	currentN := 0
 
-	Info(":deciduous_tree: Build dependencies for " + p.GetPackage().GetName())
-	for _, assertion := range dependencies { //highly dependent on the order
-		depsN++
-		Info(" :arrow_right_hook:", assertion.Package.GetName(), ":leaves:", assertion.Package.GetVersion(), "(", assertion.Package.GetCategory(), ")")
-
-	}
-
-	for _, assertion := range dependencies { //highly dependent on the order
-		currentN++
-		pkgTag := fmt.Sprintf(":package:  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().GetName(), assertion.Package.GetName())
-		Info(pkgTag, "   :zap:  Building dependency")
-		compileSpec, err := cs.FromPackage(assertion.Package)
-		if err != nil {
-			return nil, errors.New("Error while generating compilespec for " + assertion.Package.GetName())
+	packageDeps := !cs.Options.PackageTargetOnly
+	if !cs.Options.NoDeps {
+		Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
+		for _, assertion := range dependencies { //highly dependent on the order
+			depsN++
+			Info(" :arrow_right_hook:", assertion.Package.HumanReadableString(), ":leaves:")
 		}
-		compileSpec.SetOutputPath(p.GetOutputPath())
 
-		buildImageHash := "luet/cache:" + assertion.Hash.BuildHash
-		currentPackageImageHash := "luet/cache:" + assertion.Hash.PackageHash
-		Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image name", buildImageHash)
-		Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
+		for _, assertion := range dependencies { //highly dependent on the order
+			currentN++
+			pkgTag := fmt.Sprintf(":package: %d/%d %s ⤑ :hammer: build %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
+			Info(pkgTag, " starts")
+			compileSpec, err := cs.FromPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+			}
+			compileSpec.SetOutputPath(p.GetOutputPath())
 
-		lastHash = currentPackageImageHash
-		if compileSpec.GetImage() != "" {
-			// TODO: Refactor this
-			if compileSpec.ImageUnpack() { // If it is just an entire image, create a package from it
-				if compileSpec.GetImage() == "" {
-					return nil, errors.New("No image defined for package: " + assertion.Package.GetName())
-				}
-				Info(":whale: Sourcing package from image", compileSpec.GetImage())
-				artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions)
+			buildImageHash := cs.ImageRepository + ":" + assertion.Hash.BuildHash
+			currentPackageImageHash := cs.ImageRepository + ":" + assertion.Hash.PackageHash
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from", buildImageHash)
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
+
+			lastHash = currentPackageImageHash
+			if compileSpec.GetImage() != "" {
+				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
+				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 				if err != nil {
-					deperrs = append(deperrs, err)
-					break // stop at first error
+					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				}
 				departifacts = append(departifacts, artifact)
+				Info(pkgTag, ":white_check_mark: Done")
 				continue
 			}
 
-			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().GetFingerPrint()+" from image")
-			artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, compileSpec)
+			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
+			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 			if err != nil {
-				deperrs = append(deperrs, err)
-				break // stop at first error
+				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
+				//	deperrs = append(deperrs, err)
+				//		break // stop at first error
 			}
 			departifacts = append(departifacts, artifact)
 			Info(pkgTag, ":white_check_mark: Done")
-			continue
 		}
 
-		artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, compileSpec)
+	} else if len(dependencies) > 0 {
+		lastHash = cs.ImageRepository + ":" + dependencies[len(dependencies)-1].Hash.PackageHash
+	}
+
+	if !cs.Options.OnlyDeps {
+		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
+		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", lastHash)
+		artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 		if err != nil {
-			return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
-			//	deperrs = append(deperrs, err)
-			//		break // stop at first error
+			return artifact, err
 		}
-		departifacts = append(departifacts, artifact)
-		Info(pkgTag, ":collision: Done")
-	}
-	Info(":package:", p.GetPackage().GetName(), ":cyclone:  Building package target from:", lastHash)
-	artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, p)
-	if err != nil {
-		return artifact, err
-	}
-	artifact.SetDependencies(departifacts)
-	artifact.SetSourceAssertion(p.GetSourceAssertion())
+		artifact.SetDependencies(departifacts)
+		artifact.SetSourceAssertion(p.GetSourceAssertion())
 
-	return artifact, err
+		return artifact, err
+	} else {
+		return departifacts[len(departifacts)-1], nil
+	}
 }
 
+type templatedata map[string]interface{}
+
 func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 
-	pack, err := cs.Tree().FindPackage(p)
+	pack, err := cs.Database.FindPackageCandidate(p)
 	if err != nil {
 		return nil, err
 	}
 
-	buildFile := pack.Rel(BuildFile)
-	if !helpers.Exists(buildFile) {
-		return nil, errors.New("No build file present for " + p.GetFingerPrint())
+	out, err := helpers.RenderFiles(pack.Rel(BuildFile), pack.Rel(DefinitionFile))
+	if err != nil {
+		return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 	}
 
-	dat, err := ioutil.ReadFile(buildFile)
-	if err != nil {
-		return nil, err
-	}
-	return NewLuetCompilationSpec(dat, pack)
+	return NewLuetCompilationSpec([]byte(out), pack)
 }
 
 func (cs *LuetCompiler) GetBackend() CompilerBackend {

pkg/compiler/compiler_suite_test.go

@@ -18,11 +18,15 @@ package compiler_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Compiler Suite")
 }

pkg/compiler/compiler_test.go

@@ -23,6 +23,7 @@ import (
 	sd "github.com/mudler/luet/pkg/compiler/backend"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -35,13 +36,10 @@ var _ = Describe("Compiler", func() {
 
 			err := generalRecipe.Load("../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			err = compiler.Prepare(1)
-			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -56,7 +54,9 @@ var _ = Describe("Compiler", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			artifact, err := compiler.Compile(2, false, spec)
+			compiler.SetConcurrency(2)
+
+			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -80,13 +80,10 @@ var _ = Describe("Compiler", func() {
 
 			err := generalRecipe.Load("../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			err = compiler.Prepare(1)
-			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -101,7 +98,8 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec, spec2))
+			compiler.SetConcurrency(2)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
 				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
@@ -111,6 +109,27 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Templated packages", func() {
+		It("Renders", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/templates")
+			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+			pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec, err := compiler.FromPackage(pkg)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetImage()).To(Equal("b:bar"))
+		})
+	})
+
 	Context("Reconstruct image tree", func() {
 		It("Compiles it", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -120,14 +139,10 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/buildableseed")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(4))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-
-			err = compiler.Prepare(1)
-			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -144,8 +159,9 @@ var _ = Describe("Compiler", func() {
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(2)
 
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec, spec2, spec3))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2, spec3))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(3))
 
@@ -180,13 +196,10 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/layers")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(2))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			err = compiler.Prepare(1)
-			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -194,12 +207,12 @@ var _ = Describe("Compiler", func() {
 			Expect(err).ToNot(HaveOccurred())
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
-			artifacts2, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec2))
+			artifacts2, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts2)).To(Equal(1))
 
@@ -225,13 +238,10 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/include")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			err = compiler.Prepare(1)
-			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -240,8 +250,9 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
@@ -254,6 +265,184 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
+		It("Compiles and excludes files", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludes")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles includes and excludes files", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludesincludes")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+		})
+
+		It("Compiles and excludes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles includes and excludes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludeincludeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles and includes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/includeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test2"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("lib/firmware"))).ToNot(BeTrue())
+		})
+
 		It("Compiles a more complex tree", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
@@ -262,13 +451,10 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/layered")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			err = compiler.Prepare(1)
-			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -277,8 +463,9 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
@@ -292,9 +479,97 @@ var _ = Describe("Compiler", func() {
 
 			Expect(helpers.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("base-layer-0.1.metadata.yaml"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.metadata.yaml"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
+		It("Compiles with provides support", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/provides")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
+		})
+
+		It("Compiles with provides and selectors support", func() {
+			// Same test as before, but fixtures differs
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/provides_selector")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
+		})
 		It("Compiles revdeps", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "revdep")
@@ -303,13 +578,10 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/layered")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			err = compiler.Prepare(1)
-			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -319,7 +591,7 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(2))
 
@@ -336,6 +608,45 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
+		It("Compiles complex dependencies trees with best matches", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "complex")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/complex/selection")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(6))
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("build-sabayon-overlays-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("linux-sabayon-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("sabayon-sources-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("vhba"))).To(BeTrue())
+		})
+
 		It("Compiles revdeps with seeds", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
@@ -344,19 +655,16 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/buildableseed")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(4))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			err = compiler.Prepare(1)
-			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(1, false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(4))
 
@@ -399,13 +707,10 @@ var _ = Describe("Compiler", func() {
 
 			err := generalRecipe.Load("../../tests/fixtures/expansion")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			err = compiler.Prepare(1)
-			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -417,7 +722,9 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(2)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
 				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
@@ -443,13 +750,10 @@ var _ = Describe("Compiler", func() {
 
 			err := generalRecipe.Load("../../tests/fixtures/packagelayers")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(2))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			err = compiler.Prepare(1)
-			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -461,13 +765,164 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec))
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
 			Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("bin/sh"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
 	})
+
+	Context("Packages which conents are a package folder", func() {
+		It("Compiles it in parallel", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/package_dir")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{
+				Name:     "dironly",
+				Category: "test",
+				Version:  "1.0",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{
+				Name:     "dironly_filter",
+				Category: "test",
+				Version:  "1.0",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			tmpdir2, err := ioutil.TempDir("", "tree2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir2) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir2)
+
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(2))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(0))
+
+			Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test2"))).To(BeTrue())
+
+			Expect(helpers.Untar(spec2.Rel("dironly_filter-test-1.0.package.tar"), tmpdir2, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec2.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
+		})
+	})
+
+	Context("Compression", func() {
+		It("Builds packages in gzip", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/packagelayers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			compiler.SetCompressionType(GZip)
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
+			Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred())
+			//	Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
+		})
+	})
+
+	Context("Compilation of whole tree", func() {
+		It("doesn't include dependencies that would be compiled anyway", func() {
+			// As some specs are dependent from each other, don't pull it in if they would
+			// be eventually
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/includeimage")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(specs)).To(Equal(1))
+
+			Expect(specs[0].GetPackage().GetFingerPrint()).To(Equal("b-test-1.0"))
+		})
+	})
+
+	Context("File list", func() {
+		It("is generated after the compilation process and annotated in the metadata", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/packagelayers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			compiler.SetCompressionType(GZip)
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(artifacts[0].GetFiles()).To(ContainElement("bin/busybox"))
+
+			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
+
+			art, err := LoadArtifactFromYaml(spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			files := art.GetFiles()
+			Expect(files).To(ContainElement("bin/busybox"))
+		})
+	})
 })

pkg/compiler/interface.go

@@ -16,21 +16,25 @@
 package compiler
 
 import (
+	"runtime"
+
+	"github.com/mudler/luet/pkg/config"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 )
 
 type Compiler interface {
-	Compile(int, bool, CompilationSpec) (Artifact, error)
-	CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
+	Compile(bool, CompilationSpec) (Artifact, error)
+	CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
+	CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
-	Prepare(concurrency int) error
-
+	ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error)
+	SetConcurrency(i int)
 	FromPackage(pkg.Package) (CompilationSpec, error)
-
+	FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error)
 	SetBackend(CompilerBackend)
 	GetBackend() CompilerBackend
+	SetCompressionType(t CompressionImplementation)
 }
 
 type CompilerBackendOptions struct {
@@ -40,6 +44,36 @@ type CompilerBackendOptions struct {
 	Destination    string
 }
 
+type CompilerOptions struct {
+	ImageRepository          string
+	PullFirst, KeepImg, Push bool
+	Concurrency              int
+	CompressionType          CompressionImplementation
+	Clean                    bool
+	KeepImageExport          bool
+
+	OnlyDeps             bool
+	NoDeps               bool
+	SolverOptions        config.LuetSolverOptions
+	SkipIfMetadataExists bool
+
+	PackageTargetOnly bool
+}
+
+func NewDefaultCompilerOptions() *CompilerOptions {
+	return &CompilerOptions{
+		ImageRepository: "luet/cache",
+		PullFirst:       false,
+		Push:            false,
+		CompressionType: None,
+		KeepImg:         true,
+		Concurrency:     runtime.NumCPU(),
+		Clean:           true,
+		OnlyDeps:        false,
+		NoDeps:          false,
+	}
+}
+
 type CompilerBackend interface {
 	BuildImage(CompilerBackendOptions) error
 	ExportImage(CompilerBackendOptions) error
@@ -50,6 +84,10 @@ type CompilerBackend interface {
 
 	CopyImage(string, string) error
 	DownloadImage(opts CompilerBackendOptions) error
+
+	Push(opts CompilerBackendOptions) error
+
+	ImageExists(string) bool
 }
 
 type Artifact interface {
@@ -62,6 +100,19 @@ type Artifact interface {
 
 	SetCompileSpec(as CompilationSpec)
 	GetCompileSpec() CompilationSpec
+	WriteYaml(dst string) error
+	Unpack(dst string, keepPerms bool) error
+	Compress(src string, concurrency int) error
+	SetCompressionType(t CompressionImplementation)
+	FileList() ([]string, error)
+	Hash() error
+	Verify() error
+
+	SetFiles(f []string)
+	GetFiles() []string
+
+	GetChecksums() Checksums
+	SetChecksums(c Checksums)
 }
 
 type ArtifactNode struct {
@@ -78,11 +129,25 @@ type ArtifactLayer struct {
 	ToImage   string        `json:"Image2"`
 	Diffs     ArtifactDiffs `json:"Diff"`
 }
+type ArtifactLayerSummary struct {
+	FromImage   string `json:"image1"`
+	ToImage     string `json:"image2"`
+	AddFiles    int    `json:"add_files"`
+	AddSizes    int64  `json:"add_sizes"`
+	DelFiles    int    `json:"del_files"`
+	DelSizes    int64  `json:"del_sizes"`
+	ChangeFiles int    `json:"change_files"`
+	ChangeSizes int64  `json:"change_sizes"`
+}
+type ArtifactLayersSummary struct {
+	Layers []ArtifactLayerSummary `json:"summary"`
+}
 
 // CompilationSpec represent a compilation specification derived from a package
 type CompilationSpec interface {
 	ImageUnpack() bool // tells if the definition is just an image
 	GetIncludes() []string
+	GetExcludes() []string
 
 	RenderBuildImage() (string, error)
 	WriteBuildImageDefinition(string) error
@@ -107,6 +172,12 @@ type CompilationSpec interface {
 
 	GetSourceAssertion() solver.PackagesAssertions
 	SetSourceAssertion(as solver.PackagesAssertions)
+
+	GetRetrieve() []string
+	CopyRetrieves(dest string) error
+
+	SetPackageDir(string)
+	GetPackageDir() string
 }
 
 type CompilationSpecs interface {

pkg/compiler/spec.go

@@ -21,6 +21,7 @@ import (
 
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
+	"github.com/otiai10/copy"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -87,16 +88,21 @@ func (specs *LuetCompilationspecs) Unique() CompilationSpecs {
 }
 
 type LuetCompilationSpec struct {
-	Steps           []string                  `json:"steps"`   // Are run inside a container and the result layer diff is saved
+	Steps           []string                  `json:"steps"` // Are run inside a container and the result layer diff is saved
+	Env             []string                  `json:"env"`
 	Prelude         []string                  `json:"prelude"` // Are run inside the image which will be our builder
 	Image           string                    `json:"image"`
 	Seed            string                    `json:"seed"`
-	Package         pkg.Package               `json:"-"`
+	Package         *pkg.DefaultPackage       `json:"package"`
 	SourceAssertion solver.PackagesAssertions `json:"-"`
+	PackageDir      string                    `json:"package_dir" yaml:"package_dir"`
+
+	Retrieve []string `json:"retrieve"`
 
 	OutputPath string   `json:"-"` // Where the build processfiles go
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
+	Excludes   []string `json:"excludes"`
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
@@ -105,7 +111,7 @@ func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
 	if err != nil {
 		return &spec, err
 	}
-	spec.Package = p
+	spec.Package = p.(*pkg.DefaultPackage)
 	return &spec, nil
 }
 func (a *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
@@ -119,6 +125,14 @@ func (cs *LuetCompilationSpec) GetPackage() pkg.Package {
 	return cs.Package
 }
 
+func (cs *LuetCompilationSpec) GetPackageDir() string {
+	return cs.PackageDir
+}
+
+func (cs *LuetCompilationSpec) SetPackageDir(s string) {
+	cs.PackageDir = s
+}
+
 func (cs *LuetCompilationSpec) BuildSteps() []string {
 	return cs.Steps
 }
@@ -135,6 +149,14 @@ func (cs *LuetCompilationSpec) GetIncludes() []string {
 	return cs.Includes
 }
 
+func (cs *LuetCompilationSpec) GetExcludes() []string {
+	return cs.Excludes
+}
+
+func (cs *LuetCompilationSpec) GetRetrieve() []string {
+	return cs.Retrieve
+}
+
 func (cs *LuetCompilationSpec) GetSeedImage() string {
 	return cs.Seed
 }
@@ -163,13 +185,52 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 	cs.Seed = s
 }
 
+func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
+	var err error
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			matches, err := filepath.Glob(cs.Rel(s))
+
+			if err != nil {
+				continue
+			}
+
+			for _, m := range matches {
+				err = copy.Copy(m, filepath.Join(dest, filepath.Base(m)))
+			}
+		}
+	}
+	return err
+}
+
 // TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
 func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
 	spec := `
 FROM ` + cs.GetSeedImage() + `
 COPY . /luetbuild
 WORKDIR /luetbuild
-`
+ENV PACKAGE_NAME=` + cs.Package.GetName() + `
+ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
+ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
+
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			//var file string
+			// if helpers.IsValidUrl(s) {
+			// 	file = s
+			// } else {
+			// 	file = cs.Rel(s)
+			// }
+			spec = spec + `
+ADD ` + s + ` /luetbuild/`
+		}
+	}
+
+	for _, s := range cs.Env {
+		spec = spec + `
+ENV ` + s
+	}
+
 	for _, s := range cs.GetPreBuildSteps() {
 		spec = spec + `
 RUN ` + s
@@ -180,7 +241,15 @@ RUN ` + s
 // TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
 func (cs *LuetCompilationSpec) RenderStepImage(image string) (string, error) {
 	spec := `
-FROM ` + image
+FROM ` + image + `
+ENV PACKAGE_NAME=` + cs.Package.GetName() + `
+ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
+ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
+
+	for _, s := range cs.Env {
+		spec = spec + `
+ENV ` + s
+	}
 	for _, s := range cs.BuildSteps() {
 		spec = spec + `
 RUN ` + s

pkg/compiler/spec_test.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -58,11 +59,10 @@ var _ = Describe("Spec", func() {
 
 			err := generalRecipe.Load("../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -76,6 +76,7 @@ var _ = Describe("Spec", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
+			lspec.Env = []string{"test=1"}
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
@@ -84,7 +85,10 @@ var _ = Describe("Spec", func() {
 FROM alpine
 COPY . /luetbuild
 WORKDIR /luetbuild
-`))
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin
+ENV test=1`))
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
@@ -92,10 +96,84 @@ WORKDIR /luetbuild
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin
+ENV test=1
 RUN echo foo > /test
 RUN echo bar > /test2`))
 
 		})
 
 	})
+
+	It("Renders retrieve and env fields", func() {
+		generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
+
+		err := generalRecipe.Load("../../tests/fixtures/retrieve")
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+		spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
+		Expect(err).ToNot(HaveOccurred())
+
+		lspec, ok := spec.(*LuetCompilationSpec)
+		Expect(ok).To(BeTrue())
+
+		Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
+		Expect(lspec.Image).To(Equal("luet/base"))
+		Expect(lspec.Seed).To(Equal("alpine"))
+		tmpdir, err := ioutil.TempDir("", "tree")
+		Expect(err).ToNot(HaveOccurred())
+		defer os.RemoveAll(tmpdir) // clean up
+
+		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1`))
+
+		lspec.SetOutputPath("/foo/bar")
+
+		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1`))
+
+		err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(dockerfile).To(Equal(`
+FROM luet/base
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ENV test=1
+RUN echo foo > /test
+RUN echo bar > /test2`))
+
+	})
 })

pkg/config/config.go

@@ -0,0 +1,411 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/mudler/luet/pkg/helpers"
+	solver "github.com/mudler/luet/pkg/solver"
+
+	v "github.com/spf13/viper"
+)
+
+var LuetCfg = NewLuetConfig(v.GetViper())
+var AvailableResolvers = strings.Join([]string{solver.QLearningResolverType}, " ")
+
+type LuetLoggingConfig struct {
+	// Path of the logfile
+	Path string `mapstructure:"path"`
+	// Enable/Disable logging to file
+	EnableLogFile bool `mapstructure:"enable_logfile"`
+	// Enable JSON format logging in file
+	JsonFormat bool `mapstructure:"json_format"`
+
+	// Log level
+	Level string `mapstructure:"level"`
+
+	// Enable emoji
+	EnableEmoji bool `mapstructure:"enable_emoji"`
+	// Enable/Disable color in logging
+	Color bool `mapstructure:"color"`
+}
+
+type LuetGeneralConfig struct {
+	SameOwner       bool `mapstructure:"same_owner"`
+	Concurrency     int  `mapstructure:"concurrency"`
+	Debug           bool `mapstructure:"debug"`
+	ShowBuildOutput bool `mapstructure:"show_build_output"`
+	SpinnerMs       int  `mapstructure:"spinner_ms"`
+	SpinnerCharset  int  `mapstructure:"spinner_charset"`
+	FatalWarns      bool `mapstructure:"fatal_warnings"`
+}
+
+type LuetSolverOptions struct {
+	Type           string            `mapstructure:"type"`
+	LearnRate      float32           `mapstructure:"rate"`
+	Discount       float32           `mapstructure:"discount"`
+	MaxAttempts    int               `mapstructure:"max_attempts"`
+	Implementation solver.SolverType `mapstructure:"implementation"`
+}
+
+func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		if opts.LearnRate != 0.0 {
+			return solver.NewQLearningResolver(opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+
+		}
+		return solver.SimpleQLearningSolver()
+	}
+
+	return &solver.DummyPackageResolver{}
+}
+
+func (opts *LuetSolverOptions) CompactString() string {
+	return fmt.Sprintf("type: %s rate: %f, discount: %f, attempts: %d, initialobserved: %d",
+		opts.Type, opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+}
+
+type LuetSystemConfig struct {
+	DatabaseEngine string `yaml:"database_engine" mapstructure:"database_engine"`
+	DatabasePath   string `yaml:"database_path" mapstructure:"database_path"`
+	Rootfs         string `yaml:"rootfs" mapstructure:"rootfs"`
+	PkgsCachePath  string `yaml:"pkgs_cache_path" mapstructure:"pkgs_cache_path"`
+	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
+}
+
+func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
+	dbpath = filepath.Join(dbpath, "repos/"+name)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc *LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+	dbpath := filepath.Join(sc.Rootfs,
+		sc.DatabasePath)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+	var cachepath string
+	if sc.PkgsCachePath != "" {
+		cachepath = sc.PkgsCachePath
+	} else {
+		// Create dynamic cache for test suites
+		cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
+	}
+
+	if filepath.IsAbs(cachepath) {
+		ans = cachepath
+	} else {
+		ans = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
+	}
+
+	return
+}
+
+func (sc *LuetSystemConfig) GetRootFsAbs() (string, error) {
+	return filepath.Abs(sc.Rootfs)
+}
+
+type LuetRepository struct {
+	Name           string            `json:"name" yaml:"name" mapstructure:"name"`
+	Description    string            `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
+	Urls           []string          `json:"urls" yaml:"urls" mapstructure:"urls"`
+	Type           string            `json:"type" yaml:"type" mapstructure:"type"`
+	Mode           string            `json:"mode,omitempty" yaml:"mode,omitempty" mapstructure:"mode,omitempty"`
+	Priority       int               `json:"priority,omitempty" yaml:"priority,omitempty" mapstructure:"priority"`
+	Enable         bool              `json:"enable" yaml:"enable" mapstructure:"enable"`
+	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
+	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
+	TreePath       string            `json:"tree_path,omitempty" yaml:"tree_path,omitempty" mapstructure:"tree_path"`
+	MetaPath       string            `json:"meta_path,omitempty" yaml:"meta_path,omitempty" mapstructure:"meta_path"`
+
+	// Serialized options not used in repository configuration
+
+	// Incremented value that identify revision of the repository in a user-friendly way.
+	Revision int `json:"revision,omitempty" yaml:"-,omitempty" mapstructure:"-,omitempty"`
+	// Epoch time in seconds
+	LastUpdate string `json:"last_update,omitempty" yaml:"-,omitempty" mapstructure:"-,omitempty"`
+}
+
+func NewLuetRepository(name, t, descr string, urls []string, priority int, enable, cached bool) *LuetRepository {
+	return &LuetRepository{
+		Name:        name,
+		Description: descr,
+		Urls:        urls,
+		Type:        t,
+		// Used in cached repositories
+		Mode:           "",
+		Priority:       priority,
+		Enable:         enable,
+		Cached:         cached,
+		Authentication: make(map[string]string, 0),
+		TreePath:       "",
+		MetaPath:       "",
+	}
+}
+
+func NewEmptyLuetRepository() *LuetRepository {
+	return &LuetRepository{
+		Name:           "",
+		Description:    "",
+		Urls:           []string{},
+		Type:           "",
+		Priority:       9999,
+		TreePath:       "",
+		MetaPath:       "",
+		Enable:         false,
+		Cached:         false,
+		Authentication: make(map[string]string, 0),
+	}
+}
+
+func (r *LuetRepository) String() string {
+	return fmt.Sprintf("[%s] prio: %d, type: %s, enable: %t, cached: %t",
+		r.Name, r.Priority, r.Type, r.Enable, r.Cached)
+}
+
+type LuetConfig struct {
+	Viper *v.Viper
+
+	Logging LuetLoggingConfig `mapstructure:"logging"`
+	General LuetGeneralConfig `mapstructure:"general"`
+	System  LuetSystemConfig  `mapstructure:"system"`
+	Solver  LuetSolverOptions `mapstructure:"solver"`
+
+	RepositoriesConfDir  []string         `mapstructure:"repos_confdir"`
+	ConfigProtectConfDir []string         `mapstructure:"config_protect_confdir"`
+	ConfigProtectSkip    bool             `mapstructure:"config_protect_skip"`
+	ConfigFromHost       bool             `mapstructure:"config_from_host"`
+	CacheRepositories    []LuetRepository `mapstructure:"repetitors"`
+	SystemRepositories   []LuetRepository `mapstructure:"repositories"`
+
+	ConfigProtectConfFiles []ConfigProtectConfFile
+}
+
+func NewLuetConfig(viper *v.Viper) *LuetConfig {
+	if viper == nil {
+		viper = v.New()
+	}
+
+	GenDefault(viper)
+	return &LuetConfig{Viper: viper, ConfigProtectConfFiles: nil}
+}
+
+func GenDefault(viper *v.Viper) {
+	viper.SetDefault("logging.level", "info")
+	viper.SetDefault("logging.enable_logfile", false)
+	viper.SetDefault("logging.path", "/var/log/luet.log")
+	viper.SetDefault("logging.json_format", false)
+	viper.SetDefault("logging.enable_emoji", true)
+	viper.SetDefault("logging.color", true)
+
+	viper.SetDefault("general.concurrency", runtime.NumCPU())
+	viper.SetDefault("general.debug", false)
+	viper.SetDefault("general.show_build_output", false)
+	viper.SetDefault("general.spinner_ms", 100)
+	viper.SetDefault("general.spinner_charset", 22)
+	viper.SetDefault("general.fatal_warnings", false)
+
+	u, err := user.Current()
+	// os/user doesn't work in from scratch environments
+	if err != nil || (u != nil && u.Uid == "0") {
+		viper.SetDefault("general.same_owner", true)
+	} else {
+		viper.SetDefault("general.same_owner", false)
+	}
+
+	viper.SetDefault("system.database_engine", "boltdb")
+	viper.SetDefault("system.database_path", "/var/cache/luet")
+	viper.SetDefault("system.rootfs", "/")
+	viper.SetDefault("system.tmpdir_base", filepath.Join(os.TempDir(), "tmpluet"))
+	viper.SetDefault("system.pkgs_cache_path", "packages")
+
+	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
+	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
+	viper.SetDefault("config_protect_skip", false)
+	// TODO: Set default to false when we are ready for migration.
+	viper.SetDefault("config_from_host", true)
+	viper.SetDefault("cache_repositories", []string{})
+	viper.SetDefault("system_repositories", []string{})
+
+	viper.SetDefault("solver.type", "")
+	viper.SetDefault("solver.rate", 0.7)
+	viper.SetDefault("solver.discount", 1.0)
+	viper.SetDefault("solver.max_attempts", 9000)
+}
+
+func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
+	c.SystemRepositories = append(c.SystemRepositories, r)
+}
+
+func (c *LuetConfig) GetLogging() *LuetLoggingConfig {
+	return &c.Logging
+}
+
+func (c *LuetConfig) GetGeneral() *LuetGeneralConfig {
+	return &c.General
+}
+
+func (c *LuetConfig) GetSystem() *LuetSystemConfig {
+	return &c.System
+}
+
+func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
+	return &c.Solver
+}
+
+func (c *LuetConfig) GetConfigProtectConfFiles() []ConfigProtectConfFile {
+	return c.ConfigProtectConfFiles
+}
+
+func (c *LuetConfig) AddConfigProtectConfFile(file *ConfigProtectConfFile) {
+	if c.ConfigProtectConfFiles == nil {
+		c.ConfigProtectConfFiles = []ConfigProtectConfFile{*file}
+	} else {
+		c.ConfigProtectConfFiles = append(c.ConfigProtectConfFiles, *file)
+	}
+}
+
+func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
+	var ans *LuetRepository = nil
+
+	for idx, repo := range c.SystemRepositories {
+		if repo.Name == name {
+			ans = &c.SystemRepositories[idx]
+			break
+		}
+	}
+	if ans == nil {
+		return nil, errors.New("Repository " + name + " not found")
+	}
+
+	return ans, nil
+}
+
+func (c *LuetSolverOptions) String() string {
+	ans := fmt.Sprintf(`
+solver:
+  type: %s
+  rate: %f
+  discount: %f
+  max_attempts: %d`, c.Type, c.LearnRate, c.Discount,
+		c.MaxAttempts)
+
+	return ans
+}
+
+func (c *LuetGeneralConfig) String() string {
+	ans := fmt.Sprintf(`
+general:
+  concurrency: %d
+  same_owner: %t
+  debug: %t
+  fatal_warnings: %t
+  show_build_output: %t
+  spinner_ms: %d
+  spinner_charset: %d`, c.Concurrency, c.SameOwner, c.Debug,
+		c.FatalWarns, c.ShowBuildOutput,
+		c.SpinnerMs, c.SpinnerCharset)
+
+	return ans
+}
+
+func (c *LuetGeneralConfig) GetSpinnerMs() time.Duration {
+	duration, err := time.ParseDuration(fmt.Sprintf("%dms", c.SpinnerMs))
+	if err != nil {
+		return 100 * time.Millisecond
+	}
+	return duration
+}
+
+func (c *LuetLoggingConfig) SetLogLevel(s string) {
+	c.Level = s
+}
+
+func (c *LuetLoggingConfig) String() string {
+	ans := fmt.Sprintf(`
+logging:
+  enable_logfile: %t
+  path: %s
+  json_format: %t
+  color: %t
+  enable_emoji: %t
+  level: %s`, c.EnableLogFile, c.Path, c.JsonFormat,
+		c.Color, c.EnableEmoji, c.Level)
+
+	return ans
+}
+
+func (c *LuetSystemConfig) String() string {
+	ans := fmt.Sprintf(`
+system:
+  database_engine: %s
+  database_path: %s
+  pkgs_cache_path: %s
+  tmpdir_base: %s
+  rootfs: %s`,
+		c.DatabaseEngine, c.DatabasePath, c.PkgsCachePath,
+		c.TmpDirBase, c.Rootfs)
+
+	return ans
+}
+
+func (c *LuetSystemConfig) InitTmpDir() error {
+	if !helpers.Exists(c.TmpDirBase) {
+		return os.MkdirAll(c.TmpDirBase, os.ModePerm)
+	}
+	return nil
+}
+
+func (c *LuetSystemConfig) CleanupTmpDir() error {
+	return os.RemoveAll(c.TmpDirBase)
+}
+
+func (c *LuetSystemConfig) TempDir(pattern string) (string, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return "", err
+	}
+	return ioutil.TempDir(c.TmpDirBase, pattern)
+}
+
+func (c *LuetSystemConfig) TempFile(pattern string) (*os.File, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.TempFile(c.TmpDirBase, pattern)
+}

pkg/config/config_protect.go

@@ -0,0 +1,119 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+type ConfigProtectConfFile struct {
+	Filename string
+
+	Name        string   `mapstructure:"name" yaml:"name" json:"name"`
+	Directories []string `mapstructure:"dirs" yaml:"dirs" json:"dirs"`
+}
+
+func NewConfigProtectConfFile(filename string) *ConfigProtectConfFile {
+	return &ConfigProtectConfFile{
+		Filename:    filename,
+		Name:        "",
+		Directories: []string{},
+	}
+}
+
+func (c *ConfigProtectConfFile) String() string {
+	return fmt.Sprintf("[%s] filename: %s, dirs: %s", c.Name, c.Filename,
+		c.Directories)
+}
+
+type ConfigProtect struct {
+	AnnotationDir string
+	MapProtected  map[string]bool
+}
+
+func NewConfigProtect(annotationDir string) *ConfigProtect {
+	if len(annotationDir) > 0 && annotationDir[0:1] != "/" {
+		annotationDir = "/" + annotationDir
+	}
+	return &ConfigProtect{
+		AnnotationDir: annotationDir,
+		MapProtected:  make(map[string]bool, 0),
+	}
+}
+
+func (c *ConfigProtect) AddAnnotationDir(d string) {
+	c.AnnotationDir = d
+}
+
+func (c *ConfigProtect) GetAnnotationDir() string {
+	return c.AnnotationDir
+}
+
+func (c *ConfigProtect) Map(files []string) {
+	if LuetCfg.ConfigProtectSkip {
+		return
+	}
+
+	for _, file := range files {
+
+		if file[0:1] != "/" {
+			file = "/" + file
+		}
+
+		if len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
+				for _, dir := range conf.Directories {
+					// Note file is without / at begin (on unpack)
+					if strings.HasPrefix(file, filepath.Clean(dir)) {
+						// docker archive modifier works with path without / at begin.
+						c.MapProtected[file] = true
+						goto nextFile
+					}
+				}
+			}
+		}
+
+		if c.AnnotationDir != "" && strings.HasPrefix(file, filepath.Clean(c.AnnotationDir)) {
+			c.MapProtected[file] = true
+		}
+	nextFile:
+	}
+
+}
+
+func (c *ConfigProtect) Protected(file string) bool {
+	if file[0:1] != "/" {
+		file = "/" + file
+	}
+	_, ans := c.MapProtected[file]
+	return ans
+}
+
+func (c *ConfigProtect) GetProtectFiles(withSlash bool) []string {
+	ans := []string{}
+
+	for key, _ := range c.MapProtected {
+		if withSlash {
+			ans = append(ans, key)
+		} else {
+			ans = append(ans, key[1:])
+		}
+	}
+	return ans
+}

pkg/config/config_protect_test.go

@@ -0,0 +1,118 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+
+	Context("Test config protect", func() {
+
+		It("Protect1", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("/etc")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+		It("Protect2", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{},
+			))
+		})
+
+		It("Protect3: Annotation dir without initial slash", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("etc")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+	})
+
+})

pkg/config/config_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Config Suite")
+}

pkg/config/config_test.go

@@ -0,0 +1,66 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	config "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+
+	Context("Simple temporary directory creation", func() {
+
+		It("Create Temporary directory", func() {
+			// PRE: tmpdir_base contains default value.
+
+			tmpDir, err := config.LuetCfg.GetSystem().TempDir("test1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.HasPrefix(tmpDir, filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
+			Expect(helpers.Exists(tmpDir)).To(BeTrue())
+
+			defer os.RemoveAll(tmpDir)
+		})
+
+		It("Create Temporary file", func() {
+			// PRE: tmpdir_base contains default value.
+
+			tmpFile, err := config.LuetCfg.GetSystem().TempFile("testfile1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.HasPrefix(tmpFile.Name(), filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
+			Expect(helpers.Exists(tmpFile.Name())).To(BeTrue())
+
+			defer os.Remove(tmpFile.Name())
+		})
+
+		It("Config1", func() {
+			cfg := config.LuetCfg
+
+			cfg.GetLogging().Color = false
+			Expect(cfg.GetLogging().Color).To(BeFalse())
+		})
+
+	})
+
+})

pkg/helpers/archive.go

@@ -16,8 +16,11 @@
 package helpers
 
 import (
+	"archive/tar"
+	"bytes"
 	"io"
 	"os"
+	"path/filepath"
 
 	"github.com/docker/docker/pkg/archive"
 )
@@ -47,16 +50,171 @@ func Tar(src, dest string) error {
 	return err
 }
 
-// Untar just a wrapper around the docker functions
-func Untar(src, dest string, sameOwner bool) error {
+type TarModifierWrapperFunc func(path, dst string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
+type TarModifierWrapper struct {
+	DestinationPath string
+	Modifier        TarModifierWrapperFunc
+}
+
+func NewTarModifierWrapper(dst string, modifier TarModifierWrapperFunc) *TarModifierWrapper {
+	return &TarModifierWrapper{
+		DestinationPath: dst,
+		Modifier:        modifier,
+	}
+}
+
+func (m *TarModifierWrapper) GetModifier() archive.TarModifierFunc {
+	return func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+		return m.Modifier(m.DestinationPath, path, header, content)
+	}
+}
+
+func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modifier *TarModifierWrapper) error {
+	var ans error
+
+	if len(protectedFiles) <= 0 {
+		return Untar(src, dst, sameOwner)
+	}
+
+	// POST: we have files to protect. I create a ReplaceFileTarWrapper
 	in, err := os.Open(src)
 	if err != nil {
 		return err
 	}
 	defer in.Close()
 
-	return archive.Untar(in, dest, &archive.TarOptions{
-		NoLchown:        !sameOwner,
-		ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
-	})
+	// Create modifier map
+	mods := make(map[string]archive.TarModifierFunc)
+	for _, file := range protectedFiles {
+		mods[file] = modifier.GetModifier()
+	}
+
+	if sameOwner {
+		// PRE: i have root privileged.
+
+		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
+
+		opts := &archive.TarOptions{
+			// NOTE: NoLchown boolean is used for chmod of the symlink
+			// Probably it's needed set this always to true.
+			NoLchown:        true,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+			ContinueOnError: true,
+		}
+
+		ans = archive.Untar(replacerArchive, dst, opts)
+	} else {
+		ans = unTarIgnoreOwner(dst, in, mods)
+	}
+
+	return ans
+}
+
+func unTarIgnoreOwner(dest string, in io.ReadCloser, mods map[string]archive.TarModifierFunc) error {
+	tr := tar.NewReader(in)
+	for {
+		header, err := tr.Next()
+
+		var data []byte
+		var headerReplaced = false
+
+		switch {
+		case err == io.EOF:
+			goto tarEof
+		case err != nil:
+			return err
+		case header == nil:
+			continue
+		}
+
+		// the target location where the dir/file should be created
+		target := filepath.Join(dest, header.Name)
+		if mods != nil {
+			modifier, ok := mods[header.Name]
+			if ok {
+				header, data, err = modifier(header.Name, header, tr)
+				if err != nil {
+					return err
+				}
+
+				// Override target path
+				target = filepath.Join(dest, header.Name)
+				headerReplaced = true
+			}
+
+		}
+
+		// Check the file type
+		switch header.Typeflag {
+
+		// if its a dir and it doesn't exist create it
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return err
+				}
+			}
+
+			// handle creation of file
+		case tar.TypeReg:
+
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+
+			// copy over contents
+			if headerReplaced {
+				_, err = io.Copy(f, bytes.NewReader(data))
+			} else {
+				_, err = io.Copy(f, tr)
+			}
+			if err != nil {
+				return err
+			}
+
+			// manually close here after each file operation; defering would cause each
+			// file close to wait until all operations have completed.
+			f.Close()
+
+		case tar.TypeSymlink:
+			source := header.Linkname
+			err := os.Symlink(source, target)
+			if err != nil {
+				return err
+			}
+		}
+	}
+tarEof:
+
+	return nil
+}
+
+// Untar just a wrapper around the docker functions
+func Untar(src, dest string, sameOwner bool) error {
+	var ans error
+
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	if sameOwner {
+		// PRE: i have root privileged.
+
+		opts := &archive.TarOptions{
+			// NOTE: NoLchown boolean is used for chmod of the symlink
+			// Probably it's needed set this always to true.
+			NoLchown:        true,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+			ContinueOnError: true,
+		}
+
+		ans = archive.Untar(in, dest, opts)
+	} else {
+		ans = unTarIgnoreOwner(dest, in, nil)
+	}
+
+	return ans
 }

pkg/helpers/archive_test.go

@@ -0,0 +1,134 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/archive"
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// Code from moby/moby pkg/archive/archive_test
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(filepath.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+		if makeLinks {
+			if err := os.Link(filepath.Join(targetPath, fileName), filepath.Join(targetPath, fileName+"-link")); err != nil {
+				return 0, err
+			}
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}
+
+func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	// If the destination path already exists I rename target file name with postfix.
+	var basePath string
+
+	// Read data. TODO: We need change archive callback to permit to return a Reader
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	if header != nil {
+
+		switch header.Typeflag {
+		case tar.TypeReg:
+			basePath = filepath.Base(path)
+		default:
+			// Nothing to do. I return original reader
+			return header, buffer.Bytes(), nil
+		}
+
+		if basePath == "file-0" {
+			name := filepath.Join(filepath.Join(filepath.Dir(path), fmt.Sprintf("._cfg%04d_%s", 1, basePath)))
+			return &tar.Header{
+				Mode:       header.Mode,
+				Typeflag:   header.Typeflag,
+				PAXRecords: header.PAXRecords,
+				Name:       name,
+			}, buffer.Bytes(), nil
+		} else if basePath == "file-1" {
+			return header, []byte("newcontent"), nil
+		}
+
+		// else file not present
+	}
+
+	return header, buffer.Bytes(), nil
+}
+
+var _ = Describe("Helpers Archive", func() {
+	Context("Untar Protect", func() {
+
+		It("Detect existing and not-existing files", func() {
+
+			archiveSourceDir, err := ioutil.TempDir("", "archive-source")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(archiveSourceDir)
+
+			_, err = prepareUntarSourceDirectory(10, archiveSourceDir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			targetDir, err := ioutil.TempDir("", "archive-target")
+			Expect(err).ToNot(HaveOccurred())
+			//	defer os.RemoveAll(targetDir)
+
+			sourceArchive, err := archive.TarWithOptions(archiveSourceDir, &archive.TarOptions{})
+			Expect(err).ToNot(HaveOccurred())
+			defer sourceArchive.Close()
+
+			tarModifier := NewTarModifierWrapper(targetDir, tarModifierWrapperFunc)
+			mods := make(map[string]archive.TarModifierFunc)
+			mods["file-0"] = tarModifier.GetModifier()
+			mods["file-1"] = tarModifier.GetModifier()
+			mods["file-9999"] = tarModifier.GetModifier()
+
+			replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
+			//replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
+			opts := &archive.TarOptions{
+				// NOTE: NoLchown boolean is used for chmod of the symlink
+				// Probably it's needed set this always to true.
+				NoLchown:        true,
+				ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+				ContinueOnError: true,
+			}
+
+			err = archive.Untar(replacerArchive, targetDir, opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
+		})
+	})
+})

pkg/helpers/file.go

@@ -16,13 +16,80 @@
 package helpers
 
 import (
-	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"time"
+
+	copy "github.com/otiai10/copy"
 )
 
+func OrderFiles(target string, files []string) ([]string, []string) {
+
+	var newFiles []string
+	var notPresent []string
+
+	for _, f := range files {
+		target := filepath.Join(target, f)
+		fi, err := os.Lstat(target)
+		if err != nil {
+			notPresent = append(notPresent, f)
+			continue
+		}
+		if m := fi.Mode(); !m.IsDir() {
+			newFiles = append(newFiles, f)
+		}
+	}
+
+	for _, f := range files {
+		target := filepath.Join(target, f)
+		fi, err := os.Lstat(target)
+		if err != nil {
+			continue
+		}
+		if m := fi.Mode(); m.IsDir() {
+			newFiles = append(newFiles, f)
+		}
+	}
+
+	return newFiles, notPresent
+}
+
+func ListDir(dir string) ([]string, error) {
+	content := []string{}
+
+	err := filepath.Walk(dir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			content = append(content, path)
+
+			return nil
+		})
+
+	return content, err
+}
+
+// Touch creates an empty file
+func Touch(f string) error {
+	_, err := os.Stat(f)
+	if os.IsNotExist(err) {
+		file, err := os.Create(f)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+	} else {
+		currentTime := time.Now().Local()
+		err = os.Chtimes(f, currentTime, currentTime)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Exists reports whether the named file or directory exists.
 func Exists(name string) bool {
 	if _, err := os.Stat(name); err != nil {
@@ -57,44 +124,7 @@ func ensureDir(fileName string) {
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
 func CopyFile(src, dst string) (err error) {
-	in, err := os.Open(src)
-	if err != nil {
-		return
-	}
-	defer in.Close()
-
-	ensureDir(dst) // FIXME: Breaks permissions
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return
-	}
-	defer func() {
-		if e := out.Close(); e != nil {
-			err = e
-		}
-	}()
-
-	_, err = io.Copy(out, in)
-	if err != nil {
-		return
-	}
-
-	err = out.Sync()
-	if err != nil {
-		return
-	}
-
-	si, err := os.Stat(src)
-	if err != nil {
-		return
-	}
-	err = os.Chmod(dst, si.Mode()) // FIXME: Needs owners copy as well.
-	if err != nil {
-		return
-	}
-
-	return
+	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }
 
 func IsDirectory(path string) (bool, error) {
@@ -111,54 +141,5 @@ func IsDirectory(path string) (bool, error) {
 func CopyDir(src string, dst string) (err error) {
 	src = filepath.Clean(src)
 	dst = filepath.Clean(dst)
-
-	si, err := os.Stat(src)
-	if err != nil {
-		return err
-	}
-	if !si.IsDir() {
-		return fmt.Errorf("source is not a directory")
-	}
-
-	_, err = os.Stat(dst)
-	if err != nil && !os.IsNotExist(err) {
-		return
-	}
-	if err == nil {
-		//	return fmt.Errorf("destination already exists")
-	}
-
-	err = os.MkdirAll(dst, si.Mode())
-	if err != nil {
-		return
-	}
-
-	entries, err := ioutil.ReadDir(src)
-	if err != nil {
-		return
-	}
-
-	for _, entry := range entries {
-		srcPath := filepath.Join(src, entry.Name())
-		dstPath := filepath.Join(dst, entry.Name())
-
-		if entry.IsDir() {
-			err = CopyDir(srcPath, dstPath)
-			if err != nil {
-				return
-			}
-		} else {
-			// Skip symlinks.
-			if entry.Mode()&os.ModeSymlink != 0 {
-				continue
-			}
-
-			err = CopyFile(srcPath, dstPath)
-			if err != nil {
-				return
-			}
-		}
-	}
-
-	return
+	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }

pkg/helpers/file_test.go

@@ -16,6 +16,10 @@
 package helpers_test
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -28,4 +32,33 @@ var _ = Describe("Helpers", func() {
 			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
 		})
 	})
+
+	Context("Orders dir and files correctly", func() {
+		It("puts files first and folders at end", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			err = ioutil.WriteFile(filepath.Join(testDir, "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(testDir, "baz"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "bar"), 0755)
+			Expect(err).ToNot(HaveOccurred())
+			err = ioutil.WriteFile(filepath.Join(testDir, "bar", "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "baz2"), 0755)
+			Expect(err).ToNot(HaveOccurred())
+			err = ioutil.WriteFile(filepath.Join(testDir, "baz2", "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			ordered, notExisting := OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
+
+			Expect(ordered).To(Equal([]string{"baz", "bar/foo", "foo", "baz2/foo", "bar", "baz2"}))
+			Expect(notExisting).To(Equal([]string{"notexisting"}))
+		})
+	})
 })

pkg/helpers/helm.go

@@ -0,0 +1,59 @@
+package helpers
+
+import (
+	"io/ioutil"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+)
+
+// RenderHelm renders the template string with helm
+func RenderHelm(template string, values map[string]interface{}) (string, error) {
+	c := &chart.Chart{
+		Metadata: &chart.Metadata{
+			Name:    "",
+			Version: "",
+		},
+		Templates: []*chart.File{
+			{Name: "templates", Data: []byte(template)},
+		},
+		Values: map[string]interface{}{"Values": values},
+	}
+
+	v, err := chartutil.CoalesceValues(c, map[string]interface{}{})
+	if err != nil {
+		return "", errors.Wrap(err, "while rendering template")
+	}
+	out, err := engine.Render(c, v)
+	if err != nil {
+		return "", errors.Wrap(err, "while rendering template")
+	}
+
+	return out["templates"], nil
+}
+
+type templatedata map[string]interface{}
+
+func RenderFiles(toTemplate, valuesFile string) (string, error) {
+	raw, err := ioutil.ReadFile(toTemplate)
+	if err != nil {
+		return "", errors.Wrap(err, "reading file "+toTemplate)
+	}
+
+	if !Exists(valuesFile) {
+		return "", errors.Wrap(err, "file not existing "+valuesFile)
+	}
+	def, err := ioutil.ReadFile(valuesFile)
+	if err != nil {
+		return "", errors.Wrap(err, "reading file "+valuesFile)
+	}
+
+	var values templatedata
+	if err = yaml.Unmarshal(def, &values); err != nil {
+		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
+	}
+	return RenderHelm(string(raw), values)
+}

pkg/helpers/helm_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Helpers", func() {
+	Context("RenderHelm", func() {
+		It("Renders templates", func() {
+			out, err := RenderHelm("{{.Values.Test}}", map[string]interface{}{"Test": "foo"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foo"))
+		})
+	})
+})

pkg/helpers/helpers_suite_test.go

@@ -18,11 +18,15 @@ package helpers_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Helpers Suite")
 }

pkg/helpers/match.go

@@ -0,0 +1,49 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"regexp"
+)
+
+func MapMatchRegex(m *map[string]string, r *regexp.Regexp) bool {
+	ans := false
+
+	if m != nil {
+		for k, v := range *m {
+			if r.MatchString(k + "=" + v) {
+				ans = true
+				break
+			}
+		}
+	}
+
+	return ans
+}
+
+func MapHasKey(m *map[string]string, label string) bool {
+	ans := false
+	if m != nil {
+		for k, _ := range *m {
+			if k == label {
+				ans = true
+				break
+			}
+		}
+	}
+	return ans
+}

pkg/helpers/math.go

@@ -0,0 +1,24 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+func Factorial(n uint64) (result uint64) {
+	if n > 0 {
+		result = n * Factorial(n-1)
+		return result
+	}
+	return 1
+}

pkg/helpers/sys.go

@@ -0,0 +1,48 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"os"
+	"os/exec"
+	"os/user"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+// This allows a multi-platform switch in the future
+func Exec(cmd string, args []string, env []string) error {
+	path, err := exec.LookPath(cmd)
+	if err != nil {
+		return errors.Wrap(err, "Could not find binary in path: "+cmd)
+	}
+	return syscall.Exec(path, args, env)
+}
+
+func GetHomeDir() (ans string) {
+	// os/user doesn't work in from scratch environments
+	u, err := user.Current()
+	if err == nil {
+		ans = u.HomeDir
+	} else {
+		ans = ""
+	}
+	if os.Getenv("HOME") != "" {
+		ans = os.Getenv("HOME")
+	}
+	return ans
+}

pkg/installer/client/client_suite_test.go

@@ -0,0 +1,36 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestClient(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	// Set temporary directory for rootfs
+	config.LuetCfg.GetSystem().Rootfs = "/tmp/luet-root"
+	// Force dynamic path for packages cache
+	config.LuetCfg.GetSystem().PkgsCachePath = ""
+	RunSpecs(t, "Client Suite")
+}

pkg/installer/client/http.go

@@ -0,0 +1,190 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"fmt"
+	"math"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+
+	"github.com/cavaliercoder/grab"
+)
+
+type HttpClient struct {
+	RepoData RepoData
+}
+
+func NewHttpClient(r RepoData) *HttpClient {
+	return &HttpClient{RepoData: r}
+}
+
+func (c *HttpClient) PrepareReq(dst, url string) (*grab.Request, error) {
+
+	req, err := grab.NewRequest(dst, url)
+	if err != nil {
+		return nil, err
+	}
+
+	if val, ok := c.RepoData.Authentication["token"]; ok {
+		req.HTTPRequest.Header.Set("Authorization", "token "+val)
+	} else if val, ok := c.RepoData.Authentication["basic"]; ok {
+		req.HTTPRequest.Header.Set("Authorization", "Basic "+val)
+	}
+
+	return req, err
+}
+
+func Round(input float64) float64 {
+	if input < 0 {
+		return math.Ceil(input - 0.5)
+	}
+	return math.Floor(input + 0.5)
+}
+
+func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+	var u *url.URL = nil
+	var err error
+	var req *grab.Request
+	var temp string
+
+	artifactName := path.Base(artifact.GetPath())
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
+	ok := false
+
+	// Check if file is already in cache
+	if helpers.Exists(cacheFile) {
+		Info("Use artifact", artifactName, "from cache.")
+	} else {
+
+		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
+		if err != nil {
+			return nil, err
+		}
+		defer os.RemoveAll(temp)
+
+		client := grab.NewClient()
+
+		for _, uri := range c.RepoData.Urls {
+			Info("Downloading artifact", artifactName, "from", uri)
+
+			u, err = url.Parse(uri)
+			if err != nil {
+				continue
+			}
+			u.Path = path.Join(u.Path, artifactName)
+
+			req, err = c.PrepareReq(temp, u.String())
+			if err != nil {
+				continue
+			}
+
+			resp := client.Do(req)
+			if err = resp.Err(); err != nil {
+				continue
+			}
+
+			Info("Downloaded", artifactName, "of",
+				fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
+				fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
+
+			Debug("Copying file ", filepath.Join(temp, artifactName), "to", cacheFile)
+			err = helpers.CopyFile(filepath.Join(temp, artifactName), cacheFile)
+			if err != nil {
+				continue
+			}
+
+			ok = true
+			break
+		}
+
+		if !ok {
+			return nil, err
+		}
+	}
+
+	newart := artifact
+	newart.SetPath(cacheFile)
+	return newart, nil
+}
+
+func (c *HttpClient) DownloadFile(name string) (string, error) {
+	var file *os.File = nil
+	var u *url.URL = nil
+	var err error
+	var req *grab.Request
+	var temp string
+
+	ok := false
+
+	temp, err = config.LuetCfg.GetSystem().TempDir("tree")
+	if err != nil {
+		return "", err
+	}
+
+	client := grab.NewClient()
+
+	for _, uri := range c.RepoData.Urls {
+
+		file, err = config.LuetCfg.GetSystem().TempFile("HttpClient")
+		if err != nil {
+			continue
+		}
+
+		u, err = url.Parse(uri)
+		if err != nil {
+			continue
+		}
+		u.Path = path.Join(u.Path, name)
+
+		Info("Downloading", u.String())
+
+		req, err = c.PrepareReq(temp, u.String())
+		if err != nil {
+			continue
+		}
+
+		resp := client.Do(req)
+		if err = resp.Err(); err != nil {
+			continue
+		}
+
+		Info("Downloaded", filepath.Base(resp.Filename), "of",
+			fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
+			fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
+
+		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
+
+	if !ok {
+		return "", err
+	}
+
+	return file.Name(), err
+}

pkg/installer/client/http_test.go

@@ -0,0 +1,73 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+
+	compiler "github.com/mudler/luet/pkg/compiler"
+	helpers "github.com/mudler/luet/pkg/helpers"
+
+	. "github.com/mudler/luet/pkg/installer/client"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Http client", func() {
+	Context("With repository", func() {
+
+		It("Downloads single files", func() {
+			// setup small staticfile webserver with content
+			tmpdir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			Expect(err).ToNot(HaveOccurred())
+			ts := httptest.NewServer(http.FileServer(http.Dir(tmpdir)))
+			defer ts.Close()
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
+			path, err := c.DownloadFile("test.txt")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(path)).To(Equal("test"))
+			os.RemoveAll(path)
+		})
+
+		It("Downloads artifacts", func() {
+			// setup small staticfile webserver with content
+			tmpdir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			Expect(err).ToNot(HaveOccurred())
+			ts := httptest.NewServer(http.FileServer(http.Dir(tmpdir)))
+			defer ts.Close()
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
+			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(path.GetPath())).To(Equal("test"))
+			os.RemoveAll(path.GetPath())
+		})
+
+	})
+})

pkg/installer/client/interface.go

@@ -0,0 +1,21 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+type RepoData struct {
+	Urls           []string
+	Authentication map[string]string
+}

pkg/installer/client/local.go

@@ -0,0 +1,120 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/helpers"
+)
+
+type LocalClient struct {
+	RepoData RepoData
+}
+
+func NewLocalClient(r RepoData) *LocalClient {
+	return &LocalClient{RepoData: r}
+}
+
+func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+	var err error
+
+	rootfs := ""
+	artifactName := path.Base(artifact.GetPath())
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
+
+	if !config.LuetCfg.ConfigFromHost {
+		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Check if file is already in cache
+	if helpers.Exists(cacheFile) {
+		Info("Use artifact", artifactName, "from cache.")
+	} else {
+		ok := false
+		for _, uri := range c.RepoData.Urls {
+
+			uri = filepath.Join(rootfs, uri)
+
+			Info("Downloading artifact", artifactName, "from", uri)
+
+			//defer os.Remove(file.Name())
+			err = helpers.CopyFile(filepath.Join(uri, artifactName), cacheFile)
+			if err != nil {
+				continue
+			}
+			ok = true
+			break
+		}
+
+		if !ok {
+			return nil, err
+		}
+	}
+
+	newart := artifact
+	newart.SetPath(cacheFile)
+	return newart, nil
+}
+
+func (c *LocalClient) DownloadFile(name string) (string, error) {
+	var err error
+	var file *os.File = nil
+
+	rootfs := ""
+
+	if !config.LuetCfg.ConfigFromHost {
+		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return "", err
+		}
+	}
+
+	ok := false
+	for _, uri := range c.RepoData.Urls {
+
+		uri = filepath.Join(rootfs, uri)
+
+		Info("Downloading file", name, "from", uri)
+		file, err = config.LuetCfg.GetSystem().TempFile("localclient")
+		if err != nil {
+			continue
+		}
+		//defer os.Remove(file.Name())
+
+		err = helpers.CopyFile(filepath.Join(uri, name), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
+
+	if ok {
+		return file.Name(), nil
+	}
+
+	return "", err
+}

pkg/installer/client/local_test.go

@@ -0,0 +1,66 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	compiler "github.com/mudler/luet/pkg/compiler"
+	helpers "github.com/mudler/luet/pkg/helpers"
+
+	. "github.com/mudler/luet/pkg/installer/client"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Local client", func() {
+	Context("With repository", func() {
+		It("Downloads single files", func() {
+			tmpdir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			// write the whole body at once
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
+			path, err := c.DownloadFile("test.txt")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(path)).To(Equal("test"))
+			os.RemoveAll(path)
+		})
+
+		It("Downloads artifacts", func() {
+			tmpdir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			// write the whole body at once
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "test.txt"), []byte(`test`), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
+			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(path.GetPath())).To(Equal("test"))
+			os.RemoveAll(path.GetPath())
+		})
+
+	})
+})

pkg/installer/config_protect.go

@@ -0,0 +1,100 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"io/ioutil"
+	"path"
+	"path/filepath"
+	"regexp"
+
+	"github.com/ghodss/yaml"
+
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+func LoadConfigProtectConfs(c *LuetConfig) error {
+	var regexConfs = regexp.MustCompile(`.yml$`)
+	var err error
+
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, cdir := range c.ConfigProtectConfDir {
+		cdir = filepath.Join(rootfs, cdir)
+
+		Debug("Parsing Config Protect Directory", cdir, "...")
+
+		files, err := ioutil.ReadDir(cdir)
+		if err != nil {
+			Debug("Skip dir", cdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexConfs.MatchString(file.Name()) {
+				Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(cdir, file.Name()))
+			if err != nil {
+				Warning("On read file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := LoadConfigProtectConFile(file.Name(), content)
+			if err != nil {
+				Warning("On parse file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Directories) == 0 {
+				Warning("Invalid config protect file", file.Name())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddConfigProtectConfFile(r)
+		}
+	}
+	return nil
+
+}
+
+func LoadConfigProtectConFile(filename string, data []byte) (*ConfigProtectConfFile, error) {
+	ans := NewConfigProtectConfFile(filename)
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}

pkg/installer/finalizer.go

@@ -0,0 +1,90 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"os/exec"
+
+	"github.com/ghodss/yaml"
+	box "github.com/mudler/luet/pkg/box"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/pkg/errors"
+)
+
+type LuetFinalizer struct {
+	Shell     []string `json:"shell"`
+	Install   []string `json:"install"`
+	Uninstall []string `json:"uninstall"` // TODO: Where to store?
+}
+
+func (f *LuetFinalizer) RunInstall(s *System) error {
+	var cmd string
+	var args []string
+	if len(f.Shell) == 0 {
+		// Default to sh otherwise
+		cmd = "sh"
+		args = []string{"-c"}
+	} else {
+		cmd = f.Shell[0]
+		if len(f.Shell) > 1 {
+			args = f.Shell[1:]
+		}
+	}
+
+	for _, c := range f.Install {
+		toRun := append(args, c)
+		Info(":shell: Executing finalizer on ", s.Target, cmd, toRun)
+		if s.Target == "/" {
+			cmd := exec.Command(cmd, toRun...)
+			stdoutStderr, err := cmd.CombinedOutput()
+			if err != nil {
+				return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
+			}
+			Info(string(stdoutStderr))
+		} else {
+			b := box.NewBox(cmd, toRun, []string{}, []string{}, s.Target, false, true, true)
+			err := b.Run()
+			if err != nil {
+				return errors.Wrap(err, "Failed running command ")
+			}
+		}
+	}
+	return nil
+}
+
+// TODO: We don't store uninstall finalizers ?!
+func (f *LuetFinalizer) RunUnInstall() error {
+	for _, c := range f.Uninstall {
+		Debug("finalizer:", "sh", "-c", c)
+		cmd := exec.Command("sh", "-c", c)
+		stdoutStderr, err := cmd.CombinedOutput()
+		if err != nil {
+			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
+		}
+		Info(string(stdoutStderr))
+	}
+	return nil
+}
+
+func NewLuetFinalizerFromYaml(data []byte) (*LuetFinalizer, error) {
+	var p LuetFinalizer
+	err := yaml.Unmarshal(data, &p)
+	if err != nil {
+		return &p, err
+	}
+	return &p, err
+}

pkg/installer/installer.go

@@ -0,0 +1,724 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+
+	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/pkg/errors"
+)
+
+type LuetInstallerOptions struct {
+	SolverOptions                                                  config.LuetSolverOptions
+	Concurrency                                                    int
+	NoDeps                                                         bool
+	OnlyDeps                                                       bool
+	Force                                                          bool
+	PreserveSystemEssentialData                                    bool
+	FullUninstall, FullCleanUninstall                              bool
+	CheckConflicts                                                 bool
+	SolverUpgrade, RemoveUnavailableOnUpgrade, UpgradeNewRevisions bool
+}
+
+type LuetInstaller struct {
+	PackageRepositories Repositories
+
+	Options LuetInstallerOptions
+}
+
+type ArtifactMatch struct {
+	Package    pkg.Package
+	Artifact   compiler.Artifact
+	Repository Repository
+}
+
+func NewLuetInstaller(opts LuetInstallerOptions) Installer {
+	return &LuetInstaller{Options: opts}
+}
+
+func (l *LuetInstaller) Upgrade(s *System) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+
+	Info(":thinking: Computing upgrade, please hang tight... :zzz:")
+	if l.Options.UpgradeNewRevisions {
+		Info(":memo: note: will consider new build revisions while upgrading")
+	}
+	Spinner(32)
+	defer SpinnerStop()
+	// First match packages against repositories by priority
+	allRepos := pkg.NewInMemoryDatabase(false)
+	syncedRepos.SyncDatabase(allRepos)
+	// compute a "big" world
+	solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+	var uninstall pkg.Packages
+	var solution solver.PackagesAssertions
+
+	if l.Options.SolverUpgrade {
+
+		uninstall, solution, err = solv.UpgradeUniverse(l.Options.RemoveUnavailableOnUpgrade)
+		if err != nil {
+			return errors.Wrap(err, "Failed solving solution for upgrade")
+		}
+	} else {
+		uninstall, solution, err = solv.Upgrade(!l.Options.FullUninstall, l.Options.NoDeps)
+		if err != nil {
+			return errors.Wrap(err, "Failed solving solution for upgrade")
+		}
+
+	}
+	SpinnerStop()
+
+	if len(uninstall) > 0 {
+		Info(":recycle: Packages marked for uninstall:")
+	}
+
+	for _, p := range uninstall {
+		Info(fmt.Sprintf("- %s", p.HumanReadableString()))
+	}
+
+	if len(solution) > 0 {
+		Info(":zap: Packages marked for upgrade:")
+	}
+
+	toInstall := pkg.Packages{}
+	for _, assertion := range solution {
+		// Be sure to filter from solutions packages already installed in the system
+		if _, err := s.Database.FindPackage(assertion.Package); err != nil && assertion.Value {
+			Info(fmt.Sprintf("- %s", assertion.Package.HumanReadableString()))
+			toInstall = append(toInstall, assertion.Package)
+		}
+	}
+
+	if l.Options.UpgradeNewRevisions {
+		Info(":mag: Checking packages with new revisions available")
+		for _, p := range s.Database.World() {
+			matches := syncedRepos.PackageMatches(pkg.Packages{p})
+			if len(matches) == 0 {
+				// Package missing. the user should run luet upgrade --universe
+				Info(":warning: Installed packages seems to be missing from remote repositories.")
+				Info(":warning: It is suggested to run 'luet upgrade --universe'")
+				continue
+			}
+			for _, artefact := range matches[0].Repo.GetIndex() {
+				if artefact.GetCompileSpec().GetPackage() == nil {
+					return errors.New("Package in compilespec empty")
+
+				}
+				if artefact.GetCompileSpec().GetPackage().Matches(p) && artefact.GetCompileSpec().GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
+					toInstall = append(toInstall, matches[0].Package).Unique()
+					uninstall = append(uninstall, p).Unique()
+					Info(
+						fmt.Sprintf("- %s ( %s vs %s ) repo: %s (date: %s)",
+							p.HumanReadableString(),
+							artefact.GetCompileSpec().GetPackage().GetBuildTimestamp(),
+							p.GetBuildTimestamp(),
+							matches[0].Repo.GetName(),
+							matches[0].Repo.GetLastUpdate(),
+						))
+				}
+			}
+		}
+	}
+
+	return l.swap(syncedRepos, uninstall, toInstall, s)
+}
+
+func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
+	Spinner(32)
+	defer SpinnerStop()
+	syncedRepos := Repositories{}
+	for _, r := range l.PackageRepositories {
+		repo, err := r.Sync(false)
+		if err != nil {
+			return nil, errors.Wrap(err, "Failed syncing repository: "+r.GetName())
+		}
+		syncedRepos = append(syncedRepos, repo)
+	}
+
+	// compute what to install and from where
+	sort.Sort(syncedRepos)
+
+	if !inMemory {
+		l.PackageRepositories = syncedRepos
+	}
+
+	return syncedRepos, nil
+}
+
+func (l *LuetInstaller) Swap(toRemove pkg.Packages, toInstall pkg.Packages, s *System) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+	return l.swap(syncedRepos, toRemove, toInstall, s)
+}
+
+func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) error {
+	// First match packages against repositories by priority
+	allRepos := pkg.NewInMemoryDatabase(false)
+	syncedRepos.SyncDatabase(allRepos)
+	toInstall = syncedRepos.ResolveSelectors(toInstall)
+
+	if err := l.download(syncedRepos, toInstall); err != nil {
+		return errors.Wrap(err, "Pre-downloading packages")
+	}
+
+	// We don't want any conflict with the installed to raise during the upgrade.
+	// In this way we both force uninstalls and we avoid to check with conflicts
+	// against the current system state which is pending to deletion
+	// E.g. you can't check for conflicts for an upgrade of a new version of A
+	// if the old A results installed in the system. This is due to the fact that
+	// now the solver enforces the constraints and explictly denies two packages
+	// of the same version installed.
+	forced := l.Options.Force
+
+	l.Options.Force = true
+
+	for _, u := range toRemove {
+		Info(":package:", u.HumanReadableString(), "Marked for deletion")
+
+		err := l.Uninstall(u, s)
+		if err != nil && !l.Options.Force {
+			Error("Failed uninstall for ", u.HumanReadableString())
+			return errors.Wrap(err, "uninstalling "+u.HumanReadableString())
+		}
+
+	}
+	l.Options.Force = forced
+
+	return l.install(syncedRepos, toInstall, s)
+}
+
+func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+	return l.install(syncedRepos, cp, s)
+}
+
+func (l *LuetInstaller) download(syncedRepos Repositories, cp pkg.Packages) error {
+	toDownload := map[string]ArtifactMatch{}
+
+	// FIXME: This can be optimized. We don't need to re-match this to the repository
+	// But we could just do it once
+
+	// Gathers things to download
+	for _, currentPack := range cp {
+		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
+		if len(matches) == 0 {
+			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
+		}
+	A:
+		for _, artefact := range matches[0].Repo.GetIndex() {
+			if artefact.GetCompileSpec().GetPackage() == nil {
+				return errors.New("Package in compilespec empty")
+
+			}
+			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+
+				toDownload[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
+
+				break A
+			}
+		}
+	}
+	// Download packages into cache in parallel.
+	all := make(chan ArtifactMatch)
+
+	var wg = new(sync.WaitGroup)
+
+	// Download
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.downloadWorker(i, wg, all)
+	}
+	for _, c := range toDownload {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
+
+	return nil
+}
+
+// Reclaim adds packages to the system database
+// if files from artifacts in the repositories are found
+// in the system target
+func (l *LuetInstaller) Reclaim(s *System) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+
+	var toMerge []ArtifactMatch = []ArtifactMatch{}
+
+	for _, repo := range syncedRepos {
+		for _, artefact := range repo.GetIndex() {
+			Debug("Checking if",
+				artefact.GetCompileSpec().GetPackage().HumanReadableString(),
+				"from", repo.GetName(), "is installed")
+		FILES:
+			for _, f := range artefact.GetFiles() {
+				if helpers.Exists(filepath.Join(s.Target, f)) {
+					p, err := repo.GetTree().GetDatabase().FindPackage(artefact.GetCompileSpec().GetPackage())
+					if err != nil {
+						return err
+					}
+					Info(":mag: Found package:", p.HumanReadableString())
+					toMerge = append(toMerge, ArtifactMatch{Artifact: artefact, Package: p})
+					break FILES
+				}
+			}
+		}
+	}
+
+	for _, match := range toMerge {
+		pack := match.Package
+		vers, _ := s.Database.FindPackageVersions(pack)
+
+		if len(vers) >= 1 {
+			Warning("Filtering out package " + pack.HumanReadableString() + ", already reclaimed")
+			continue
+		}
+		_, err := s.Database.CreatePackage(pack)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed creating package")
+		}
+		s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.GetFiles()})
+		Info(":zap: Reclaimed package:", pack.HumanReadableString())
+	}
+	Info("Done!")
+
+	return nil
+}
+
+func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *System) error {
+	var p pkg.Packages
+
+	// Check if the package is installed first
+	for _, pi := range cp {
+
+		vers, _ := s.Database.FindPackageVersions(pi)
+
+		if len(vers) >= 1 {
+			Warning("Filtering out package " + pi.HumanReadableString() + ", it has other versions already installed. Uninstall one of them first ")
+			continue
+			//return errors.New("Package " + pi.GetFingerPrint() + " has other versions already installed. Uninstall one of them first: " + strings.Join(vers, " "))
+
+		}
+		p = append(p, pi)
+	}
+
+	if len(p) == 0 {
+		Warning("No package to install, bailing out with no errors")
+		return nil
+	}
+	// First get metas from all repos (and decodes trees)
+
+	// First match packages against repositories by priority
+	//	matches := syncedRepos.PackageMatches(p)
+
+	// compute a "big" world
+	allRepos := pkg.NewInMemoryDatabase(false)
+	syncedRepos.SyncDatabase(allRepos)
+	p = syncedRepos.ResolveSelectors(p)
+	toInstall := map[string]ArtifactMatch{}
+	var packagesToInstall pkg.Packages
+	var err error
+	var solution solver.PackagesAssertions
+
+	if !l.Options.NoDeps {
+		Info(":deciduous_tree: Computing installation, hang tight")
+		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		solution, err = solv.Install(p)
+		/// TODO: PackageAssertions needs to be a map[fingerprint]pack so lookup is in O(1)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed solving solution for package")
+		}
+		Info(":deciduous_tree: Finished calculating dependencies")
+		// Gathers things to install
+		Info(":deciduous_tree: Checking for packages already installed, and prepare for installation")
+		for _, assertion := range solution {
+			if assertion.Value {
+				if _, err := s.Database.FindPackage(assertion.Package); err == nil {
+					// skip matching if it is installed already
+					continue
+				}
+				packagesToInstall = append(packagesToInstall, assertion.Package)
+			}
+		}
+	} else if !l.Options.OnlyDeps {
+		for _, currentPack := range p {
+			if _, err := s.Database.FindPackage(currentPack); err == nil {
+				// skip matching if it is installed already
+				continue
+			}
+			packagesToInstall = append(packagesToInstall, currentPack)
+		}
+	}
+	Info(":deciduous_tree: Finding packages to install from :cloud:")
+	// Gathers things to install
+	for _, currentPack := range packagesToInstall {
+		// Check if package is already installed.
+
+		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
+		if len(matches) == 0 {
+			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
+		}
+	A:
+		for _, artefact := range matches[0].Repo.GetIndex() {
+			if artefact.GetCompileSpec().GetPackage() == nil {
+				return errors.New("Package in compilespec empty")
+
+			}
+			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+				currentPack.SetBuildTimestamp(artefact.GetCompileSpec().GetPackage().GetBuildTimestamp())
+				// Filter out already installed
+				if _, err := s.Database.FindPackage(currentPack); err != nil {
+					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
+					Info("\t:package:", currentPack.HumanReadableString(), ":cloud:", matches[0].Repo.GetName())
+				}
+				break A
+			}
+		}
+	}
+	// Install packages into rootfs in parallel.
+	all := make(chan ArtifactMatch)
+
+	var wg = new(sync.WaitGroup)
+
+	// Download first
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.downloadWorker(i, wg, all)
+	}
+
+	for _, c := range toInstall {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
+
+	all = make(chan ArtifactMatch)
+
+	wg = new(sync.WaitGroup)
+
+	// Do the real install
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.installerWorker(i, wg, all, s)
+	}
+
+	for _, c := range toInstall {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
+
+	for _, c := range toInstall {
+		// Annotate to the system that the package was installed
+		_, err := s.Database.CreatePackage(c.Package)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed creating package")
+		}
+	}
+	var toFinalize []pkg.Package
+	if !l.Options.NoDeps {
+		// TODO: Lower those errors as warning
+		for _, w := range p {
+			// Finalizers needs to run in order and in sequence.
+			ordered, err := solution.Order(allRepos, w.GetFingerPrint())
+			if err != nil {
+				return errors.Wrap(err, "While order a solution for "+w.HumanReadableString())
+			}
+		ORDER:
+			for _, ass := range ordered {
+				if ass.Value {
+					installed, ok := toInstall[ass.Package.GetFingerPrint()]
+					if !ok {
+						// It was a dep already installed in the system, so we can skip it safely
+						continue ORDER
+					}
+					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
+					if err != nil {
+						return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
+					}
+
+					toFinalize = append(toFinalize, treePackage)
+				}
+			}
+
+		}
+	} else {
+		for _, c := range toInstall {
+			treePackage, err := c.Repository.GetTree().GetDatabase().FindPackage(c.Package)
+			if err != nil {
+				return errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
+			}
+			toFinalize = append(toFinalize, treePackage)
+		}
+	}
+
+	return s.ExecuteFinalizers(toFinalize, l.Options.Force)
+}
+
+func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) {
+
+	artifact, err := a.Repository.Client().DownloadArtifact(a.Artifact)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error on download artifact")
+	}
+
+	err = artifact.Verify()
+	if err != nil && !l.Options.Force {
+		return nil, errors.Wrap(err, "Artifact integrity check failure")
+	}
+	return artifact, nil
+}
+
+func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
+
+	artifact, err := l.downloadPackage(a)
+	if err != nil && !l.Options.Force {
+		return errors.Wrap(err, "Failed downloading package")
+	}
+
+	files, err := artifact.FileList()
+	if err != nil && !l.Options.Force {
+		return errors.Wrap(err, "Could not open package archive")
+	}
+
+	err = artifact.Unpack(s.Target, true)
+	if err != nil && !l.Options.Force {
+		return errors.Wrap(err, "Error met while unpacking rootfs")
+	}
+
+	// First create client and download
+	// Then unpack to system
+	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.Package.GetFingerPrint(), Files: files})
+}
+
+func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch) error {
+	defer wg.Done()
+
+	for p := range c {
+		// TODO: Keep trace of what was added from the tar, and save it into system
+		_, err := l.downloadPackage(p)
+		if err != nil && !l.Options.Force {
+			//TODO: Uninstall, rollback.
+			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
+			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
+		}
+		if err == nil {
+			Info(":package: ", p.Package.HumanReadableString(), "downloaded")
+		} else if err != nil && l.Options.Force {
+			Info(":package: ", p.Package.HumanReadableString(), "downloaded with failures (force download)")
+		}
+	}
+
+	return nil
+}
+
+func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System) error {
+	defer wg.Done()
+
+	for p := range c {
+		// TODO: Keep trace of what was added from the tar, and save it into system
+		err := l.installPackage(p, s)
+		if err != nil && !l.Options.Force {
+			//TODO: Uninstall, rollback.
+			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
+			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
+		}
+		if err == nil {
+			Info(":package: ", p.Package.HumanReadableString(), "installed")
+		} else if err != nil && l.Options.Force {
+			Info(":package: ", p.Package.HumanReadableString(), "installed with failures (force install)")
+		}
+	}
+
+	return nil
+}
+
+func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
+	var cp *config.ConfigProtect
+	annotationDir := ""
+
+	files, err := s.Database.GetPackageFiles(p)
+	if err != nil {
+		return errors.Wrap(err, "Failed getting installed files")
+	}
+
+	if !config.LuetCfg.ConfigProtectSkip {
+
+		if p.HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := p.GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
+			}
+		}
+
+		cp = config.NewConfigProtect(annotationDir)
+		cp.Map(files)
+	}
+
+	toRemove, notPresent := helpers.OrderFiles(s.Target, files)
+
+	// Remove from target
+	for _, f := range toRemove {
+		target := filepath.Join(s.Target, f)
+
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Preserving protected file:", f)
+			continue
+		}
+
+		Debug("Removing", target)
+		if l.Options.PreserveSystemEssentialData &&
+			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) ||
+			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath()) {
+			Warning("Preserve ", f, " which is required by luet ( you have to delete it manually if you really need to)")
+			continue
+		}
+
+		fi, err := os.Lstat(target)
+		if err != nil {
+			Warning("File not found (it was before?) ", err.Error())
+			continue
+		}
+		switch mode := fi.Mode(); {
+		case mode.IsDir():
+			files, err := ioutil.ReadDir(target)
+			if err != nil {
+				Warning("Failed reading folder", target, err.Error())
+			}
+			if len(files) != 0 {
+				Debug("Preserving not-empty folder", target)
+				continue
+			}
+		}
+
+		if err = os.Remove(target); err != nil {
+			Warning("Failed removing file (maybe not present in the system target anymore ?)", target, err.Error())
+		}
+	}
+
+	for _, f := range notPresent {
+		target := filepath.Join(s.Target, f)
+
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Preserving protected file:", f)
+			continue
+		}
+
+		if err = os.Remove(target); err != nil {
+			Debug("Failed removing file (not present in the system target)", target, err.Error())
+		}
+	}
+
+	err = s.Database.RemovePackageFiles(p)
+	if err != nil {
+		return errors.Wrap(err, "Failed removing package files from database")
+	}
+	err = s.Database.RemovePackage(p)
+	if err != nil {
+		return errors.Wrap(err, "Failed removing package from database")
+	}
+
+	Info(":recycle:", p.GetFingerPrint(), "Removed :heavy_check_mark:")
+	return nil
+}
+
+func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
+	Spinner(32)
+	defer SpinnerStop()
+
+	Info(":recycle: Uninstalling :package:", p.HumanReadableString(), "hang tight")
+
+	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) TODO - mark the uninstallation in db
+	// Get installed definition
+	checkConflicts := l.Options.CheckConflicts
+	full := l.Options.FullUninstall
+	if l.Options.Force == true { // IF forced, we want to remove the package and all its requires
+		checkConflicts = false
+		full = false
+	}
+
+	// Create a temporary DB with the installed packages
+	// so the solver is much faster finding the deptree
+	installedtmp := pkg.NewInMemoryDatabase(false)
+
+	for _, i := range s.Database.World() {
+		_, err := installedtmp.CreatePackage(i)
+		if err != nil {
+			return errors.Wrap(err, "Failed create temporary in-memory db")
+		}
+	}
+
+	if !l.Options.NoDeps {
+		Info(":mag: Finding :package:", p.HumanReadableString(), "dependency graph :deciduous_tree:")
+		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		var solution pkg.Packages
+		var err error
+		if l.Options.FullCleanUninstall {
+			solution, err = solv.UninstallUniverse(pkg.Packages{p})
+			if err != nil {
+				return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+			}
+		} else {
+			solution, err = solv.Uninstall(p, checkConflicts, full)
+			if err != nil && !l.Options.Force {
+				return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+			}
+		}
+
+		for _, p := range solution {
+			Info(":recycle: Uninstalling", p.HumanReadableString())
+			err := l.uninstall(p, s)
+			if err != nil && !l.Options.Force {
+				return errors.Wrap(err, "Uninstall failed")
+			}
+		}
+	} else {
+		Info(":recycle: Uninstalling", p.HumanReadableString(), "without deps")
+		err := l.uninstall(p, s)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Uninstall failed")
+		}
+		Info(":recycle: :package:", p.HumanReadableString(), "uninstalled :heavy_check_mark:")
+	}
+	return nil
+
+}
+
+func (l *LuetInstaller) Repositories(r []Repository) { l.PackageRepositories = r }

pkg/installer/installer_suite_test.go

@@ -0,0 +1,36 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestInstaller(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	// Set temporary directory for rootfs
+	config.LuetCfg.GetSystem().Rootfs = "/tmp/luet-root"
+	// Force dynamic path for packages cache
+	config.LuetCfg.GetSystem().PkgsCachePath = ""
+	RunSpecs(t, "Installer Suite")
+}

pkg/installer/interface.go

@@ -0,0 +1,75 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	compiler "github.com/mudler/luet/pkg/compiler"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	//"github.com/mudler/luet/pkg/solver"
+)
+
+type Installer interface {
+	Install(pkg.Packages, *System) error
+	Uninstall(pkg.Package, *System) error
+	Upgrade(s *System) error
+	Reclaim(s *System) error
+
+	Repositories([]Repository)
+	SyncRepositories(bool) (Repositories, error)
+	Swap(pkg.Packages, pkg.Packages, *System) error
+}
+
+type Client interface {
+	DownloadArtifact(compiler.Artifact) (compiler.Artifact, error)
+	DownloadFile(string) (string, error)
+}
+
+type Repositories []Repository
+
+type Repository interface {
+	GetName() string
+	GetDescription() string
+	GetUrls() []string
+	SetUrls([]string)
+	AddUrl(string)
+	GetPriority() int
+	GetIndex() compiler.ArtifactIndex
+	SetIndex(i compiler.ArtifactIndex)
+	GetTree() tree.Builder
+	SetTree(tree.Builder)
+	Write(path string, resetRevision bool) error
+	Sync(bool) (Repository, error)
+	GetTreePath() string
+	SetTreePath(string)
+	GetMetaPath() string
+	SetMetaPath(string)
+	GetType() string
+	SetType(string)
+	SetAuthentication(map[string]string)
+	GetAuthentication() map[string]string
+	GetRevision() int
+	IncrementRevision()
+	GetLastUpdate() string
+	SetLastUpdate(string)
+	Client() Client
+
+	SetPriority(int)
+	GetRepositoryFile(string) (LuetRepositoryFile, error)
+	SetRepositoryFile(string, LuetRepositoryFile)
+	SetName(p string)
+	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
+}

pkg/installer/repository.go

@@ -0,0 +1,867 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/installer/client"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+)
+
+const (
+	REPOSITORY_METAFILE = "repository.meta.yaml"
+	REPOSITORY_SPECFILE = "repository.yaml"
+	TREE_TARBALL        = "tree.tar"
+
+	REPOFILE_TREE_KEY = "tree"
+	REPOFILE_META_KEY = "meta"
+)
+
+type LuetRepositoryFile struct {
+	FileName        string                             `json:"filename"`
+	CompressionType compiler.CompressionImplementation `json:"compressiontype,omitempty"`
+	Checksums       compiler.Checksums                 `json:"checksums,omitempty"`
+}
+
+type LuetSystemRepository struct {
+	*config.LuetRepository
+
+	Index           compiler.ArtifactIndex        `json:"index"`
+	Tree            tree.Builder                  `json:"-"`
+	RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"`
+}
+
+type LuetSystemRepositorySerialized struct {
+	Name            string                        `json:"name"`
+	Description     string                        `json:"description,omitempty"`
+	Urls            []string                      `json:"urls"`
+	Priority        int                           `json:"priority"`
+	Type            string                        `json:"type"`
+	Revision        int                           `json:"revision,omitempty"`
+	LastUpdate      string                        `json:"last_update,omitempty"`
+	TreePath        string                        `json:"treepath"`
+	MetaPath        string                        `json:"metapath"`
+	RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"`
+}
+
+type LuetSystemRepositoryMetadata struct {
+	Index []*compiler.PackageArtifact `json:"index,omitempty"`
+}
+
+type LuetSearchModeType string
+
+const (
+	SLabel      LuetSearchModeType = "label"
+	SRegexPkg   LuetSearchModeType = "regexPkg"
+	SRegexLabel LuetSearchModeType = "regexLabel"
+)
+
+type LuetSearchOpts struct {
+	Pattern string
+	Mode    LuetSearchModeType
+}
+
+func NewLuetSystemRepositoryMetadata(file string, removeFile bool) (*LuetSystemRepositoryMetadata, error) {
+	ans := &LuetSystemRepositoryMetadata{}
+	err := ans.ReadFile(file, removeFile)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}
+
+func (m *LuetSystemRepositoryMetadata) WriteFile(path string) error {
+	data, err := yaml.Marshal(m)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(path, data, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *LuetSystemRepositoryMetadata) ReadFile(file string, removeFile bool) error {
+	if file == "" {
+		return errors.New("Invalid path for repository metadata")
+	}
+
+	dat, err := ioutil.ReadFile(file)
+	if err != nil {
+		return err
+	}
+	if removeFile {
+		defer os.Remove(file)
+	}
+
+	err = yaml.Unmarshal(dat, m)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *LuetSystemRepositoryMetadata) ToArtifactIndex() (ans compiler.ArtifactIndex) {
+	for _, a := range m.Index {
+		ans = append(ans, a)
+	}
+	return
+}
+
+func NewDefaultTreeRepositoryFile() LuetRepositoryFile {
+	return LuetRepositoryFile{
+		FileName:        TREE_TARBALL,
+		CompressionType: compiler.GZip,
+	}
+}
+
+func NewDefaultMetaRepositoryFile() LuetRepositoryFile {
+	return LuetRepositoryFile{
+		FileName:        REPOSITORY_METAFILE + ".tar",
+		CompressionType: compiler.None,
+	}
+}
+
+func (f *LuetRepositoryFile) SetFileName(n string) {
+	f.FileName = n
+}
+
+func (f *LuetRepositoryFile) GetFileName() string {
+	return f.FileName
+}
+func (f *LuetRepositoryFile) SetCompressionType(c compiler.CompressionImplementation) {
+	f.CompressionType = c
+}
+func (f *LuetRepositoryFile) GetCompressionType() compiler.CompressionImplementation {
+	return f.CompressionType
+}
+func (f *LuetRepositoryFile) SetChecksums(c compiler.Checksums) {
+	f.Checksums = c
+}
+func (f *LuetRepositoryFile) GetChecksums() compiler.Checksums {
+	return f.Checksums
+}
+
+func GenerateRepository(name, descr, t string, urls []string, priority int, src string, treesDir []string, db pkg.PackageDatabase) (Repository, error) {
+
+	tr := tree.NewInstallerRecipe(db)
+
+	for _, treeDir := range treesDir {
+		err := tr.Load(treeDir)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	art, err := buildPackageIndex(src, tr.GetDatabase())
+	if err != nil {
+		return nil, err
+	}
+
+	return NewLuetSystemRepository(
+		config.NewLuetRepository(name, t, descr, urls, priority, true, false),
+		art, tr), nil
+}
+
+func NewSystemRepository(repo config.LuetRepository) Repository {
+	return &LuetSystemRepository{
+		LuetRepository:  &repo,
+		RepositoryFiles: map[string]LuetRepositoryFile{},
+	}
+}
+
+func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder) Repository {
+	return &LuetSystemRepository{
+		LuetRepository:  repo,
+		Index:           art,
+		Tree:            builder,
+		RepositoryFiles: map[string]LuetRepositoryFile{},
+	}
+}
+
+func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, error) {
+	var p *LuetSystemRepositorySerialized
+	err := yaml.Unmarshal(data, &p)
+	if err != nil {
+		return nil, err
+	}
+
+	r := &LuetSystemRepository{
+		LuetRepository: config.NewLuetRepository(
+			p.Name,
+			p.Type,
+			p.Description,
+			p.Urls,
+			p.Priority,
+			true,
+			false,
+		),
+		RepositoryFiles: p.RepositoryFiles,
+	}
+	if p.Revision > 0 {
+		r.Revision = p.Revision
+	}
+	if p.LastUpdate != "" {
+		r.LastUpdate = p.LastUpdate
+	}
+	r.Tree = tree.NewInstallerRecipe(db)
+
+	return r, err
+}
+
+func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) {
+
+	var art []compiler.Artifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil // Skip with no errors
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		artifact, err := compiler.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
+			Info(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				artifact.GetCompileSpec().GetPackage().HumanReadableString()))
+			return nil
+		}
+
+		art = append(art, artifact)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
+func (r *LuetSystemRepository) SetPriority(n int) {
+	r.LuetRepository.Priority = n
+}
+
+func (r *LuetSystemRepository) GetName() string {
+	return r.LuetRepository.Name
+}
+func (r *LuetSystemRepository) GetDescription() string {
+	return r.LuetRepository.Description
+}
+
+func (r *LuetSystemRepository) GetAuthentication() map[string]string {
+	return r.LuetRepository.Authentication
+}
+
+func (r *LuetSystemRepository) GetType() string {
+	return r.LuetRepository.Type
+}
+func (r *LuetSystemRepository) SetType(p string) {
+	r.LuetRepository.Type = p
+}
+
+func (r *LuetSystemRepository) SetName(p string) {
+	r.LuetRepository.Name = p
+}
+
+func (r *LuetSystemRepository) AddUrl(p string) {
+	r.LuetRepository.Urls = append(r.LuetRepository.Urls, p)
+}
+func (r *LuetSystemRepository) GetUrls() []string {
+	return r.LuetRepository.Urls
+}
+func (r *LuetSystemRepository) SetUrls(urls []string) {
+	r.LuetRepository.Urls = urls
+}
+func (r *LuetSystemRepository) GetPriority() int {
+	return r.LuetRepository.Priority
+}
+func (r *LuetSystemRepository) GetTreePath() string {
+	return r.TreePath
+}
+func (r *LuetSystemRepository) SetTreePath(p string) {
+	r.TreePath = p
+}
+func (r *LuetSystemRepository) GetMetaPath() string {
+	return r.MetaPath
+}
+func (r *LuetSystemRepository) SetMetaPath(p string) {
+	r.MetaPath = p
+}
+func (r *LuetSystemRepository) SetTree(b tree.Builder) {
+	r.Tree = b
+}
+func (r *LuetSystemRepository) GetIndex() compiler.ArtifactIndex {
+	return r.Index
+}
+func (r *LuetSystemRepository) SetIndex(i compiler.ArtifactIndex) {
+	r.Index = i
+}
+func (r *LuetSystemRepository) GetTree() tree.Builder {
+	return r.Tree
+}
+func (r *LuetSystemRepository) GetRevision() int {
+	return r.LuetRepository.Revision
+}
+func (r *LuetSystemRepository) GetLastUpdate() string {
+	return r.LuetRepository.LastUpdate
+}
+func (r *LuetSystemRepository) SetLastUpdate(u string) {
+	r.LuetRepository.LastUpdate = u
+}
+func (r *LuetSystemRepository) IncrementRevision() {
+	r.LuetRepository.Revision++
+}
+func (r *LuetSystemRepository) SetAuthentication(auth map[string]string) {
+	r.LuetRepository.Authentication = auth
+}
+func (r *LuetSystemRepository) GetRepositoryFile(name string) (LuetRepositoryFile, error) {
+	ans, ok := r.RepositoryFiles[name]
+	if ok {
+		return ans, nil
+	}
+	return ans, errors.New("Repository file " + name + " not found!")
+}
+func (r *LuetSystemRepository) SetRepositoryFile(name string, f LuetRepositoryFile) {
+	r.RepositoryFiles[name] = f
+}
+
+func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repository, error) {
+	dat, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading file "+file)
+	}
+	if removeFile {
+		defer os.Remove(file)
+	}
+
+	var repo Repository
+	repo, err = NewLuetSystemRepositoryFromYaml(dat, pkg.NewInMemoryDatabase(false))
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading repository from file "+file)
+	}
+
+	// Check if mandatory key are present
+	_, err = repo.GetRepositoryFile(REPOFILE_TREE_KEY)
+	if err != nil {
+		return nil, errors.New("Invalid repository without the " + REPOFILE_TREE_KEY + " key file.")
+	}
+	_, err = repo.GetRepositoryFile(REPOFILE_META_KEY)
+	if err != nil {
+		return nil, errors.New("Invalid repository without the " + REPOFILE_META_KEY + " key file.")
+	}
+
+	return repo, err
+}
+
+func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
+	err := os.MkdirAll(dst, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
+
+	repospec := filepath.Join(dst, REPOSITORY_SPECFILE)
+	if resetRevision {
+		r.Revision = 0
+	} else {
+		if _, err := os.Stat(repospec); !os.IsNotExist(err) {
+			// Read existing file for retrieve revision
+			spec, err := r.ReadSpecFile(repospec, false)
+			if err != nil {
+				return err
+			}
+			r.Revision = spec.GetRevision()
+		}
+	}
+	r.Revision++
+
+	Info(fmt.Sprintf(
+		"For repository %s creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
+	// Create tree and repository file
+	archive, err := config.LuetCfg.GetSystem().TempDir("archive")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for archive")
+	}
+	defer os.RemoveAll(archive) // clean up
+	err = r.GetTree().Save(archive)
+	if err != nil {
+		return errors.Wrap(err, "Error met while saving the tree")
+	}
+
+	treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY)
+	if err != nil {
+		treeFile = NewDefaultTreeRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
+	}
+
+	a := compiler.NewPackageArtifact(filepath.Join(dst, treeFile.GetFileName()))
+	a.SetCompressionType(treeFile.GetCompressionType())
+	err = a.Compress(archive, 1)
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating package archive")
+	}
+
+	// Update the tree name with the name created by compression selected.
+	treeFile.SetFileName(path.Base(a.GetPath()))
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	treeFile.SetChecksums(a.GetChecksums())
+	r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
+
+	// Create Metadata struct and serialized repository
+	meta, serialized := r.Serialize()
+
+	// Create metadata file and repository file
+	metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	defer os.RemoveAll(metaTmpDir) // clean up
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+
+	metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY)
+	if err != nil {
+		metaFile = NewDefaultMetaRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	}
+
+	repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE)
+	// Create repository.meta.yaml file
+	err = meta.WriteFile(repoMetaSpec)
+	if err != nil {
+		return err
+	}
+
+	a = compiler.NewPackageArtifact(filepath.Join(dst, metaFile.GetFileName()))
+	a.SetCompressionType(metaFile.GetCompressionType())
+	err = a.Compress(metaTmpDir, 1)
+	if err != nil {
+		return errors.Wrap(err, "Error met while archiving repository metadata")
+	}
+
+	metaFile.SetFileName(path.Base(a.GetPath()))
+	r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for metadata")
+	}
+	metaFile.SetChecksums(a.GetChecksums())
+
+	data, err := yaml.Marshal(serialized)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(repospec, data, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *LuetSystemRepository) Client() Client {
+	switch r.GetType() {
+	case "disk":
+		return client.NewLocalClient(client.RepoData{Urls: r.GetUrls()})
+	case "http":
+		return client.NewHttpClient(
+			client.RepoData{
+				Urls:           r.GetUrls(),
+				Authentication: r.GetAuthentication(),
+			})
+	}
+
+	return nil
+}
+func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
+	var repoUpdated bool = false
+	var treefs, metafs string
+	aurora := GetAurora()
+
+	Debug("Sync of the repository", r.Name, "in progress...")
+	c := r.Client()
+	if c == nil {
+		return nil, errors.New("No client could be generated from repository.")
+	}
+
+	// Retrieve remote repository.yaml for retrieve revision and date
+	file, err := c.DownloadFile(REPOSITORY_SPECFILE)
+	if err != nil {
+		return nil, errors.Wrap(err, "While downloading "+REPOSITORY_SPECFILE)
+	}
+
+	repobasedir := config.LuetCfg.GetSystem().GetRepoDatabaseDirPath(r.GetName())
+	repo, err := r.ReadSpecFile(file, false)
+	if err != nil {
+		return nil, err
+	}
+	// Remove temporary file that contains repository.html.
+	// Example: /tmp/HttpClient236052003
+	defer os.RemoveAll(file)
+
+	if r.Cached {
+		if !force {
+			localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir, REPOSITORY_SPECFILE), false)
+			if localRepo != nil {
+				if localRepo.GetRevision() == repo.GetRevision() &&
+					localRepo.GetLastUpdate() == repo.GetLastUpdate() {
+					repoUpdated = true
+				}
+			}
+		}
+		if r.GetTreePath() == "" {
+			treefs = filepath.Join(repobasedir, "treefs")
+		} else {
+			treefs = r.GetTreePath()
+		}
+		if r.GetMetaPath() == "" {
+			metafs = filepath.Join(repobasedir, "metafs")
+		} else {
+			metafs = r.GetMetaPath()
+		}
+
+	} else {
+		treefs, err = config.LuetCfg.GetSystem().TempDir("treefs")
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+		// Note: If we always remove them, later on, no other structure can access
+		// to the tree for e.g. to retrieve finalizers
+
+		metafs, err = config.LuetCfg.GetSystem().TempDir("metafs")
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met whilte creating tempdir for metafs")
+		}
+		//defer os.RemoveAll(metafs)
+
+	}
+
+	// POST: treeFile and metaFile are present. I check this inside
+	// ReadSpecFile and NewLuetSystemRepositoryFromYaml
+	treeFile, _ := repo.GetRepositoryFile(REPOFILE_TREE_KEY)
+	metaFile, _ := repo.GetRepositoryFile(REPOFILE_META_KEY)
+
+	if !repoUpdated {
+
+		// Get Tree
+		a := compiler.NewPackageArtifact(treeFile.GetFileName())
+		artifactTree, err := c.DownloadArtifact(a)
+		if err != nil {
+			return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName())
+		}
+		defer os.Remove(artifactTree.GetPath())
+
+		artifactTree.SetChecksums(treeFile.GetChecksums())
+		artifactTree.SetCompressionType(treeFile.GetCompressionType())
+
+		err = artifactTree.Verify()
+		if err != nil {
+			return nil, errors.Wrap(err, "Tree integrity check failure")
+		}
+
+		Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
+
+		// Get Repository Metadata
+		a = compiler.NewPackageArtifact(metaFile.GetFileName())
+		artifactMeta, err := c.DownloadArtifact(a)
+		if err != nil {
+			return nil, errors.Wrap(err, "While downloading "+metaFile.GetFileName())
+		}
+		defer os.Remove(artifactMeta.GetPath())
+
+		artifactMeta.SetChecksums(metaFile.GetChecksums())
+		artifactMeta.SetCompressionType(metaFile.GetCompressionType())
+
+		err = artifactMeta.Verify()
+		if err != nil {
+			return nil, errors.Wrap(err, "Metadata integrity check failure")
+		}
+
+		Debug("Metadata tarball for the repository " + r.GetName() + " downloaded correctly.")
+
+		if r.Cached {
+			// Copy updated repository.yaml file to repo dir now that the tree is synced.
+			err = helpers.CopyFile(file, filepath.Join(repobasedir, REPOSITORY_SPECFILE))
+			if err != nil {
+				return nil, errors.Wrap(err, "Error on update "+REPOSITORY_SPECFILE)
+			}
+			// Remove previous tree
+			os.RemoveAll(treefs)
+			// Remove previous meta dir
+			os.RemoveAll(metafs)
+		}
+		Debug("Decompress tree of the repository " + r.Name + "...")
+
+		err = artifactTree.Unpack(treefs, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while unpacking tree")
+		}
+
+		// FIXME: It seems that tar with only one file doesn't create destination
+		//       directory. I create directory directly for now.
+		os.MkdirAll(metafs, os.ModePerm)
+		err = artifactMeta.Unpack(metafs, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while unpacking metadata")
+		}
+
+		tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64)
+
+		InfoC(
+			aurora.Bold(
+				aurora.Red(":house: Repository "+repo.GetName()+" revision: ")).String() +
+				aurora.Bold(aurora.Green(repo.GetRevision())).String() + " - " +
+				aurora.Bold(aurora.Green(time.Unix(tsec, 0).String())).String(),
+		)
+
+	} else {
+		Info("Repository", repo.GetName(), "is already up to date.")
+	}
+
+	meta, err := NewLuetSystemRepositoryMetadata(
+		filepath.Join(metafs, REPOSITORY_METAFILE), false,
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "While processing "+REPOSITORY_METAFILE)
+	}
+	repo.SetIndex(meta.ToArtifactIndex())
+
+	reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
+	err = reciper.Load(treefs)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while unpacking rootfs")
+	}
+
+	repo.SetTree(reciper)
+	repo.SetTreePath(treefs)
+
+	// Copy the local available data to the one which was synced
+	// e.g. locally we can override the type (disk), or priority
+	// while remotely it could be advertized differently
+	repo.SetUrls(r.GetUrls())
+	repo.SetAuthentication(r.GetAuthentication())
+	repo.SetType(r.GetType())
+	repo.SetPriority(r.GetPriority())
+	repo.SetName(r.GetName())
+	InfoC(
+		aurora.Bold(
+			aurora.Yellow(":information_source: Repository "+repo.GetName()+" priority: ")).String() +
+			aurora.Bold(aurora.Green(repo.GetPriority())).String() + " - type " +
+			aurora.Bold(aurora.Green(repo.GetType())).String(),
+	)
+	return repo, nil
+}
+
+func (r *LuetSystemRepository) Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized) {
+
+	serialized := LuetSystemRepositorySerialized{
+		Name:            r.Name,
+		Description:     r.Description,
+		Urls:            r.Urls,
+		Priority:        r.Priority,
+		Type:            r.Type,
+		Revision:        r.Revision,
+		LastUpdate:      r.LastUpdate,
+		RepositoryFiles: r.RepositoryFiles,
+	}
+
+	// Check if is needed set the index or simply use
+	// value returned by CleanPath
+	r.Index = r.Index.CleanPath()
+
+	meta := &LuetSystemRepositoryMetadata{
+		Index: []*compiler.PackageArtifact{},
+	}
+	for _, a := range r.Index {
+		art := a.(*compiler.PackageArtifact)
+		meta.Index = append(meta.Index, art)
+	}
+
+	return meta, serialized
+}
+
+func (r Repositories) Len() int      { return len(r) }
+func (r Repositories) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
+func (r Repositories) Less(i, j int) bool {
+	return r[i].GetPriority() < r[j].GetPriority()
+}
+
+func (r Repositories) World() pkg.Packages {
+	cache := map[string]pkg.Package{}
+	world := pkg.Packages{}
+
+	// Get Uniques. Walk in reverse so the definitions of most prio-repo overwrites lower ones
+	// In this way, when we will walk again later the deps sorting them by most higher prio we have better chance of success.
+	for i := len(r) - 1; i >= 0; i-- {
+		for _, p := range r[i].GetTree().GetDatabase().World() {
+			cache[p.GetFingerPrint()] = p
+		}
+	}
+
+	for _, v := range cache {
+		world = append(world, v)
+	}
+
+	return world
+}
+
+func (r Repositories) SyncDatabase(d pkg.PackageDatabase) {
+	cache := map[string]bool{}
+
+	// Get Uniques. Walk in reverse so the definitions of most prio-repo overwrites lower ones
+	// In this way, when we will walk again later the deps sorting them by most higher prio we have better chance of success.
+	for i := len(r) - 1; i >= 0; i-- {
+		for _, p := range r[i].GetTree().GetDatabase().World() {
+			if _, ok := cache[p.GetFingerPrint()]; !ok {
+				cache[p.GetFingerPrint()] = true
+				d.CreatePackage(p)
+			}
+		}
+	}
+
+}
+
+type PackageMatch struct {
+	Repo    Repository
+	Package pkg.Package
+}
+
+func (re Repositories) PackageMatches(p pkg.Packages) []PackageMatch {
+	// TODO: Better heuristic. here we pick the first repo that contains the atom, sorted by priority but
+	// we should do a permutations and get the best match, and in case there are more solutions the user should be able to pick
+	sort.Sort(re)
+
+	var matches []PackageMatch
+PACKAGE:
+	for _, pack := range p {
+		for _, r := range re {
+			c, err := r.GetTree().GetDatabase().FindPackage(pack)
+			if err == nil {
+				matches = append(matches, PackageMatch{Package: c, Repo: r})
+				continue PACKAGE
+			}
+		}
+	}
+
+	return matches
+
+}
+
+func (re Repositories) ResolveSelectors(p pkg.Packages) pkg.Packages {
+	// If a selector is given, get the best from each repo
+	sort.Sort(re) // respect prio
+	var matches pkg.Packages
+PACKAGE:
+	for _, pack := range p {
+	REPOSITORY:
+		for _, r := range re {
+			if pack.IsSelector() {
+				c, err := r.GetTree().GetDatabase().FindPackageCandidate(pack)
+				// If FindPackageCandidate returns the same package, it means it couldn't find one.
+				// Skip this repository and keep looking.
+				if c.String() == pack.String() {
+					continue REPOSITORY
+				}
+				if err == nil {
+					matches = append(matches, c)
+					continue PACKAGE
+				}
+			} else {
+				// If it's not a selector, just append it
+				matches = append(matches, pack)
+			}
+		}
+	}
+
+	return matches
+
+}
+
+func (re Repositories) SearchPackages(p string, o LuetSearchOpts) []PackageMatch {
+	sort.Sort(re)
+	var matches []PackageMatch
+	var err error
+
+	for _, r := range re {
+		var repoMatches pkg.Packages
+
+		switch o.Mode {
+		case SRegexPkg:
+			repoMatches, err = r.GetTree().GetDatabase().FindPackageMatch(p)
+		case SLabel:
+			repoMatches, err = r.GetTree().GetDatabase().FindPackageLabel(p)
+		case SRegexLabel:
+			repoMatches, err = r.GetTree().GetDatabase().FindPackageLabelMatch(p)
+		}
+
+		if err == nil && len(repoMatches) > 0 {
+			for _, pack := range repoMatches {
+				matches = append(matches, PackageMatch{Package: pack, Repo: r})
+			}
+		}
+	}
+
+	return matches
+}
+
+func (re Repositories) SearchLabelMatch(s string) []PackageMatch {
+	return re.SearchPackages(s, LuetSearchOpts{Pattern: s, Mode: SRegexLabel})
+}
+
+func (re Repositories) SearchLabel(s string) []PackageMatch {
+	return re.SearchPackages(s, LuetSearchOpts{Pattern: s, Mode: SLabel})
+}
+
+func (re Repositories) Search(s string) []PackageMatch {
+	return re.SearchPackages(s, LuetSearchOpts{Pattern: s, Mode: SRegexPkg})
+}

pkg/installer/repository_test.go

@@ -0,0 +1,221 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer_test
+
+import (
+
+	//	. "github.com/mudler/luet/pkg/installer"
+
+	"io/ioutil"
+	"os"
+
+	"github.com/mudler/luet/pkg/compiler"
+	backend "github.com/mudler/luet/pkg/compiler/backend"
+	config "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/installer"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	"github.com/mudler/luet/pkg/tree"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Repository", func() {
+	Context("Generation", func() {
+		It("Generate repository metadata", func() {
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			content1, err := helpers.Read(spec.Rel("test5"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := helpers.Read(spec.Rel("test6"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("artifact5\n"))
+			Expect(content2).To(Equal("artifact6\n"))
+
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+		})
+
+		It("Generate repository metadata of files ONLY referenced in a tree", func() {
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			generalRecipe2 := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe2.Load("../../tests/fixtures/finalizers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+			Expect(spec2.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			compiler2.SetConcurrency(1)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			artifact2, err := compiler2.Compile(false, spec2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact2.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact2.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			content1, err := helpers.Read(spec.Rel("test5"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := helpers.Read(spec.Rel("test6"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("artifact5\n"))
+			Expect(content2).To(Equal("artifact6\n"))
+
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+
+			// We check now that the artifact not referenced in the tree
+			// (spec2) is not indexed in the repository
+			repository, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			repos, err := repository.Sync(true)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = repos.GetTree().GetDatabase().FindPackage(spec.GetPackage())
+			Expect(err).ToNot(HaveOccurred())
+			_, err = repos.GetTree().GetDatabase().FindPackage(spec2.GetPackage())
+			Expect(err).To(HaveOccurred()) // should throw error
+		})
+	})
+	Context("Matching packages", func() {
+		It("Matches packages in different repositories by priority", func() {
+			package1 := &pkg.DefaultPackage{Name: "Test"}
+			package2 := &pkg.DefaultPackage{Name: "Test2"}
+			builder1 := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
+			builder2 := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
+
+			_, err := builder1.GetDatabase().CreatePackage(package1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = builder2.GetDatabase().CreatePackage(package2)
+			Expect(err).ToNot(HaveOccurred())
+			repo1 := &LuetSystemRepository{LuetRepository: &config.LuetRepository{Name: "test1"}, Tree: builder1}
+			repo2 := &LuetSystemRepository{LuetRepository: &config.LuetRepository{Name: "test2"}, Tree: builder2}
+			repositories := Repositories{repo1, repo2}
+			matches := repositories.PackageMatches([]pkg.Package{package1})
+			Expect(matches).To(Equal([]PackageMatch{{Repo: repo1, Package: package1}}))
+
+		})
+
+	})
+})

pkg/installer/system.go

@@ -0,0 +1,47 @@
+package installer
+
+import (
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/mudler/luet/pkg/helpers"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	"github.com/pkg/errors"
+)
+
+type System struct {
+	Database pkg.PackageDatabase
+	Target   string
+}
+
+func (s *System) World() (pkg.Packages, error) {
+	return s.Database.World(), nil
+}
+
+type templatedata map[string]interface{}
+
+func (s *System) ExecuteFinalizers(packs []pkg.Package, force bool) error {
+	executedFinalizer := map[string]bool{}
+	for _, p := range packs {
+		if helpers.Exists(p.Rel(tree.FinalizerFile)) {
+			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile))
+			if err != nil && !force {
+				return errors.Wrap(err, "reading file "+p.Rel(tree.FinalizerFile))
+			}
+
+			if _, exists := executedFinalizer[p.GetFingerPrint()]; !exists {
+				Info("Executing finalizer for " + p.HumanReadableString())
+				finalizer, err := NewLuetFinalizerFromYaml([]byte(out))
+				if err != nil && !force {
+					return errors.Wrap(err, "Error reading finalizer "+p.Rel(tree.FinalizerFile))
+				}
+				err = finalizer.RunInstall(s)
+				if err != nil && !force {
+					return errors.Wrap(err, "Error executing install finalizer "+p.Rel(tree.FinalizerFile))
+				}
+				executedFinalizer[p.GetFingerPrint()] = true
+			}
+		}
+	}
+	return nil
+}

pkg/logger/logger.go

@@ -3,98 +3,231 @@ package logger
 import (
 	"fmt"
 	"os"
-	"time"
+	"regexp"
+
+	. "github.com/mudler/luet/pkg/config"
 
 	"github.com/briandowns/spinner"
 	"github.com/kyokomi/emoji"
 	. "github.com/logrusorgru/aurora"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
-var s *spinner.Spinner = spinner.New(spinner.CharSets[22], 100*time.Millisecond)
+var s *spinner.Spinner = nil
+var z *zap.Logger = nil
+var aurora Aurora = nil
 
-// TODO: handle this from configuration
-var debug = false
+func NewSpinner() {
+	if s == nil {
+		s = spinner.New(
+			spinner.CharSets[LuetCfg.GetGeneral().SpinnerCharset],
+			LuetCfg.GetGeneral().GetSpinnerMs())
+	}
+}
+
+func InitAurora() {
+	if aurora == nil {
+		aurora = NewAurora(LuetCfg.GetLogging().Color)
+	}
+}
+
+func GetAurora() Aurora {
+	return aurora
+}
+
+func ZapLogger() error {
+	var err error
+	if z == nil {
+		// TODO: test permission for open logfile.
+		cfg := zap.NewProductionConfig()
+		cfg.OutputPaths = []string{LuetCfg.GetLogging().Path}
+		cfg.Level = level2AtomicLevel(LuetCfg.GetLogging().Level)
+		cfg.ErrorOutputPaths = []string{}
+		if LuetCfg.GetLogging().JsonFormat {
+			cfg.Encoding = "json"
+		} else {
+			cfg.Encoding = "console"
+		}
+		cfg.DisableCaller = true
+		cfg.DisableStacktrace = true
+		cfg.EncoderConfig.TimeKey = "time"
+		cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+
+		z, err = cfg.Build()
+		if err != nil {
+			fmt.Fprint(os.Stderr, "Error on initialize file logger: "+err.Error()+"\n")
+			return err
+		}
+	}
+
+	return nil
+}
 
 func Spinner(i int) {
-
+	var confLevel int
+	if LuetCfg.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = level2Number(LuetCfg.GetLogging().Level)
+	}
+	if 2 > confLevel {
+		return
+	}
 	if i > 43 {
 		i = 43
 	}
 
-	if !debug && !s.Active() {
+	if s != nil && !s.Active() {
 		//	s.UpdateCharSet(spinner.CharSets[i])
 		s.Start() // Start the spinner
 	}
 }
 
 func SpinnerText(suffix, prefix string) {
-	s.Lock()
-	defer s.Unlock()
-	if debug {
-		fmt.Println(fmt.Sprintf("%s %s",
-			Bold(Cyan(prefix)).String(),
-			Bold(Magenta(suffix)).BgBlack().String(),
-		))
-	} else {
-		s.Suffix = Bold(Magenta(suffix)).BgBlack().String()
-		s.Prefix = Bold(Cyan(prefix)).String()
+	if s != nil {
+		s.Lock()
+		defer s.Unlock()
+		if LuetCfg.GetGeneral().Debug {
+			fmt.Println(fmt.Sprintf("%s %s",
+				Bold(Cyan(prefix)).String(),
+				Bold(Magenta(suffix)).BgBlack().String(),
+			))
+		} else {
+			s.Suffix = Bold(Magenta(suffix)).BgBlack().String()
+			s.Prefix = Bold(Cyan(prefix)).String()
+		}
 	}
 }
 
 func SpinnerStop() {
-	if !debug {
+	var confLevel int
+	if LuetCfg.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = level2Number(LuetCfg.GetLogging().Level)
+	}
+	if 2 > confLevel {
+		return
+	}
+	if s != nil {
 		s.Stop()
 	}
 }
 
-func msg(level string, msg ...interface{}) {
+func level2Number(level string) int {
+	switch level {
+	case "error":
+		return 0
+	case "warning":
+		return 1
+	case "info":
+		return 2
+	default:
+		return 3
+	}
+}
+
+func log2File(level, msg string) {
+	switch level {
+	case "error":
+		z.Error(msg)
+	case "warning":
+		z.Warn(msg)
+	case "info":
+		z.Info(msg)
+	default:
+		z.Debug(msg)
+	}
+}
+
+func level2AtomicLevel(level string) zap.AtomicLevel {
+	switch level {
+	case "error":
+		return zap.NewAtomicLevelAt(zap.ErrorLevel)
+	case "warning":
+		return zap.NewAtomicLevelAt(zap.WarnLevel)
+	case "info":
+		return zap.NewAtomicLevelAt(zap.InfoLevel)
+	default:
+		return zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+}
+
+func msg(level string, withoutColor bool, msg ...interface{}) {
 	var message string
+	var confLevel, msgLevel int
+
+	if LuetCfg.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = level2Number(LuetCfg.GetLogging().Level)
+	}
+	msgLevel = level2Number(level)
+	if msgLevel > confLevel {
+		return
+	}
+
 	for _, m := range msg {
 		message += " " + fmt.Sprintf("%v", m)
 	}
 
 	var levelMsg string
-	switch level {
-	case "warning":
-		levelMsg = Bold(Yellow(":construction: " + message)).BgBlack().String()
-	case "debug":
-		levelMsg = White(message).BgBlack().String()
-	case "info":
-		levelMsg = Bold(White(message)).BgBlack().String()
-	case "error":
-		levelMsg = Bold(Red(":bomb: " + message + ":fire:")).BgBlack().String()
+
+	if withoutColor || !LuetCfg.GetLogging().Color {
+		levelMsg = message
+	} else {
+		switch level {
+		case "warning":
+			levelMsg = Bold(Yellow(":construction: " + message)).BgBlack().String()
+		case "debug":
+			levelMsg = White(message).BgBlack().String()
+		case "info":
+			levelMsg = Bold(White(message)).BgBlack().String()
+		case "error":
+			levelMsg = Bold(Red(":bomb: " + message + ":fire:")).BgBlack().String()
+		}
 	}
 
-	levelMsg = emoji.Sprint(levelMsg)
+	if LuetCfg.GetLogging().EnableEmoji {
+		levelMsg = emoji.Sprint(levelMsg)
+	} else {
+		re := regexp.MustCompile(`[:][\w]+[:]`)
+		levelMsg = re.ReplaceAllString(levelMsg, "")
+	}
 
-	//if s.Active() {
-	//		SpinnerText(levelMsg, "")
-	//	return
-	//	}
-
-	cmd := []interface{}{}
-	for _, f := range msg {
-		cmd = append(cmd, f)
+	if z != nil {
+		log2File(level, message)
 	}
 
 	fmt.Println(levelMsg)
-	//fmt.Println(cmd...)
 }
 
 func Warning(mess ...interface{}) {
-	msg("warning", mess...)
+	msg("warning", false, mess...)
+	if LuetCfg.GetGeneral().FatalWarns {
+		os.Exit(2)
+	}
 }
 
 func Debug(mess ...interface{}) {
-	msg("debug", mess...)
+	msg("debug", false, mess...)
+}
+
+func DebugC(mess ...interface{}) {
+	msg("debug", true, mess...)
 }
 
 func Info(mess ...interface{}) {
-	msg("info", mess...)
+	msg("info", false, mess...)
+}
+
+func InfoC(mess ...interface{}) {
+	msg("info", true, mess...)
 }
 
 func Error(mess ...interface{}) {
-	msg("error", mess...)
+	msg("error", false, mess...)
 }
 
 func Fatal(mess ...interface{}) {

pkg/package/annotations.go

@@ -0,0 +1,23 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+type AnnotationKey string
+
+const (
+	ConfigProtectAnnnotation AnnotationKey = "config_protect"
+)

pkg/package/database.go

@@ -23,6 +23,35 @@ type PackageDatabase interface {
 	Get(s string) (string, error)
 	Set(k, v string) error
 
-	Create([]byte) (string, error)
+	Create(string, []byte) (string, error)
 	Retrieve(ID string) ([]byte, error)
 }
+
+type PackageSet interface {
+	GetPackages() []string //Ids
+	CreatePackage(pkg Package) (string, error)
+	GetPackage(ID string) (Package, error)
+	Clean() error
+	FindPackage(Package) (Package, error)
+	FindPackages(p Package) (Packages, error)
+	UpdatePackage(p Package) error
+	GetAllPackages(packages chan Package) error
+	RemovePackage(Package) error
+
+	GetPackageFiles(Package) ([]string, error)
+	SetPackageFiles(*PackageFile) error
+	RemovePackageFiles(Package) error
+	FindPackageVersions(p Package) (Packages, error)
+	World() Packages
+
+	FindPackageCandidate(p Package) (Package, error)
+	FindPackageLabel(labelKey string) (Packages, error)
+	FindPackageLabelMatch(pattern string) (Packages, error)
+	FindPackageMatch(pattern string) (Packages, error)
+}
+
+type PackageFile struct {
+	ID                 int `storm:"id,increment"` // primary key with auto increment
+	PackageFingerprint string
+	Files              []string
+}

pkg/package/database_bench_test.go

@@ -0,0 +1,116 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	"io/ioutil"
+	"os"
+	"strconv"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Database  benchmark", func() {
+
+	Context("BoltDB", func() {
+
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		var db PackageSet
+
+		BeforeEach(func() {
+
+			tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			db = NewBoltDatabase(tmpfile.Name())
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+
+		Measure("it should be fast in computing world from a 50000 dataset", func(b Benchmarker) {
+			for i := 0; i < 50000; i++ {
+				a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			runtime := b.Time("runtime", func() {
+				packs := db.World()
+				Expect(len(packs)).To(Equal(50000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 30), "World() shouldn't take too long.")
+
+		}, 1)
+
+		Measure("it should be fast in computing world from a 100000 dataset", func(b Benchmarker) {
+			for i := 0; i < 100000; i++ {
+				a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			runtime := b.Time("runtime", func() {
+				packs := db.World()
+				Expect(len(packs)).To(Equal(100000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 30), "World() shouldn't take too long.")
+
+		}, 1)
+	})
+
+	Context("InMemory", func() {
+
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		var db PackageSet
+
+		BeforeEach(func() {
+
+			tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			db = NewInMemoryDatabase(false)
+			if os.Getenv("BENCHMARK_TESTS") != "true" {
+				Skip("BENCHMARK_TESTS not enabled")
+			}
+		})
+
+		Measure("it should be fast in computing world from a 100000 dataset", func(b Benchmarker) {
+
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 100000; i++ {
+					a = NewPackage("A"+strconv.Itoa(i), ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					_, err := db.CreatePackage(a)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				packs := db.World()
+				Expect(len(packs)).To(Equal(100000))
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 10), "World() shouldn't take too long.")
+
+		}, 2)
+	})
+})

pkg/package/database_boltdb.go

@@ -16,7 +16,10 @@
 package pkg
 
 import (
+	"encoding/base64"
+	"fmt"
 	"os"
+	"regexp"
 	"strconv"
 	"sync"
 	"time"
@@ -32,7 +35,8 @@ import (
 
 type BoltDatabase struct {
 	sync.Mutex
-	Path string
+	Path             string
+	ProvidesDatabase map[string]map[string]Package
 }
 
 func NewBoltDatabase(path string) PackageDatabase {
@@ -40,27 +44,54 @@ func NewBoltDatabase(path string) PackageDatabase {
 	// 	BoltInstance = &BoltDatabase{Path: path}
 	// }
 	//return BoltInstance, nil
-	return &BoltDatabase{Path: path}
+	return &BoltDatabase{Path: path, ProvidesDatabase: map[string]map[string]Package{}}
 }
 
 func (db *BoltDatabase) Get(s string) (string, error) {
-	return "", errors.New("Not implemented")
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return "", err
+	}
+	defer bolt.Close()
+	var str string
+	bolt.Get("solver", s, &str)
+
+	return str, errors.New("Not implemented")
 }
 
 func (db *BoltDatabase) Set(k, v string) error {
-	return errors.New("Not implemented")
-
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return err
+	}
+	defer bolt.Close()
+	return bolt.Set("solver", k, v)
 }
+func (db *BoltDatabase) Create(id string, v []byte) (string, error) {
+	enc := base64.StdEncoding.EncodeToString(v)
 
-func (db *BoltDatabase) Create(v []byte) (string, error) {
-	return "", errors.New("Not implemented")
+	return id, db.Set(id, enc)
 }
 
 func (db *BoltDatabase) Retrieve(ID string) ([]byte, error) {
-	return []byte{}, errors.New("Not implemented")
+	pa, err := db.Get(ID)
+	if err != nil {
+		return nil, err
+	}
+
+	enc, err := base64.StdEncoding.DecodeString(pa)
+	if err != nil {
+		return nil, err
+	}
+	return enc, nil
 }
 
 func (db *BoltDatabase) FindPackage(tofind Package) (Package, error) {
+	// Provides: Return the replaced package here
+	if provided, err := db.getProvide(tofind); err == nil {
+		return provided, nil
+	}
+
 	p := &DefaultPackage{}
 	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
@@ -76,18 +107,13 @@ func (db *BoltDatabase) FindPackage(tofind Package) (Package, error) {
 }
 
 func (db *BoltDatabase) UpdatePackage(p Package) error {
-
-	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	// TODO: Change, but by query we cannot update by ID
+	err := db.RemovePackage(p)
 	if err != nil {
 		return err
 	}
-	defer bolt.Close()
+	_, err = db.CreatePackage(p)
 
-	dp, ok := p.(*DefaultPackage)
-	if !ok {
-		return errors.New("Bolt DB support only DefaultPackage type for now")
-	}
-	err = bolt.Update(dp)
 	if err != nil {
 		return err
 	}
@@ -106,7 +132,9 @@ func (db *BoltDatabase) GetPackage(ID string) (Package, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = bolt.One("ID", iid, p)
+	err = bolt.Select(q.Eq("ID", iid)).Limit(1).First(p)
+
+	//err = bolt.One("id", iid, p)
 	return p, err
 }
 
@@ -134,26 +162,17 @@ func (db *BoltDatabase) GetAllPackages(packages chan Package) error {
 		return err
 	}
 	defer bolt.Close()
-	// Fetching records one by one (useful when the bucket contains a lot of records)
-	//query := bolt.Select()
-
-	var packs []Package
+	var packs []DefaultPackage
 	err = bolt.All(&packs)
 	if err != nil {
 		return err
 	}
 
 	for _, r := range packs {
-		packages <- r
+		packages <- &r
 	}
 
 	return nil
-
-	// return query.Each(new(DefaultPackage), func(record interface{}) error {
-	// 	u := record.(*DefaultPackage)
-	// 	packages <- u
-	// 	return err
-	// })
 }
 
 // Encode encodes the package to string.
@@ -175,11 +194,243 @@ func (db *BoltDatabase) CreatePackage(p Package) (string, error) {
 		return "", errors.Wrap(err, "Error saving package to "+db.Path)
 	}
 
+	// Create extra cache between package -> []versions
+	db.Lock()
+	defer db.Unlock()
+	// TODO: Replace with a bolt implementation (and not in memory)
+	// Provides: Store package provides, we will reuse this when walking deps
+	for _, provide := range dp.Provides {
+		if _, ok := db.ProvidesDatabase[provide.GetPackageName()]; !ok {
+			db.ProvidesDatabase[provide.GetPackageName()] = make(map[string]Package)
+
+		}
+
+		db.ProvidesDatabase[provide.GetPackageName()][provide.GetVersion()] = p
+	}
+
 	return strconv.Itoa(dp.ID), err
 }
 
+// Dup from memory implementation
+func (db *BoltDatabase) getProvide(p Package) (Package, error) {
+	db.Lock()
+	pa, ok := db.ProvidesDatabase[p.GetPackageName()][p.GetVersion()]
+	if !ok {
+		versions, ok := db.ProvidesDatabase[p.GetPackageName()]
+		db.Unlock()
+
+		if !ok {
+			return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
+		}
+
+		for ve, _ := range versions {
+
+			match, err := p.VersionMatchSelector(ve, nil)
+			if err != nil {
+				return nil, errors.Wrap(err, "Error on match version")
+			}
+			if match {
+				pa, ok := db.ProvidesDatabase[p.GetPackageName()][ve]
+				if !ok {
+					return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
+				}
+				return pa, nil //pick the first (we shouldn't have providers that are conflicting)
+				// TODO: A find dbcall here would recurse, but would give chance to have providers of providers
+			}
+		}
+
+		return nil, errors.New("No package provides this")
+	}
+	db.Unlock()
+	return db.FindPackage(pa)
+}
+
 func (db *BoltDatabase) Clean() error {
 	db.Lock()
 	defer db.Unlock()
 	return os.RemoveAll(db.Path)
 }
+
+func (db *BoltDatabase) GetPackageFiles(p Package) ([]string, error) {
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return []string{}, errors.Wrap(err, "Error opening boltdb "+db.Path)
+	}
+	defer bolt.Close()
+
+	files := bolt.From("files")
+	var pf PackageFile
+	err = files.One("PackageFingerprint", p.GetFingerPrint(), &pf)
+	if err != nil {
+		return []string{}, errors.Wrap(err, "While finding files")
+	}
+	return pf.Files, nil
+}
+func (db *BoltDatabase) SetPackageFiles(p *PackageFile) error {
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return errors.Wrap(err, "Error opening boltdb "+db.Path)
+	}
+	defer bolt.Close()
+
+	files := bolt.From("files")
+	return files.Save(p)
+}
+func (db *BoltDatabase) RemovePackageFiles(p Package) error {
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return errors.Wrap(err, "Error opening boltdb "+db.Path)
+	}
+	defer bolt.Close()
+
+	files := bolt.From("files")
+	var pf PackageFile
+	err = files.One("PackageFingerprint", p.GetFingerPrint(), &pf)
+	if err != nil {
+		return errors.Wrap(err, "While finding files")
+	}
+	return files.DeleteStruct(&pf)
+}
+
+func (db *BoltDatabase) RemovePackage(p Package) error {
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return errors.Wrap(err, "Error opening boltdb "+db.Path)
+	}
+	defer bolt.Close()
+	var found DefaultPackage
+	err = bolt.Select(q.Eq("Name", p.GetName()), q.Eq("Category", p.GetCategory()), q.Eq("Version", p.GetVersion())).Limit(1).Delete(&found)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Package not found: %s", p.HumanReadableString()))
+	}
+	return nil
+}
+
+func (db *BoltDatabase) World() Packages {
+	var packs []DefaultPackage
+
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
+	if err != nil {
+		return Packages([]Package{})
+	}
+	defer bolt.Close()
+	err = bolt.All(&packs)
+	if err != nil {
+		return Packages([]Package{})
+	}
+	models := make([]Package, len(packs))
+	for i, _ := range packs {
+		models[i] = &packs[i]
+	}
+
+	return Packages(models)
+}
+
+func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
+
+	required, err := db.FindPackage(p)
+	if err != nil {
+
+		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
+		packages, err := p.Expand(db)
+		//	Info("Expanded", packages, err)
+		if err != nil || len(packages) == 0 {
+			required = p
+		} else {
+			required = packages.Best(nil)
+
+		}
+		return required, nil
+		//required = &DefaultPackage{Name: "test"}
+	}
+
+	return required, err
+
+}
+
+// FindPackages return the list of the packages beloging to cat/name  (any versions in requested range)
+// FIXME: Optimize, see inmemorydb
+func (db *BoltDatabase) FindPackages(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+	var versionsInWorld []Package
+	for _, w := range db.World() {
+		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
+			continue
+		}
+
+		match, err := p.SelectorMatchVersion(w.GetVersion(), nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error on match selector")
+		}
+		if match {
+			versionsInWorld = append(versionsInWorld, w)
+		}
+	}
+	return Packages(versionsInWorld), nil
+}
+
+// FindPackageVersions return the list of the packages beloging to cat/name
+func (db *BoltDatabase) FindPackageVersions(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+
+	var versionsInWorld []Package
+	for _, w := range db.World() {
+		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
+			continue
+		}
+
+		versionsInWorld = append(versionsInWorld, w)
+	}
+	return Packages(versionsInWorld), nil
+}
+
+func (db *BoltDatabase) FindPackageLabel(labelKey string) (Packages, error) {
+	var ans []Package
+
+	for _, pack := range db.World() {
+		if pack.HasLabel(labelKey) {
+			ans = append(ans, pack)
+		}
+	}
+	return Packages(ans), nil
+}
+
+func (db *BoltDatabase) FindPackageLabelMatch(pattern string) (Packages, error) {
+	var ans []Package
+
+	re := regexp.MustCompile(pattern)
+	if re == nil {
+		return nil, errors.New("Invalid regex " + pattern + "!")
+	}
+
+	for _, pack := range db.World() {
+		if pack.MatchLabel(re) {
+			ans = append(ans, pack)
+		}
+	}
+
+	return Packages(ans), nil
+}
+
+func (db *BoltDatabase) FindPackageMatch(pattern string) (Packages, error) {
+	var ans []Package
+
+	re := regexp.MustCompile(pattern)
+	if re == nil {
+		return nil, errors.New("Invalid regex " + pattern + "!")
+	}
+
+	for _, pack := range db.World() {
+		if re.MatchString(pack.HumanReadableString()) {
+			ans = append(ans, pack)
+		}
+	}
+
+	return Packages(ans), nil
+}

pkg/package/database_boltdb_test.go

@@ -0,0 +1,152 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or ItNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	"io/ioutil"
+	"os"
+	"regexp"
+
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("BoltDB Database", func() {
+
+	tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+	defer os.Remove(tmpfile.Name()) // clean up
+	var db PackageDatabase
+
+	BeforeEach(func() {
+		tmpfile, _ = ioutil.TempFile(os.TempDir(), "tests")
+		defer os.Remove(tmpfile.Name()) // clean up
+		db = NewBoltDatabase(tmpfile.Name())
+	})
+	Context("Simple package", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		It("Find packages", func() {
+			ID, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.GetPackage(ID)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack).To(Equal(a))
+			ids := db.GetPackages()
+
+			Expect(ids).To(Equal([]string{"1"}))
+
+			pack, err = db.FindPackage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Expands correctly", func() {
+
+			a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+			re := regexp.MustCompile("project[0-9][=].*")
+			for _, p := range []Package{a1, a11, a01} {
+				_, err := db.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			lst, err := a.Expand(db)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(lst).To(ContainElement(a11))
+			Expect(lst).To(ContainElement(a1))
+			Expect(lst).ToNot(ContainElement(a01))
+			Expect(len(lst)).To(Equal(2))
+			p := lst.Best(nil)
+			Expect(p).To(Equal(a11))
+			// Test annotation with null map
+			Expect(a.MatchAnnotation(re)).To(Equal(false))
+		})
+
+		It("Find best package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+
+		})
+
+		It("Find specific package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Provides replaces definitions", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+			a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+			Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+
+			s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackage(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+		})
+
+	})
+
+})

pkg/package/database_mem.go

@@ -18,29 +18,39 @@ package pkg
 import (
 	"encoding/base64"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"hash/crc32"
+	"regexp"
 	"sync"
+
+	"github.com/pkg/errors"
 )
 
 var DBInMemoryInstance = &InMemoryDatabase{
-	Mutex: &sync.Mutex{},
-
-	Database: map[string]string{}}
+	Mutex:            &sync.Mutex{},
+	FileDatabase:     map[string][]string{},
+	Database:         map[string]string{},
+	CacheNoVersion:   map[string]map[string]interface{}{},
+	ProvidesDatabase: map[string]map[string]Package{},
+}
 
 type InMemoryDatabase struct {
 	*sync.Mutex
-	Database map[string]string
+	Database         map[string]string
+	FileDatabase     map[string][]string
+	CacheNoVersion   map[string]map[string]interface{}
+	ProvidesDatabase map[string]map[string]Package
 }
 
 func NewInMemoryDatabase(singleton bool) PackageDatabase {
 	// In memoryDB is a singleton
 	if !singleton {
 		return &InMemoryDatabase{
-			Mutex: &sync.Mutex{},
-
-			Database: map[string]string{}}
+			Mutex:            &sync.Mutex{},
+			FileDatabase:     map[string][]string{},
+			Database:         map[string]string{},
+			CacheNoVersion:   map[string]map[string]interface{}{},
+			ProvidesDatabase: map[string]map[string]Package{},
+		}
 	}
 	return DBInMemoryInstance
 }
@@ -50,7 +60,7 @@ func (db *InMemoryDatabase) Get(s string) (string, error) {
 	defer db.Unlock()
 	pa, ok := db.Database[s]
 	if !ok {
-		return "", errors.New("No key found with that id")
+		return "", errors.New(fmt.Sprintf("No key found for: %s", s))
 	}
 	return pa, nil
 }
@@ -63,12 +73,10 @@ func (db *InMemoryDatabase) Set(k, v string) error {
 	return nil
 }
 
-func (db *InMemoryDatabase) Create(v []byte) (string, error) {
+func (db *InMemoryDatabase) Create(id string, v []byte) (string, error) {
 	enc := base64.StdEncoding.EncodeToString(v)
-	crc32q := crc32.MakeTable(0xD5828281)
-	ID := fmt.Sprintf("%08x", crc32.Checksum([]byte(enc), crc32q)) // TODO: Replace with package fingerprint?
 
-	return ID, db.Set(ID, enc)
+	return id, db.Set(id, enc)
 }
 
 func (db *InMemoryDatabase) Retrieve(ID string) ([]byte, error) {
@@ -93,7 +101,13 @@ func (db *InMemoryDatabase) GetPackage(ID string) (Package, error) {
 
 	p := &DefaultPackage{}
 
-	if err := json.Unmarshal(enc, &p); err != nil {
+	rawIn := json.RawMessage(enc)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return &DefaultPackage{}, err
+	}
+
+	if err := json.Unmarshal(bytes, &p); err != nil {
 		return nil, err
 	}
 	return p, nil
@@ -119,75 +133,157 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 		return "", errors.New("InMemoryDatabase suports only DefaultPackage")
 	}
 
-	res, err := json.Marshal(pd)
+	res, err := pd.JSON()
 	if err != nil {
 		return "", err
 	}
 
-	ID, err := db.Create(res)
+	ID, err := db.Create(pd.GetFingerPrint(), res)
 	if err != nil {
 		return "", err
 	}
+
+	// Create extra cache between package -> []versions
+	db.Lock()
+	defer db.Unlock()
+
+	// Provides: Store package provides, we will reuse this when walking deps
+	for _, provide := range pd.Provides {
+		if _, ok := db.ProvidesDatabase[provide.GetPackageName()]; !ok {
+			db.ProvidesDatabase[provide.GetPackageName()] = make(map[string]Package)
+
+		}
+
+		db.ProvidesDatabase[provide.GetPackageName()][provide.GetVersion()] = p
+	}
+
+	_, ok = db.CacheNoVersion[p.GetPackageName()]
+	if !ok {
+		db.CacheNoVersion[p.GetPackageName()] = make(map[string]interface{})
+	}
+	db.CacheNoVersion[p.GetPackageName()][p.GetVersion()] = nil
+
 	return ID, nil
 }
 
+func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
+	db.Lock()
+	pa, ok := db.ProvidesDatabase[p.GetPackageName()][p.GetVersion()]
+	if !ok {
+		versions, ok := db.ProvidesDatabase[p.GetPackageName()]
+		db.Unlock()
+
+		if !ok {
+			return nil, errors.New("No versions found for package")
+		}
+
+		for ve, _ := range versions {
+
+			match, err := p.VersionMatchSelector(ve, nil)
+			if err != nil {
+				return nil, errors.Wrap(err, "Error on match version")
+			}
+			if match {
+				pa, ok := db.ProvidesDatabase[p.GetPackageName()][ve]
+				if !ok {
+					return nil, errors.New("No versions found for package")
+				}
+				return pa, nil
+			}
+		}
+
+		return nil, errors.New("No package provides this")
+	}
+	db.Unlock()
+	return db.FindPackage(pa)
+}
+
 func (db *InMemoryDatabase) encodePackage(p Package) (string, string, error) {
 	pd, ok := p.(*DefaultPackage)
 	if !ok {
 		return "", "", errors.New("InMemoryDatabase suports only DefaultPackage")
 	}
 
-	res, err := json.Marshal(pd)
+	res, err := pd.JSON()
 	if err != nil {
 		return "", "", err
 	}
 	enc := base64.StdEncoding.EncodeToString(res)
-	crc32q := crc32.MakeTable(0xD5828281)
-	ID := fmt.Sprintf("%08x", crc32.Checksum([]byte(enc), crc32q)) // TODO: Replace with package fingerprint?
 
-	return ID, enc, nil
+	return p.GetFingerPrint(), enc, nil
 }
 
 func (db *InMemoryDatabase) FindPackage(p Package) (Package, error) {
 
-	// TODO: Replace this piece, when IDs are fingerprint, findpackage becames O(1)
+	// Provides: Return the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		return provided, nil
+	}
 
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
+	return db.GetPackage(p.GetFingerPrint())
+}
+
+// FindPackages return the list of the packages beloging to cat/name
+func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+
+	versions, ok := db.CacheNoVersion[p.GetPackageName()]
+	if !ok {
+		return nil, errors.New("No versions found for package")
+	}
+	var versionsInWorld []Package
+	for ve, _ := range versions {
+		w, err := db.FindPackage(&DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")
 		}
-		if pack.Matches(p) {
-			return pack, nil
+		versionsInWorld = append(versionsInWorld, w)
+	}
+	return Packages(versionsInWorld), nil
+}
+
+// FindPackages return the list of the packages beloging to cat/name (any versions in requested range)
+func (db *InMemoryDatabase) FindPackages(p Package) (Packages, error) {
+
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+	versions, ok := db.CacheNoVersion[p.GetPackageName()]
+	if !ok {
+		return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
+	}
+	var versionsInWorld []Package
+	for ve, _ := range versions {
+		match, err := p.SelectorMatchVersion(ve, nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error on match selector")
+		}
+
+		if match {
+			w, err := db.FindPackage(&DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
+			if err != nil {
+				return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")
+			}
+			versionsInWorld = append(versionsInWorld, w)
 		}
 	}
-	return nil, errors.New("Package not found")
+	return Packages(versionsInWorld), nil
 }
 
 func (db *InMemoryDatabase) UpdatePackage(p Package) error {
-	var id string
-	found := false
-	for _, k := range db.GetPackages() {
-		pack, err := db.GetPackage(k)
-		if err != nil {
-			return err
-		}
-		if pack.Matches(p) {
-			id = k
-			found = true
-			break
-		}
-	}
-	if found {
 
-		_, enc, err := db.encodePackage(p)
-		if err != nil {
-			return err
-		}
-
-		return db.Set(id, enc)
+	_, enc, err := db.encodePackage(p)
+	if err != nil {
+		return err
 	}
-	return errors.New("Package not found")
+
+	return db.Set(p.GetFingerPrint(), enc)
+
+	return errors.New(fmt.Sprintf("Package not found: %s", p.HumanReadableString()))
 }
 
 func (db *InMemoryDatabase) GetPackages() []string {
@@ -204,3 +300,127 @@ func (db *InMemoryDatabase) Clean() error {
 	db.Database = map[string]string{}
 	return nil
 }
+
+func (db *InMemoryDatabase) GetPackageFiles(p Package) ([]string, error) {
+
+	db.Lock()
+	defer db.Unlock()
+
+	pa, ok := db.FileDatabase[p.GetFingerPrint()]
+	if !ok {
+		return pa, errors.New(fmt.Sprintf("No key found for: %s", p.HumanReadableString()))
+	}
+
+	return pa, nil
+}
+func (db *InMemoryDatabase) SetPackageFiles(p *PackageFile) error {
+	db.Lock()
+	defer db.Unlock()
+	db.FileDatabase[p.PackageFingerprint] = p.Files
+	return nil
+}
+func (db *InMemoryDatabase) RemovePackageFiles(p Package) error {
+	db.Lock()
+	defer db.Unlock()
+	delete(db.FileDatabase, p.GetFingerPrint())
+	return nil
+}
+
+func (db *InMemoryDatabase) RemovePackage(p Package) error {
+	db.Lock()
+	defer db.Unlock()
+
+	delete(db.Database, p.GetFingerPrint())
+	return nil
+}
+func (db *InMemoryDatabase) World() Packages {
+	var all []Package
+	// FIXME: This should all be locked in the db - for now forbid the solver to be run in threads.
+	for _, k := range db.GetPackages() {
+		pack, err := db.GetPackage(k)
+		if err == nil {
+			all = append(all, pack)
+		}
+	}
+	return Packages(all)
+}
+
+func (db *InMemoryDatabase) FindPackageCandidate(p Package) (Package, error) {
+
+	required, err := db.FindPackage(p)
+	if err != nil {
+		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
+		packages, err := p.Expand(db)
+		//	Info("Expanded", packages, err)
+		if err != nil || len(packages) == 0 {
+			required = p
+		} else {
+			required = packages.Best(nil)
+
+		}
+		return required, nil
+		//required = &DefaultPackage{Name: "test"}
+	}
+
+	return required, err
+
+}
+
+func (db *InMemoryDatabase) FindPackageLabel(labelKey string) (Packages, error) {
+	var ans []Package
+
+	for _, k := range db.GetPackages() {
+		pack, err := db.GetPackage(k)
+		if err != nil {
+			return ans, err
+		}
+		if pack.HasLabel(labelKey) {
+			ans = append(ans, pack)
+		}
+	}
+
+	return Packages(ans), nil
+}
+
+func (db *InMemoryDatabase) FindPackageLabelMatch(pattern string) (Packages, error) {
+	var ans []Package
+
+	re := regexp.MustCompile(pattern)
+	if re == nil {
+		return nil, errors.New("Invalid regex " + pattern + "!")
+	}
+
+	for _, k := range db.GetPackages() {
+		pack, err := db.GetPackage(k)
+		if err != nil {
+			return ans, err
+		}
+		if pack.MatchLabel(re) {
+			ans = append(ans, pack)
+		}
+	}
+
+	return Packages(ans), nil
+}
+
+func (db *InMemoryDatabase) FindPackageMatch(pattern string) (Packages, error) {
+	var ans []Package
+
+	re := regexp.MustCompile(pattern)
+	if re == nil {
+		return nil, errors.New("Invalid regex " + pattern + "!")
+	}
+
+	for _, k := range db.GetPackages() {
+		pack, err := db.GetPackage(k)
+		if err != nil {
+			return ans, err
+		}
+
+		if re.MatchString(pack.HumanReadableString()) {
+			ans = append(ans, pack)
+		}
+	}
+
+	return Packages(ans), nil
+}

pkg/package/database_mem_test.go

@@ -0,0 +1,127 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Database", func() {
+
+	db := NewInMemoryDatabase(false)
+	Context("Simple package", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		//	a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		//		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+		//		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+		It("Saves and get data back correctly", func() {
+
+			ID, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.GetPackage(ID)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Gets all", func() {
+
+			ids := db.GetPackages()
+
+			Expect(ids).To(Equal([]string{"A-->=1.0"}))
+
+		})
+		It("Find packages", func() {
+
+			pack, err := db.FindPackage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Find best package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+
+		})
+
+		It("Find specific package candidate", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+			s := NewPackage("A", "=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackageCandidate(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Provides replaces definitions", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+			a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+			Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+
+			s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+			pack, err := db.FindPackage(s)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a3))
+		})
+	})
+
+})

pkg/package/database_test.go

@@ -1,60 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package pkg_test
-
-import (
-	. "github.com/mudler/luet/pkg/package"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Database", func() {
-
-	db := NewInMemoryDatabase(false)
-	Context("Simple package", func() {
-		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
-		//	a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-		//		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
-		//		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
-		It("Saves and get data back correctly", func() {
-
-			ID, err := db.CreatePackage(a)
-			Expect(err).ToNot(HaveOccurred())
-
-			pack, err := db.GetPackage(ID)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(pack).To(Equal(a))
-
-		})
-
-		It("Gets all", func() {
-
-			ids := db.GetPackages()
-
-			Expect(ids).To(Equal([]string{"b536b2bd"}))
-
-		})
-		It("Find packages", func() {
-
-			pack, err := db.FindPackage(a)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(pack).To(Equal(a))
-
-		})
-	})
-
-})

pkg/package/package.go

@@ -16,41 +16,59 @@
 package pkg
 
 import (
+	"bytes"
+	"crypto/md5"
 	"encoding/json"
 	"fmt"
+	"io"
 	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
 
+	"github.com/mudler/luet/pkg/helpers"
+	version "github.com/mudler/luet/pkg/versioner"
+
+	gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/crillab/gophersat/bf"
-	version "github.com/hashicorp/go-version"
-	"github.com/jinzhu/copier"
-
 	"github.com/ghodss/yaml"
+	"github.com/jinzhu/copier"
+	"github.com/pkg/errors"
 )
 
 // Package is a package interface (TBD)
 // FIXME: Currently some of the methods are returning DefaultPackages due to JSON serialization of the package
 type Package interface {
 	Encode(PackageDatabase) (string, error)
+	Related(definitiondb PackageDatabase) Packages
+
+	BuildFormula(PackageDatabase, PackageDatabase) ([]bf.Formula, error)
 
-	BuildFormula(PackageDatabase) ([]bf.Formula, error)
-	IsFlagged(bool) Package
-	Flagged() bool
 	GetFingerPrint() string
+	GetPackageName() string
 	Requires([]*DefaultPackage) Package
 	Conflicts([]*DefaultPackage) Package
-	Revdeps(world *[]Package) []Package
+	Revdeps(PackageDatabase) Packages
+	ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages
+	LabelDeps(PackageDatabase, string) Packages
+
+	GetProvides() []*DefaultPackage
+	SetProvides([]*DefaultPackage) Package
 
 	GetRequires() []*DefaultPackage
 	GetConflicts() []*DefaultPackage
-	Expand(*[]Package) ([]Package, error)
+	Expand(PackageDatabase) (Packages, error)
 	SetCategory(string)
 
 	GetName() string
+	SetName(string)
 	GetCategory() string
 
 	GetVersion() string
-	RequiresContains(Package) bool
+	SetVersion(string)
+	RequiresContains(PackageDatabase, Package) (bool, error)
 	Matches(m Package) bool
+	BumpBuildVersion() error
 
 	AddUse(use string)
 	RemoveUse(use string)
@@ -62,54 +80,107 @@ type Package interface {
 	SetPath(string)
 	GetPath() string
 	Rel(string) string
-}
 
-type PackageSet interface {
-	GetPackages() []string //Ids
-	CreatePackage(pkg Package) (string, error)
-	GetPackage(ID string) (Package, error)
-	Clean() error
-	FindPackage(Package) (Package, error)
-	UpdatePackage(p Package) error
-	GetAllPackages(packages chan Package) error
+	GetDescription() string
+	SetDescription(string)
+
+	AddURI(string)
+	GetURI() []string
+
+	SetLicense(string)
+	GetLicense() string
+
+	AddLabel(string, string)
+	GetLabels() map[string]string
+	HasLabel(string) bool
+	MatchLabel(*regexp.Regexp) bool
+
+	AddAnnotation(string, string)
+	GetAnnotations() map[string]string
+	HasAnnotation(string) bool
+	MatchAnnotation(*regexp.Regexp) bool
+
+	IsHidden() bool
+	IsSelector() bool
+	VersionMatchSelector(string, version.Versioner) (bool, error)
+	SelectorMatchVersion(string, version.Versioner) (bool, error)
+
+	String() string
+	HumanReadableString() string
+	HashFingerprint(string) string
+
+	SetBuildTimestamp(s string)
+	GetBuildTimestamp() string
+
+	Clone() Package
 }
 
 type Tree interface {
 	GetPackageSet() PackageDatabase
 	Prelude() string // A tree might have a prelude to be able to consume a tree
 	SetPackageSet(s PackageDatabase)
-	World() ([]Package, error)
+	World() (Packages, error)
 	FindPackage(Package) (Package, error)
-	ResolveDeps(int) error
 }
 
+type Packages []Package
+
 // >> Unmarshallers
 // DefaultPackageFromYaml decodes a package from yaml bytes
-func DefaultPackageFromYaml(source []byte) (DefaultPackage, error) {
-	var pkg DefaultPackage
-	err := yaml.Unmarshal(source, &pkg)
+func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) {
+
+	var unescaped DefaultPackage
+	source, err := yaml.YAMLToJSON(yml)
 	if err != nil {
-		return pkg, err
+		return DefaultPackage{}, err
 	}
-	return pkg, nil
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return DefaultPackage{}, err
+	}
+	err = json.Unmarshal(bytes, &unescaped)
+	if err != nil {
+		return DefaultPackage{}, err
+	}
+	return unescaped, nil
+}
+
+// Major and minor gets escaped when marshalling in JSON, making compiler fails recognizing selectors for expansion
+func (t *DefaultPackage) JSON() ([]byte, error) {
+	buffer := &bytes.Buffer{}
+	encoder := json.NewEncoder(buffer)
+	encoder.SetEscapeHTML(false)
+	err := encoder.Encode(t)
+	return buffer.Bytes(), err
 }
 
 // DefaultPackage represent a standard package definition
 type DefaultPackage struct {
-	ID               int      `json:"-" storm:"id,increment"` // primary key with auto increment
-	Name             string   `json:"name"`                   // Affects YAML field names too.
-	Version          string   `json:"version"`                // Affects YAML field names too.
-	Category         string   `json:"category"`               // Affects YAML field names too.
-	UseFlags         []string `json:"use_flags"`              // Affects YAML field names too.
-	State            State
-	PackageRequires  []*DefaultPackage `json:"requires"`  // Affects YAML field names too.
-	PackageConflicts []*DefaultPackage `json:"conflicts"` // Affects YAML field names too.
-	IsSet            bool              `json:"set"`       // Affects YAML field names too.
+	ID               int               `storm:"id,increment" json:"id"` // primary key with auto increment
+	Name             string            `json:"name"`                    // Affects YAML field names too.
+	Version          string            `json:"version"`                 // Affects YAML field names too.
+	Category         string            `json:"category"`                // Affects YAML field names too.
+	UseFlags         []string          `json:"use_flags,omitempty"`     // Affects YAML field names too.
+	State            State             `json:"state,omitempty"`
+	PackageRequires  []*DefaultPackage `json:"requires"`           // Affects YAML field names too.
+	PackageConflicts []*DefaultPackage `json:"conflicts"`          // Affects YAML field names too.
+	Provides         []*DefaultPackage `json:"provides,omitempty"` // Affects YAML field names too.
+	Hidden           bool              `json:"hidden,omitempty"`   // Affects YAML field names too.
 
-	// TODO: Annotations?
+	// Annotations are used for core features/options
+	Annotations map[string]string `json:"annotations,omitempty"` // Affects YAML field names too
 
 	// Path is set only internally when tree is loaded from disk
 	Path string `json:"path,omitempty"`
+
+	Description    string   `json:"description,omitempty"`
+	Uri            []string `json:"uri,omitempty"`
+	License        string   `json:"license,omitempty"`
+	BuildTimestamp string   `json:"buildtimestamp,omitempty"`
+
+	Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
 }
 
 // State represent the package state
@@ -117,13 +188,19 @@ type State string
 
 // NewPackage returns a new package
 func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*DefaultPackage) *DefaultPackage {
-	return &DefaultPackage{Name: name, Version: version, PackageRequires: requires, PackageConflicts: conflicts}
+	return &DefaultPackage{
+		Name:             name,
+		Version:          version,
+		PackageRequires:  requires,
+		PackageConflicts: conflicts,
+		Labels:           nil,
+	}
 }
 
 func (p *DefaultPackage) String() string {
-	b, err := json.Marshal(p)
+	b, err := p.JSON()
 	if err != nil {
-		return fmt.Sprintf("{ id: \"%d\", name: \"%s\" }", p.ID, p.Name)
+		return fmt.Sprintf("{ id: \"%d\", name: \"%s\", version: \"%s\", category: \"%s\"  }", p.ID, p.Name, p.Version, p.Category)
 	}
 	return fmt.Sprintf("%s", string(b))
 }
@@ -134,6 +211,40 @@ func (p *DefaultPackage) GetFingerPrint() string {
 	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version)
 }
 
+func (p *DefaultPackage) HashFingerprint(salt string) string {
+	h := md5.New()
+	io.WriteString(h, fmt.Sprintf("%s-%s", p.GetFingerPrint(), salt))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func (p *DefaultPackage) HumanReadableString() string {
+	return fmt.Sprintf("%s/%s-%s", p.Category, p.Name, p.Version)
+}
+
+func FromString(s string) Package {
+	var unescaped DefaultPackage
+
+	err := json.Unmarshal([]byte(s), &unescaped)
+	if err != nil {
+		return &unescaped
+	}
+	return &unescaped
+}
+
+func (p *DefaultPackage) GetPackageName() string {
+	return fmt.Sprintf("%s-%s", p.Name, p.Category)
+}
+
+// GetBuildTimestamp returns the package build timestamp
+func (p *DefaultPackage) GetBuildTimestamp() string {
+	return p.BuildTimestamp
+}
+
+// SetBuildTimestamp sets the package Build timestamp
+func (p *DefaultPackage) SetBuildTimestamp(s string) {
+	p.BuildTimestamp = s
+}
+
 // GetPath returns the path where the definition file was found
 func (p *DefaultPackage) GetPath() string {
 	return p.Path
@@ -147,6 +258,30 @@ func (p *DefaultPackage) SetPath(s string) {
 	p.Path = s
 }
 
+func (p *DefaultPackage) IsSelector() bool {
+	return strings.ContainsAny(p.GetVersion(), "<>=")
+}
+
+func (p *DefaultPackage) IsHidden() bool {
+	return p.Hidden
+}
+
+func (p *DefaultPackage) HasLabel(label string) bool {
+	return helpers.MapHasKey(&p.Labels, label)
+}
+
+func (p *DefaultPackage) MatchLabel(r *regexp.Regexp) bool {
+	return helpers.MapMatchRegex(&p.Labels, r)
+}
+
+func (p *DefaultPackage) HasAnnotation(label string) bool {
+	return helpers.MapHasKey(&p.Annotations, label)
+}
+
+func (p *DefaultPackage) MatchAnnotation(r *regexp.Regexp) bool {
+	return helpers.MapMatchRegex(&p.Annotations, r)
+}
+
 // AddUse adds a use to a package
 func (p *DefaultPackage) AddUse(use string) {
 	for _, v := range p.UseFlags {
@@ -175,7 +310,11 @@ func (p *DefaultPackage) Encode(db PackageDatabase) (string, error) {
 }
 
 func (p *DefaultPackage) Yaml() ([]byte, error) {
-	y, err := yaml.Marshal(p)
+	j, err := p.JSON()
+	if err != nil {
+		return []byte{}, err
+	}
+	y, err := yaml.JSONToYAML(j)
 	if err != nil {
 
 		return []byte{}, err
@@ -183,15 +322,6 @@ func (p *DefaultPackage) Yaml() ([]byte, error) {
 	return y, nil
 }
 
-func (p *DefaultPackage) IsFlagged(b bool) Package {
-	p.IsSet = b
-	return p
-}
-
-func (p *DefaultPackage) Flagged() bool {
-	return p.IsSet
-}
-
 func (p *DefaultPackage) GetName() string {
 	return p.Name
 }
@@ -199,18 +329,66 @@ func (p *DefaultPackage) GetName() string {
 func (p *DefaultPackage) GetVersion() string {
 	return p.Version
 }
-
+func (p *DefaultPackage) SetVersion(v string) {
+	p.Version = v
+}
+func (p *DefaultPackage) GetDescription() string {
+	return p.Description
+}
+func (p *DefaultPackage) SetDescription(s string) {
+	p.Description = s
+}
+func (p *DefaultPackage) GetLicense() string {
+	return p.License
+}
+func (p *DefaultPackage) SetLicense(s string) {
+	p.License = s
+}
+func (p *DefaultPackage) AddURI(s string) {
+	p.Uri = append(p.Uri, s)
+}
+func (p *DefaultPackage) GetURI() []string {
+	return p.Uri
+}
 func (p *DefaultPackage) GetCategory() string {
 	return p.Category
 }
-
 func (p *DefaultPackage) SetCategory(s string) {
 	p.Category = s
 }
+
+func (p *DefaultPackage) SetName(s string) {
+	p.Name = s
+}
+
 func (p *DefaultPackage) GetUses() []string {
 	return p.UseFlags
 }
-
+func (p *DefaultPackage) AddLabel(k, v string) {
+	if p.Labels == nil {
+		p.Labels = make(map[string]string, 0)
+	}
+	p.Labels[k] = v
+}
+func (p *DefaultPackage) AddAnnotation(k, v string) {
+	if p.Annotations == nil {
+		p.Annotations = make(map[string]string, 0)
+	}
+	p.Annotations[k] = v
+}
+func (p *DefaultPackage) GetLabels() map[string]string {
+	return p.Labels
+}
+func (p *DefaultPackage) GetAnnotations() map[string]string {
+	return p.Annotations
+}
+func (p *DefaultPackage) GetProvides() []*DefaultPackage {
+	return p.Provides
+}
+func (p *DefaultPackage) SetProvides(req []*DefaultPackage) Package {
+	p.Provides = req
+	return p
+}
 func (p *DefaultPackage) GetRequires() []*DefaultPackage {
 	return p.PackageRequires
 }
@@ -230,7 +408,6 @@ func (p *DefaultPackage) Clone() Package {
 	copier.Copy(&new, &p)
 	return new
 }
-
 func (p *DefaultPackage) Matches(m Package) bool {
 	if p.GetFingerPrint() == m.GetFingerPrint() {
 		return true
@@ -238,23 +415,19 @@ func (p *DefaultPackage) Matches(m Package) bool {
 	return false
 }
 
-func (p *DefaultPackage) Expand(world *[]Package) ([]Package, error) {
+func (p *DefaultPackage) Expand(definitiondb PackageDatabase) (Packages, error) {
+	var versionsInWorld Packages
 
-	var versionsInWorld []Package
-	for _, w := range *world {
-		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
-			continue
-		}
-
-		v, err := version.NewVersion(w.GetVersion())
+	all, err := definitiondb.FindPackages(p)
+	if err != nil {
+		return nil, err
+	}
+	for _, w := range all {
+		match, err := p.SelectorMatchVersion(w.GetVersion(), nil)
 		if err != nil {
 			return nil, err
 		}
-		constraints, err := version.NewConstraint(p.GetVersion())
-		if err != nil {
-			return nil, err
-		}
-		if constraints.Check(v) {
+		if match {
 			versionsInWorld = append(versionsInWorld, w)
 		}
 	}
@@ -262,16 +435,16 @@ func (p *DefaultPackage) Expand(world *[]Package) ([]Package, error) {
 	return versionsInWorld, nil
 }
 
-func (p *DefaultPackage) Revdeps(world *[]Package) []Package {
-	var versionsInWorld []Package
-	for _, w := range *world {
+func (p *DefaultPackage) Revdeps(definitiondb PackageDatabase) Packages {
+	var versionsInWorld Packages
+	for _, w := range definitiondb.World() {
 		if w.Matches(p) {
 			continue
 		}
 		for _, re := range w.GetRequires() {
 			if re.Matches(p) {
 				versionsInWorld = append(versionsInWorld, w)
-				versionsInWorld = append(versionsInWorld, w.Revdeps(world)...)
+				versionsInWorld = append(versionsInWorld, w.Revdeps(definitiondb)...)
 			}
 		}
 	}
@@ -279,55 +452,289 @@ func (p *DefaultPackage) Revdeps(world *[]Package) []Package {
 	return versionsInWorld
 }
 
+func walkPackage(p Package, definitiondb PackageDatabase, visited map[string]interface{}) Packages {
+	var versionsInWorld Packages
+	if _, ok := visited[p.HumanReadableString()]; ok {
+		return versionsInWorld
+	}
+	visited[p.HumanReadableString()] = true
+
+	revdepvisited := make(map[string]interface{})
+	revdeps := p.ExpandedRevdeps(definitiondb, revdepvisited)
+	for _, r := range revdeps {
+		versionsInWorld = append(versionsInWorld, r)
+	}
+
+	if !p.IsSelector() {
+		versionsInWorld = append(versionsInWorld, p)
+	}
+
+	for _, re := range p.GetRequires() {
+		versions, _ := re.Expand(definitiondb)
+		for _, r := range versions {
+
+			versionsInWorld = append(versionsInWorld, r)
+			versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...)
+		}
+
+	}
+	for _, re := range p.GetConflicts() {
+		versions, _ := re.Expand(definitiondb)
+		for _, r := range versions {
+
+			versionsInWorld = append(versionsInWorld, r)
+			versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...)
+
+		}
+	}
+	return versionsInWorld.Unique()
+}
+
+func (p *DefaultPackage) Related(definitiondb PackageDatabase) Packages {
+	return walkPackage(p, definitiondb, map[string]interface{}{})
+}
+
+// ExpandedRevdeps returns the package reverse dependencies,
+// matching also selectors in versions (>, <, >=, <=)
+func (p *DefaultPackage) ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages {
+	var versionsInWorld Packages
+	if _, ok := visited[p.HumanReadableString()]; ok {
+		return versionsInWorld
+	}
+	visited[p.HumanReadableString()] = true
+
+	for _, w := range definitiondb.World() {
+		if w.Matches(p) {
+			continue
+		}
+		match := false
+
+		for _, re := range w.GetRequires() {
+			if re.Matches(p) {
+				match = true
+			}
+			if !match {
+
+				packages, _ := re.Expand(definitiondb)
+				for _, pa := range packages {
+					if pa.Matches(p) {
+						match = true
+					}
+				}
+			}
+
+			//	if ok, _ := w.RequiresContains(definitiondb, p); ok {
+
+		}
+
+		if match {
+			versionsInWorld = append(versionsInWorld, w)
+
+			versionsInWorld = append(versionsInWorld, w.ExpandedRevdeps(definitiondb, visited).Unique()...)
+
+		}
+		//	}
+
+	}
+	//visited[p.HumanReadableString()] = true
+	return versionsInWorld.Unique()
+}
+
+func (p *DefaultPackage) LabelDeps(definitiondb PackageDatabase, labelKey string) Packages {
+	var pkgsWithLabelInWorld Packages
+	// TODO: check if integrate some index to improve
+	// research instead of iterate all list.
+	for _, w := range definitiondb.World() {
+		if w.HasLabel(labelKey) {
+			pkgsWithLabelInWorld = append(pkgsWithLabelInWorld, w)
+		}
+	}
+
+	return pkgsWithLabelInWorld
+}
+
 func DecodePackage(ID string, db PackageDatabase) (Package, error) {
 	return db.GetPackage(ID)
 }
 
-func NormalizeFlagged(p Package) {
-	for _, r := range p.GetRequires() {
-		r.IsFlagged(true)
-		NormalizeFlagged(r)
+func (pack *DefaultPackage) scanRequires(definitiondb PackageDatabase, s Package, visited map[string]interface{}) (bool, error) {
+	if _, ok := visited[pack.HumanReadableString()]; ok {
+		return false, nil
 	}
-	for _, r := range p.GetConflicts() {
-		r.IsFlagged(true)
-		NormalizeFlagged(r)
+	visited[pack.HumanReadableString()] = true
+	p, err := definitiondb.FindPackage(pack)
+	if err != nil {
+		p = pack //relax things
+		//return false, errors.Wrap(err, "Package not found in definition db")
 	}
-}
 
-func (p *DefaultPackage) RequiresContains(s Package) bool {
 	for _, re := range p.GetRequires() {
 		if re.Matches(s) {
-			return true
+			return true, nil
 		}
 
-		if re.RequiresContains(s) {
-			return true
+		packages, _ := re.Expand(definitiondb)
+		for _, pa := range packages {
+			if pa.Matches(s) {
+				return true, nil
+			}
+		}
+		if contains, err := re.scanRequires(definitiondb, s, visited); err == nil && contains {
+			return true, nil
 		}
 	}
 
-	return false
+	return false, nil
 }
 
-func (p *DefaultPackage) BuildFormula(db PackageDatabase) ([]bf.Formula, error) {
-	encodedA, err := p.IsFlagged(true).Encode(db)
+// RequiresContains recursively scans into the database packages dependencies to find a match with the given package
+// It is used by the solver during uninstall.
+func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Package) (bool, error) {
+	return pack.scanRequires(definitiondb, s, make(map[string]interface{}))
+}
+
+// Best returns the best version of the package (the most bigger) from a list
+// Accepts a versioner interface to change the ordering policy. If null is supplied
+// It defaults to version.WrappedVersioner which supports both semver and debian versioning
+func (set Packages) Best(v version.Versioner) Package {
+	if v == nil {
+		v = &version.WrappedVersioner{}
+	}
+	var versionsMap map[string]Package = make(map[string]Package)
+	if len(set) == 0 {
+		panic("Best needs a list with elements")
+	}
+
+	versionsRaw := []string{}
+	for _, p := range set {
+		versionsRaw = append(versionsRaw, p.GetVersion())
+		versionsMap[p.GetVersion()] = p
+	}
+	sorted := v.Sort(versionsRaw)
+
+	return versionsMap[sorted[len(sorted)-1]]
+}
+
+func (set Packages) Unique() Packages {
+	var result Packages
+	uniq := make(map[string]Package)
+	for _, p := range set {
+		uniq[p.GetFingerPrint()] = p
+	}
+	for _, p := range uniq {
+		result = append(result, p)
+	}
+	return result
+}
+
+func (pack *DefaultPackage) buildFormula(definitiondb PackageDatabase, db PackageDatabase, visited map[string]interface{}) ([]bf.Formula, error) {
+	if _, ok := visited[pack.HumanReadableString()]; ok {
+		return nil, nil
+	}
+	visited[pack.HumanReadableString()] = true
+	p, err := definitiondb.FindPackage(pack)
+	if err != nil {
+		p = pack // Relax failures and trust the def
+	}
+	encodedA, err := p.Encode(db)
 	if err != nil {
 		return nil, err
 	}
-	NormalizeFlagged(p)
 
 	A := bf.Var(encodedA)
 
 	var formulas []bf.Formula
 
-	for _, required := range p.PackageRequires {
+	// Do conflict with other packages versions (if A is selected, then conflict with other versions of A)
+	packages, _ := definitiondb.FindPackageVersions(p)
+	if len(packages) > 0 {
+		for _, cp := range packages {
+			encodedB, err := cp.Encode(db)
+			if err != nil {
+				return nil, err
+			}
+			B := bf.Var(encodedB)
+			if !p.Matches(cp) {
+				formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Not(A), bf.Not(B))))
+			}
+		}
+	}
+
+	for _, requiredDef := range p.GetRequires() {
+		required, err := definitiondb.FindPackage(requiredDef)
+		if err != nil || requiredDef.IsSelector() {
+			if err == nil {
+				requiredDef = required.(*DefaultPackage)
+			}
+
+			packages, err := definitiondb.FindPackages(requiredDef)
+			if err != nil || len(packages) == 0 {
+				required = requiredDef
+			} else {
+
+				var ALO, priorityConstraints, priorityALO []bf.Formula
+
+				// Try to prio best match
+				// Force the solver to consider first our candidate (if does exists).
+				// Then builds ALO and AMO for the requires.
+				c, candidateErr := definitiondb.FindPackageCandidate(requiredDef)
+				var C bf.Formula
+				if candidateErr == nil {
+					// We have a desired candidate, try to look a solution with that included first
+					for _, o := range packages {
+						encodedB, err := o.Encode(db)
+						if err != nil {
+							return nil, err
+						}
+						B := bf.Var(encodedB)
+						if !o.Matches(c) {
+							priorityConstraints = append(priorityConstraints, bf.Not(B))
+							priorityALO = append(priorityALO, B)
+						}
+					}
+					encodedC, err := c.Encode(db)
+					if err != nil {
+						return nil, err
+					}
+					C = bf.Var(encodedC)
+					// Or the Candidate is true, or all the others might be not true
+					// This forces the CDCL sat implementation to look first at a solution with C=true
+					formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.And(C, bf.Or(priorityConstraints...)), bf.And(bf.Not(C), bf.Or(priorityALO...)))))
+				}
+
+				// AMO - At most one
+				for _, o := range packages {
+					encodedB, err := o.Encode(db)
+					if err != nil {
+						return nil, err
+					}
+					B := bf.Var(encodedB)
+					ALO = append(ALO, B)
+					for _, i := range packages {
+						encodedI, err := i.Encode(db)
+						if err != nil {
+							return nil, err
+						}
+						I := bf.Var(encodedI)
+						if !o.Matches(i) {
+							formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Not(I), bf.Not(B))))
+						}
+					}
+				}
+				formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(ALO...))) // ALO - At least one
+				continue
+			}
+
+		}
+
 		encodedB, err := required.Encode(db)
 		if err != nil {
 			return nil, err
 		}
 		B := bf.Var(encodedB)
 		formulas = append(formulas, bf.Or(bf.Not(A), B))
-
-		f, err := required.BuildFormula(db)
+		r := required.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
+		f, err := r.buildFormula(definitiondb, db, visited)
 		if err != nil {
 			return nil, err
 		}
@@ -335,7 +742,39 @@ func (p *DefaultPackage) BuildFormula(db PackageDatabase) ([]bf.Formula, error)
 
 	}
 
-	for _, required := range p.PackageConflicts {
+	for _, requiredDef := range p.GetConflicts() {
+		required, err := definitiondb.FindPackage(requiredDef)
+		if err != nil || requiredDef.IsSelector() {
+			if err == nil {
+				requiredDef = required.(*DefaultPackage)
+			}
+			packages, err := definitiondb.FindPackages(requiredDef)
+			if err != nil || len(packages) == 0 {
+				required = requiredDef
+			} else {
+				if len(packages) == 1 {
+					required = packages[0]
+				} else {
+					for _, p := range packages {
+						encodedB, err := p.Encode(db)
+						if err != nil {
+							return nil, err
+						}
+						B := bf.Var(encodedB)
+						formulas = append(formulas, bf.Or(bf.Not(A),
+							bf.Not(B)))
+						r := p.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
+						f, err := r.buildFormula(definitiondb, db, visited)
+						if err != nil {
+							return nil, err
+						}
+						formulas = append(formulas, f...)
+					}
+					continue
+				}
+			}
+		}
+
 		encodedB, err := required.Encode(db)
 		if err != nil {
 			return nil, err
@@ -344,22 +783,28 @@ func (p *DefaultPackage) BuildFormula(db PackageDatabase) ([]bf.Formula, error)
 		formulas = append(formulas, bf.Or(bf.Not(A),
 			bf.Not(B)))
 
-		f, err := required.BuildFormula(db)
+		r := required.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
+		f, err := r.buildFormula(definitiondb, db, visited)
 		if err != nil {
 			return nil, err
 		}
 		formulas = append(formulas, f...)
+
 	}
+
 	return formulas, nil
 }
 
+func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db PackageDatabase) ([]bf.Formula, error) {
+	return pack.buildFormula(definitiondb, db, make(map[string]interface{}))
+}
+
 func (p *DefaultPackage) Explain() {
 
 	fmt.Println("====================")
 	fmt.Println("Name: ", p.GetName())
 	fmt.Println("Category: ", p.GetCategory())
 	fmt.Println("Version: ", p.GetVersion())
-	fmt.Println("Installed: ", p.IsSet)
 
 	for _, req := range p.GetRequires() {
 		fmt.Println("\t-> ", req)
@@ -372,3 +817,108 @@ func (p *DefaultPackage) Explain() {
 	fmt.Println("====================")
 
 }
+
+func (p *DefaultPackage) BumpBuildVersion() error {
+	cat := p.Category
+	if cat == "" {
+		// Use fake category for parse package
+		cat = "app"
+	}
+	gp, err := gentoo.ParsePackageStr(
+		fmt.Sprintf("%s/%s-%s", cat,
+			p.Name, p.GetVersion()))
+	if err != nil {
+		return errors.Wrap(err, "Error on parser version")
+	}
+
+	buildPrefix := ""
+	buildId := 0
+
+	if gp.VersionBuild != "" {
+		// Check if version build is a number
+		buildId, err = strconv.Atoi(gp.VersionBuild)
+		if err == nil {
+			goto end
+		}
+		// POST: is not only a number
+
+		// TODO: check if there is a better way to handle all use cases.
+
+		r1 := regexp.MustCompile(`^r[0-9]*$`)
+		if r1 == nil {
+			return errors.New("Error on create regex for -r[0-9]")
+		}
+		if r1.MatchString(gp.VersionBuild) {
+			buildId, err = strconv.Atoi(strings.ReplaceAll(gp.VersionBuild, "r", ""))
+			if err == nil {
+				buildPrefix = "r"
+				goto end
+			}
+		}
+
+		p1 := regexp.MustCompile(`^p[0-9]*$`)
+		if p1 == nil {
+			return errors.New("Error on create regex for -p[0-9]")
+		}
+		if p1.MatchString(gp.VersionBuild) {
+			buildId, err = strconv.Atoi(strings.ReplaceAll(gp.VersionBuild, "p", ""))
+			if err == nil {
+				buildPrefix = "p"
+				goto end
+			}
+		}
+
+		rc1 := regexp.MustCompile(`^rc[0-9]*$`)
+		if rc1 == nil {
+			return errors.New("Error on create regex for -rc[0-9]")
+		}
+		if rc1.MatchString(gp.VersionBuild) {
+			buildId, err = strconv.Atoi(strings.ReplaceAll(gp.VersionBuild, "rc", ""))
+			if err == nil {
+				buildPrefix = "rc"
+				goto end
+			}
+		}
+
+		// Check if version build contains a dot
+		dotIdx := strings.LastIndex(gp.VersionBuild, ".")
+		if dotIdx > 0 {
+			buildPrefix = gp.VersionBuild[0 : dotIdx+1]
+			bVersion := gp.VersionBuild[dotIdx+1:]
+			buildId, err = strconv.Atoi(bVersion)
+			if err == nil {
+				goto end
+			}
+		}
+
+		buildPrefix = gp.VersionBuild + "."
+		buildId = 0
+	}
+
+end:
+
+	buildId++
+	p.Version = fmt.Sprintf("%s%s+%s%d",
+		gp.Version, gp.VersionSuffix, buildPrefix, buildId)
+
+	return nil
+}
+
+func (p *DefaultPackage) SelectorMatchVersion(ver string, v version.Versioner) (bool, error) {
+	if !p.IsSelector() {
+		return false, errors.New("Package is not a selector")
+	}
+	if v == nil {
+		v = &version.WrappedVersioner{}
+	}
+
+	return v.ValidateSelector(ver, p.GetVersion()), nil
+}
+
+func (p *DefaultPackage) VersionMatchSelector(selector string, v version.Versioner) (bool, error) {
+	if v == nil {
+		v = &version.WrappedVersioner{}
+	}
+
+	return v.ValidateSelector(p.GetVersion(), selector), nil
+}

pkg/package/package_suite_test.go

@@ -18,11 +18,15 @@ package pkg_test
 import (
 	"testing"
 
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Package Suite")
 }

pkg/package/package_test.go

@@ -16,30 +16,138 @@
 package pkg_test
 
 import (
+	"regexp"
+
 	. "github.com/mudler/luet/pkg/package"
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 var _ = Describe("Package", func() {
 
+	Context("Encoding/Decoding", func() {
+		a := &DefaultPackage{Name: "test", Version: "1", Category: "t"}
+		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+		It("Encodes and decodes correctly", func() {
+			Expect(a.String()).ToNot(Equal(""))
+			p := FromString(a.String())
+			Expect(p).To(Equal(a))
+		})
+
+		It("Generates packages fingerprint's hashes", func() {
+			Expect(a.HashFingerprint("")).ToNot(Equal(a1.HashFingerprint("")))
+			Expect(a.HashFingerprint("")).To(Equal("76972ef6991ec6102f33b401105c1351"))
+		})
+	})
+
 	Context("Simple package", func() {
 		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
 		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+		re := regexp.MustCompile("project[0-9][=].*")
 		It("Expands correctly", func() {
-			lst, err := a.Expand(&[]Package{a1, a11, a01})
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a1, a11, a01} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			lst, err := a.Expand(definitions)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(lst).To(ContainElement(a11))
 			Expect(lst).To(ContainElement(a1))
 			Expect(lst).ToNot(ContainElement(a01))
 			Expect(len(lst)).To(Equal(2))
-			s := solver.NewSolver([]pkg.Package{}, []pkg.Package{}, NewInMemoryDatabase(false))
-			p := s.Best(lst)
+			p := lst.Best(nil)
 			Expect(p).To(Equal(a11))
+			// Test annotation with null map
+			Expect(a.MatchAnnotation(re)).To(Equal(false))
+		})
+	})
+
+	Context("Find label on packages", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.AddLabel("project1", "test1")
+		a.AddLabel("label2", "value1")
+		b := NewPackage("B", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b.AddLabel("project2", "test2")
+		b.AddLabel("label2", "value1")
+		It("Expands correctly", func() {
+			var err error
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b} {
+				_, err = definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			re := regexp.MustCompile("project[0-9][=].*")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(re).ToNot(BeNil())
+			Expect(a.HasLabel("label2")).To(Equal(true))
+			Expect(a.HasLabel("label3")).To(Equal(false))
+			Expect(a.HasLabel("project1")).To(Equal(true))
+			Expect(b.HasLabel("project2")).To(Equal(true))
+			Expect(b.HasLabel("label2")).To(Equal(true))
+			Expect(b.MatchLabel(re)).To(Equal(true))
+			Expect(a.MatchLabel(re)).To(Equal(true))
+
+		})
+	})
+
+	Context("Find annotations on packages", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.AddAnnotation("project1", "test1")
+		a.AddAnnotation("label2", "value1")
+		b := NewPackage("B", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b.AddAnnotation("project2", "test2")
+		b.AddAnnotation("label2", "value1")
+		It("Expands correctly", func() {
+			var err error
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b} {
+				_, err = definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			re := regexp.MustCompile("project[0-9][=].*")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(re).ToNot(BeNil())
+			Expect(a.HasAnnotation("label2")).To(Equal(true))
+			Expect(a.HasAnnotation("label3")).To(Equal(false))
+			Expect(a.HasAnnotation("project1")).To(Equal(true))
+			Expect(b.HasAnnotation("project2")).To(Equal(true))
+			Expect(b.HasAnnotation("label2")).To(Equal(true))
+			Expect(b.MatchAnnotation(re)).To(Equal(true))
+			Expect(a.MatchAnnotation(re)).To(Equal(true))
+
+		})
+	})
+
+	Context("Check description", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.SetDescription("Description A")
+
+		It("Set and get correctly a description", func() {
+			Expect(a.GetDescription()).To(Equal("Description A"))
+		})
+	})
+
+	Context("Check licenses", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.SetLicense("MIT")
+
+		It("Set and get correctly a license", func() {
+			Expect(a.GetLicense()).To(Equal("MIT"))
+		})
+	})
+
+	Context("Check URI", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.AddURI("ftp://ftp.freeradius.org/pub/radius/freearadius-server-3.0.20.tar.gz")
+
+		It("Set and get correctly an uri", func() {
+			Expect(a.GetURI()).To(Equal([]string{
+				"ftp://ftp.freeradius.org/pub/radius/freearadius-server-3.0.20.tar.gz",
+			}))
 		})
 	})
 
@@ -49,7 +157,12 @@ var _ = Describe("Package", func() {
 		c := NewPackage("C", "1.1", []*DefaultPackage{b}, []*DefaultPackage{})
 		d := NewPackage("D", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
 		It("Computes correctly", func() {
-			lst := a.Revdeps(&[]Package{a, b, c, d})
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b, c, d} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			lst := a.Revdeps(definitions)
 			Expect(lst).To(ContainElement(b))
 			Expect(lst).To(ContainElement(c))
 			Expect(len(lst)).To(Equal(2))
@@ -64,11 +177,103 @@ var _ = Describe("Package", func() {
 		e := NewPackage("E", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
 
 		It("Computes correctly", func() {
-			lst := b.Revdeps(&[]Package{a, b, c, d, e})
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b, c, d, e} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			lst := a.Revdeps(definitions)
 			Expect(lst).To(ContainElement(c))
 			Expect(lst).To(ContainElement(d))
 			Expect(lst).To(ContainElement(e))
-			Expect(len(lst)).To(Equal(3))
+			Expect(len(lst)).To(Equal(4))
+		})
+	})
+
+	Context("revdeps", func() {
+		a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b := NewPackage("B", "1.0", []*DefaultPackage{&DefaultPackage{Name: "A", Version: ">=1.0"}}, []*DefaultPackage{})
+		c := NewPackage("C", "1.1", []*DefaultPackage{&DefaultPackage{Name: "B", Version: ">=1.0"}}, []*DefaultPackage{})
+		d := NewPackage("D", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
+		e := NewPackage("E", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
+
+		It("doesn't resolve selectors", func() {
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b, c, d, e} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			lst := a.Revdeps(definitions)
+			Expect(len(lst)).To(Equal(0))
+		})
+	})
+	Context("Expandedrevdeps", func() {
+		a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b := NewPackage("B", "1.0", []*DefaultPackage{&DefaultPackage{Name: "A", Version: ">=1.0"}}, []*DefaultPackage{})
+		c := NewPackage("C", "1.1", []*DefaultPackage{&DefaultPackage{Name: "B", Version: ">=1.0"}}, []*DefaultPackage{})
+		d := NewPackage("D", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
+		e := NewPackage("E", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
+
+		It("Computes correctly", func() {
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b, c, d, e} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			visited := make(map[string]interface{})
+			lst := a.ExpandedRevdeps(definitions, visited)
+			Expect(lst).To(ContainElement(c))
+			Expect(lst).To(ContainElement(d))
+			Expect(lst).To(ContainElement(e))
+			Expect(len(lst)).To(Equal(4))
+		})
+	})
+
+	Context("Expandedrevdeps", func() {
+		a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b := NewPackage("B", "1.0", []*DefaultPackage{&DefaultPackage{Name: "A", Version: ">=1.0"}}, []*DefaultPackage{})
+		c := NewPackage("C", "1.1", []*DefaultPackage{&DefaultPackage{Name: "B", Version: ">=1.0"}}, []*DefaultPackage{})
+		d := NewPackage("D", "0.1", []*DefaultPackage{&DefaultPackage{Name: "C", Version: ">=1.0"}}, []*DefaultPackage{})
+		e := NewPackage("E", "0.1", []*DefaultPackage{&DefaultPackage{Name: "C", Version: ">=1.0"}}, []*DefaultPackage{})
+
+		It("Computes correctly", func() {
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b, c, d, e} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			visited := make(map[string]interface{})
+
+			lst := a.ExpandedRevdeps(definitions, visited)
+			Expect(lst).To(ContainElement(b))
+			Expect(lst).To(ContainElement(c))
+			Expect(lst).To(ContainElement(d))
+			Expect(lst).To(ContainElement(e))
+			Expect(len(lst)).To(Equal(4))
+		})
+	})
+
+	Context("Expandedrevdeps", func() {
+		a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b := NewPackage("B", "1.0", []*DefaultPackage{&DefaultPackage{Name: "A", Version: ">=1.0"}}, []*DefaultPackage{})
+		c := NewPackage("C", "1.1", []*DefaultPackage{&DefaultPackage{Name: "B", Version: ">=1.0"}, &DefaultPackage{Name: "A", Version: ">=0"}}, []*DefaultPackage{})
+		d := NewPackage("D", "0.1", []*DefaultPackage{&DefaultPackage{Name: "C", Version: ">=1.0"}}, []*DefaultPackage{})
+		e := NewPackage("E", "0.1", []*DefaultPackage{&DefaultPackage{Name: "C", Version: ">=1.0"}}, []*DefaultPackage{})
+
+		It("Computes correctly", func() {
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b, c, d, e} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			visited := make(map[string]interface{})
+
+			lst := a.ExpandedRevdeps(definitions, visited)
+			Expect(lst).To(ContainElement(b))
+			Expect(lst).To(ContainElement(c))
+			Expect(lst).To(ContainElement(d))
+			Expect(lst).To(ContainElement(e))
+			Expect(len(lst)).To(Equal(4))
 		})
 	})
 
@@ -77,11 +282,20 @@ var _ = Describe("Package", func() {
 		a1 := NewPackage("A", "1.0", []*DefaultPackage{a}, []*DefaultPackage{})
 		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
 		a01 := NewPackage("A", "0.1", []*DefaultPackage{a1, a11}, []*DefaultPackage{})
+
 		It("returns correctly", func() {
-			Expect(a01.RequiresContains(a1)).To(BeTrue())
-			Expect(a01.RequiresContains(a11)).To(BeTrue())
-			Expect(a01.RequiresContains(a)).To(BeTrue())
-			Expect(a.RequiresContains(a11)).ToNot(BeTrue())
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, a1, a11, a01} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			Expect(a01.RequiresContains(definitions, a1)).To(BeTrue())
+			Expect(a01.RequiresContains(definitions, a11)).To(BeTrue())
+			Expect(a01.RequiresContains(definitions, a)).To(BeTrue())
+			Expect(a.RequiresContains(definitions, a11)).ToNot(BeTrue())
+			Expect(a.IsSelector()).To(BeTrue())
+			Expect(a1.IsSelector()).To(BeFalse())
+
 		})
 	})
 
@@ -100,7 +314,6 @@ var _ = Describe("Package", func() {
 
 			Expect(p.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(p.GetName()).To(Equal(a.GetName()))
-			Expect(p.Flagged()).To(Equal(a.Flagged()))
 			Expect(p.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(p.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(p.GetRequires())).To(Equal(len(a.GetRequires())))
@@ -116,9 +329,15 @@ var _ = Describe("Package", func() {
 	Context("BuildFormula", func() {
 		It("builds empty constraints", func() {
 			db := NewInMemoryDatabase(false)
-
 			a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-			f, err := a1.BuildFormula(db)
+
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a1} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			f, err := a1.BuildFormula(definitions, db)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(f).To(BeNil())
 		})
@@ -130,11 +349,18 @@ var _ = Describe("Package", func() {
 			a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
 			a1.Requires([]*DefaultPackage{a11})
 			a1.Conflicts([]*DefaultPackage{a21})
-			f, err := a1.BuildFormula(db)
+
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a1, a21, a11} {
+				_, err := definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			f, err := a1.BuildFormula(definitions, db)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(len(f)).To(Equal(2))
-			Expect(f[0].String()).To(Equal("or(not(c31f5842), a4910f77)"))
-			Expect(f[1].String()).To(Equal("or(not(c31f5842), not(a97670be))"))
+			Expect(len(f)).To(Equal(8))
+			//	Expect(f[0].String()).To(Equal("or(not(c31f5842), a4910f77)"))
+			//	Expect(f[1].String()).To(Equal("or(not(c31f5842), not(a97670be))"))
 		})
 	})
 
@@ -147,7 +373,6 @@ var _ = Describe("Package", func() {
 			a2 := a.Clone()
 			Expect(a2.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(a2.GetName()).To(Equal(a.GetName()))
-			Expect(a2.Flagged()).To(Equal(a.Flagged()))
 			Expect(a2.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(a2.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(a2.GetRequires())).To(Equal(len(a.GetRequires())))
@@ -173,4 +398,69 @@ var _ = Describe("Package", func() {
 			Expect(len(a1.GetUses())).To(Equal(0))
 		})
 	})
+
+	Context("Check Bump build Version", func() {
+		It("Bump without build version", func() {
+			a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			err := a1.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(a1.GetVersion()).To(Equal("1.0+1"))
+		})
+		It("Bump 2", func() {
+			p := NewPackage("A", "1.0+1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+2"))
+		})
+
+		It("Bump 3", func() {
+			p := NewPackage("A", "1.0+100", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+101"))
+		})
+
+		It("Bump 4", func() {
+			p := NewPackage("A", "1.0+r1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+r2"))
+		})
+
+		It("Bump 5", func() {
+			p := NewPackage("A", "1.0+p1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+p2"))
+		})
+
+		It("Bump 6", func() {
+			p := NewPackage("A", "1.0+pre20200315", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+pre20200315.1"))
+		})
+
+		It("Bump 7", func() {
+			p := NewPackage("A", "1.0+pre20200315.1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+pre20200315.2"))
+		})
+
+		It("Bump 8", func() {
+			p := NewPackage("A", "1.0+d-r1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+d-r1.1"))
+		})
+
+		It("Bump 9", func() {
+			p := NewPackage("A", "1.0+p20200315.1", []*DefaultPackage{}, []*DefaultPackage{})
+			err := p.BumpBuildVersion()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetVersion()).To(Equal("1.0+p20200315.2"))
+		})
+	})
+
 })