Tag 0.17.4

Fixes #237

.chglog/CHANGELOG.tpl.md

@@ -0,0 +1,49 @@
+{{ if .Versions -}}
+{{ if .Unreleased.CommitGroups -}}
+{{ range .Unreleased.CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ else }}
+{{ range .Unreleased.Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+
+{{ if .CommitGroups -}}
+{{ range .CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ else }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} (https://github.com/mudler/luet/commit/{{.Hash.Short}})
+{{ end }}
+{{ end -}}
+
+{{- if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+{{ range .Notes }}
+{{ .Body }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}
+
+{{- if .Versions }}
+[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD
+{{ range .Versions -}}
+{{ if .Tag.Previous -}}
+[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}
+{{ end -}}
+{{ end -}}
+{{ end -}}

.chglog/config.yml

@@ -0,0 +1,27 @@
+style: github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://github.com/mudler/luet
+options:
+  commits:
+    # filters:
+    #   Type:
+    #     - feat
+    #     - fix
+    #     - perf
+    #     - refactor
+  commit_groups:
+     title_maps:
+       feat: Features
+       fix: Bug Fixes
+       perf: Performance Improvements
+       refactor: Code Refactoring
+       ci: Continous Integration
+  header:
+    pattern: "(.*)"
+    pattern_maps:
+      - Subject
+  notes:
+    keywords:
+      - BREAKING CHANGE

.github/workflows/release.yml

@@ -1,8 +1,11 @@
 on: push
+concurrency:
+  group: ci-${{ github.head_ref || github.ref }}-${{ github.repository }}
+  cancel-in-progress: true
 name: Build and release on push
 jobs:
-  release:
-    name: Test and Release
+  tests-integration:
+    name: Integration tests
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
@@ -14,14 +17,12 @@ jobs:
       - name: setup-docker
         uses: docker-practice/actions-setup-docker@0.0.1
       - name: Login to quay
-        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
       - name: Install deps
         run: |
             sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
             sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
             sudo chmod a+x "/usr/bin/img"
-      - name: Build test
-        run: sudo -E env "PATH=$PATH" make multiarch-build-small
       - name: Login to quay with img
         run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
       - name: Tests with Img backend
@@ -36,7 +37,58 @@ jobs:
                     env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
                     env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
                     env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
-                    make test-integration test-coverage
+                    make test-integration
+
+  tests-unit:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+      - name: Build test
+        run: sudo -E env "PATH=$PATH" make multiarch-build-small
+      - name: Tests
+        run: |
+              sudo -E \
+                    env "PATH=$PATH" \
+                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                    make test-coverage
+
+  release:
+    name: Test and Release
+    needs: ["tests-integration","tests-unit"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
       - name: Build
         run: sudo -E env "PATH=$PATH" make multiarch-build-small && sudo chmod -R 777 release/
       - name: Release

.github/workflows/test.yml

@@ -2,7 +2,31 @@
 on: pull_request
 name: Build and Test
 jobs:
-  test:
+  tests-integration:
+    strategy:
+      matrix:
+        go-version: [1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+    - name: Tests with Img backend
+      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make test-integration
+  tests-unit:
     strategy:
       matrix:
         go-version: [1.14.x]
@@ -24,7 +48,5 @@ jobs:
             sudo chmod a+x "/usr/bin/img"
     - name: Build
       run: sudo -E env "PATH=$PATH" make multiarch-build-small
-    - name: Tests with Img backend
-      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
     - name: Tests
-      run: sudo -E env "PATH=$PATH" make test-integration test-coverage
+      run: sudo -E env "PATH=$PATH" make test-coverage

CONTRIBUTING.md

@@ -0,0 +1,53 @@
+
+We love your input! We want to make contributing to this project as easy and transparent as possible, whether it's:
+
+- Reporting a bug
+- Discussing the current state of the code
+- Submitting a fix
+- Proposing new features
+- Becoming a maintainer
+
+## We Develop with Github
+We use github to host code, to track issues and feature requests, as well as accept pull requests.
+
+## Stay in touch
+
+Join us in [slack](https://luet.slack.com/join/shared_invite/enQtOTQxMjcyNDQ0MDUxLWQ5ODVlNTI1MTYzNDRkYzkyYmM1YWE5YjM0NTliNDEzNmQwMTkxNDRhNDIzM2Y5NDBlOTZjZTYxYWQyNDE4YzY#/) and hang out with the community! It will be much easier to get started and do your first steps in contributing to the project.
+
+## All Code Changes Happen Through Pull Requests
+Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests:
+
+1. Fork the repo you want to contribute to and create your branch from `develop`.
+2. If you've added code that should be tested, add tests.
+3. If you've changed APIs, update the [documentation](https://github.com/Luet-lab/docs).
+4. Ensure the test suite passes.
+5. Make sure your code lints.
+6. Issue that pull request!
+
+## Any contributions you make will be under the Software License of the repository
+In short, when you submit code changes, your submissions are understood to be under the same License that covers the project. Feel free to contact the maintainers if that's a concern.
+
+## Report bugs using Github's [issues](https://github.com/mudler/luet/issues)
+We use GitHub issues to track public bugs. Report a bug by [opening a new issue](https://github.com/mudler/luet/issues/new); it's that easy!
+
+## Write bug reports with detail, background, and sample code
+Try to be as more descriptive as possible. When opening a new issue you will be prompted to choose between a bug or a feature request, with a small template to fill details with. Be specific!
+
+**Great Bug Reports** tend to have:
+
+- A quick summary and/or background
+- Steps to reproduce
+  - Be specific!
+  - Give sample code if you can. 
+- What you expected would happen
+- What actually happens
+- Notes (possibly including why you think this might be happening, or stuff you tried that didn't work)
+
+People *love* thorough bug reports.
+
+
+## License
+By contributing, you agree that your contributions will be licensed under the project Licenses.
+
+## References
+This document was adapted from the open-source contribution guidelines from https://gist.github.com/briandk/3d2e8b3ec8daf5a27a62

README.md

@@ -1,8 +1,12 @@
+
+<p align="center">
+  <img width=150 height=150 src="https://user-images.githubusercontent.com/2420543/119691600-0293d700-be4b-11eb-827f-49ff1174a07a.png">
+</p>
+
 # luet - Container-based Package manager
 
 [![Docker Repository on Quay](https://quay.io/repository/luet/base/status "Docker Repository on Quay")](https://quay.io/repository/luet/base)
-[![Go Report Card](https://goreportcard.com/badge/github.com/mudler/luet)](https://goreportcard.com/report/github.com/mudler/luet)
-[![Build Status](https://travis-ci.org/mudler/luet.svg?branch=master)](https://travis-ci.org/mudler/luet)
+[![Build and release on push](https://github.com/mudler/luet/actions/workflows/release.yml/badge.svg)](https://github.com/mudler/luet/actions/workflows/release.yml)
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
 [![codecov](https://codecov.io/gh/mudler/luet/branch/master/graph/badge.svg)](https://codecov.io/gh/mudler/luet)
 
@@ -12,6 +16,9 @@ It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concep
 
 It is written entirely in Golang and where used as package manager, it can run in from scratch environment, with zero dependencies.
 
+[![asciicast](https://asciinema.org/a/388348.svg)](https://asciinema.org/a/388348)
+
+
 ## In a glance
 
 - Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.

cmd/build.go

@@ -16,14 +16,19 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	"github.com/mudler/luet/pkg/installer"
+
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -40,17 +45,17 @@ var buildCmd = &cobra.Command{
 	Long: `Builds one or more packages from a tree (current directory is implied):
 
 	$ luet build utils/busybox utils/yq ...
-	
+
 Builds all packages
-	
+
 	$ luet build --all
-	
+
 Builds only the leaf packages:
-	
+
 	$ luet build --full
 
 Build package revdeps:
-	
+
 	$ luet build --revdeps utils/yq
 
 Build package without dependencies (needs the images already in the host, or either need to be available online):
@@ -65,13 +70,13 @@ Build packages specifying multiple definition trees:
 		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
 		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
 		viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged"))
-		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
 		viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
 		viper.BindPFlag("all", cmd.Flags().Lookup("all"))
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
 		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		viper.BindPFlag("values", cmd.Flags().Lookup("values"))
+		viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
 
 		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
 		viper.BindPFlag("push", cmd.Flags().Lookup("push"))
@@ -79,10 +84,11 @@ Build packages specifying multiple definition trees:
 		viper.BindPFlag("wait", cmd.Flags().Lookup("wait"))
 		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
 
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSolverFlags(cmd)
+
+		LuetCfg.Viper.BindPFlag("general.show_build_output", cmd.Flags().Lookup("live-output"))
+		LuetCfg.Viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
+
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
@@ -93,10 +99,9 @@ Build packages specifying multiple definition trees:
 		privileged := viper.GetBool("privileged")
 		revdeps := viper.GetBool("revdeps")
 		all := viper.GetBool("all")
-		databaseType := viper.GetString("database")
 		compressionType := viper.GetString("compression")
 		imageRepository := viper.GetString("image-repository")
-		values := viper.GetString("values")
+		values := viper.GetStringSlice("values")
 		wait := viper.GetBool("wait")
 		push := viper.GetBool("push")
 		pull := viper.GetBool("pull")
@@ -105,79 +110,75 @@ Build packages specifying multiple definition trees:
 		onlydeps := viper.GetBool("onlydeps")
 		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
 		full, _ := cmd.Flags().GetBool("full")
+		rebuild, _ := cmd.Flags().GetBool("rebuild")
+
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		var results Results
+		backendArgs := viper.GetStringSlice("backend-args")
 
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
 			LuetCfg.GetLogging().SetLogLevel("error")
 		}
 		pretend, _ := cmd.Flags().GetBool("pretend")
-		compilerSpecs := compiler.NewLuetCompilationspecs()
+		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
+
+		compilerSpecs := compilerspec.NewLuetCompilationspecs()
 		var db pkg.PackageDatabase
 
-		compilerBackend := backend.NewBackend(backendType)
+		compilerBackend, err := compiler.NewBackend(backendType)
+		helpers.CheckErr(err)
 
-		switch databaseType {
-		case "memory":
-			db = pkg.NewInMemoryDatabase(false)
-
-		case "boltdb":
-			tmpdir, err := ioutil.TempDir("", "package")
-			if err != nil {
-				Fatal(err)
-			}
-			db = pkg.NewBoltDatabase(tmpdir)
-
-		}
+		db = pkg.NewInMemoryDatabase(false)
 		defer db.Clean()
 
 		generalRecipe := tree.NewCompilerRecipe(db)
 
+		if fromRepo {
+			if err := installer.LoadBuildTree(generalRecipe, db, LuetCfg); err != nil {
+				Warning("errors while loading trees from repositories", err.Error())
+			}
+		}
+
 		for _, src := range treePaths {
 			Info("Loading tree", src)
-
-			err := generalRecipe.Load(src)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
+			helpers.CheckErr(generalRecipe.Load(src))
 		}
 
 		Info("Building in", dst)
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		opts := util.SetSolverConfig()
+		pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		LuetCfg.GetGeneral().ShowBuildOutput = LuetCfg.Viper.GetBool("general.show_build_output")
 
-		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		Debug("Solver", opts.CompactString())
 
-		opts := compiler.NewDefaultCompilerOptions()
-		opts.SolverOptions = *LuetCfg.GetSolverOptions()
-		opts.ImageRepository = imageRepository
-		opts.PullFirst = pull
-		opts.KeepImg = keepImages
-		opts.Push = push
-		opts.OnlyDeps = onlydeps
-		opts.NoDeps = nodeps
-		opts.Wait = wait
-		opts.PackageTargetOnly = onlyTarget
-		opts.BuildValuesFile = values
-		var solverOpts solver.Options
 		if concurrent {
-			solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
+			opts.Options = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
 		} else {
-			solverOpts = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
 		}
 
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts, solverOpts)
-		luetCompiler.SetConcurrency(concurrency)
-		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
+			options.NoDeps(nodeps),
+			options.WithBackendType(backendType),
+			options.PushImages(push),
+			options.WithBuildValues(values),
+			options.WithPullRepositories(pullRepo),
+			options.WithPushRepository(imageRepository),
+			options.Rebuild(rebuild),
+			options.WithSolverOptions(*opts),
+			options.Wait(wait),
+			options.OnlyTarget(onlyTarget),
+			options.PullFirst(pull),
+			options.KeepImg(keepImages),
+			options.OnlyDeps(onlydeps),
+			options.BackendArgs(backendArgs),
+			options.Concurrency(concurrency),
+			options.WithCompressionType(compression.Implementation(compressionType)),
+		)
+
 		if full {
 			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
 			if err != nil {
@@ -190,7 +191,6 @@ Build packages specifying multiple definition trees:
 			}
 		} else if !all {
 			for _, a := range args {
-
 				pack, err := helpers.ParsePackageStr(a)
 				if err != nil {
 					Fatal("Invalid package string ", a, ": ", err.Error())
@@ -218,13 +218,13 @@ Build packages specifying multiple definition trees:
 			}
 		}
 
-		var artifact []compiler.Artifact
+		var artifact []*artifact.PackageArtifact
 		var errs []error
 		if revdeps {
 			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
 		} else if pretend {
-			toCalculate := []compiler.CompilationSpec{}
+			toCalculate := []*compilerspec.LuetCompilationSpec{}
 			if full {
 				var err error
 				toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...)
@@ -236,11 +236,12 @@ Build packages specifying multiple definition trees:
 			}
 
 			for _, sp := range toCalculate {
-				packs, err := luetCompiler.ComputeDepTree(sp)
+				ht := compiler.NewHashTree(generalRecipe.GetDatabase())
+				hashTree, err := ht.Query(luetCompiler, sp)
 				if err != nil {
 					errs = append(errs, err)
 				}
-				for _, p := range packs {
+				for _, p := range hashTree.Dependencies {
 					results.Packages = append(results.Packages,
 						PackageResult{
 							Name:       p.Package.GetName(),
@@ -274,6 +275,7 @@ Build packages specifying multiple definition trees:
 				}
 			}
 		} else {
+
 			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
 		}
 		if len(errs) != 0 {
@@ -283,7 +285,7 @@ Build packages specifying multiple definition trees:
 			Fatal("Bailing out")
 		}
 		for _, a := range artifact {
-			Info("Artifact generated:", a.GetPath())
+			Info("Artifact generated:", a.Path)
 		}
 	},
 }
@@ -293,14 +295,15 @@ func init() {
 	if err != nil {
 		Fatal(err)
 	}
+
 	buildCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the tree to use.")
 	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
-	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
-	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
+	buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
-	buildCmd.Flags().String("values", "", "Build values file to interpolate with each package")
+	buildCmd.Flags().StringSlice("values", []string{}, "Build values file to interpolate with each package")
+	buildCmd.Flags().StringSliceP("backend-args", "a", []string{}, "Backend args")
 
 	buildCmd.Flags().String("destination", filepath.Join(path, "build"), "Destination folder")
 	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip, zstd")
@@ -317,8 +320,11 @@ func init() {
 	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
-
+	buildCmd.Flags().Bool("live-output", LuetCfg.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
+	buildCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
+	buildCmd.Flags().Bool("rebuild", false, "To combine with --pull. Allows to rebuild the target package even if an image is available, against a local values file")
 	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
+	buildCmd.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")
 
 	buildCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 

cmd/cleanup.go

@@ -20,8 +20,9 @@ import (
 	"os"
 	"path/filepath"
 
-	config "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/cmd/util"
+	. "github.com/mudler/luet/pkg/config"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/spf13/cobra"
@@ -31,13 +32,16 @@ var cleanupCmd = &cobra.Command{
 	Use:   "cleanup",
 	Short: "Clean packages cache.",
 	Long:  `remove downloaded packages tarballs and clean cache directory`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var cleaned int = 0
-
+		util.SetSystemConfig()
 		// Check if cache dir exists
-		if helpers.Exists(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
+		if fileHelper.Exists(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
 
-			files, err := ioutil.ReadDir(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
+			files, err := ioutil.ReadDir(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
 			if err != nil {
 				Fatal("Error on read cachedir ", err.Error())
 			}
@@ -47,12 +51,12 @@ var cleanupCmd = &cobra.Command{
 					continue
 				}
 
-				if config.LuetCfg.GetGeneral().Debug {
+				if LuetCfg.GetGeneral().Debug {
 					Info("Removing ", file.Name())
 				}
 
 				err := os.RemoveAll(
-					filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
+					filepath.Join(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
 				if err != nil {
 					Fatal("Error on removing", file.Name())
 				}
@@ -66,5 +70,8 @@ var cleanupCmd = &cobra.Command{
 }
 
 func init() {
+	cleanupCmd.Flags().String("system-dbpath", "", "System db path")
+	cleanupCmd.Flags().String("system-target", "", "System rootpath")
+	cleanupCmd.Flags().String("system-engine", "", "System DB engine")
 	RootCmd.AddCommand(cleanupCmd)
 }

cmd/create-repo.go

@@ -18,11 +18,13 @@ import (
 	"os"
 	"path/filepath"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
-	. "github.com/mudler/luet/pkg/logger"
+
+	//	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
@@ -74,7 +76,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var err error
-		var repo installer.Repository
+		var repo *installer.LuetSystemRepository
 
 		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("output")
@@ -90,19 +92,19 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		metaName := viper.GetString("meta-filename")
 		source_repo := viper.GetString("repo")
 		backendType := viper.GetString("backend")
+		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
 
 		treeFile := installer.NewDefaultTreeRepositoryFile()
 		metaFile := installer.NewDefaultMetaRepositoryFile()
-		compilerBackend := backend.NewBackend(backendType)
+		compilerBackend, err := compiler.NewBackend(backendType)
+		helpers.CheckErr(err)
 		force := viper.GetBool("force-push")
 		imagePush := viper.GetBool("push-images")
 
 		if source_repo != "" {
 			// Search for system repository
 			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
+			helpers.CheckErr(err)
 
 			if len(treePaths) <= 0 {
 				treePaths = []string{lrepo.TreePath}
@@ -118,19 +120,23 @@ Create a repository from the metadata description defined in the luet.yaml confi
 				lrepo.Priority,
 				packages,
 				treePaths,
-				pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force)
+				pkg.NewInMemoryDatabase(false),
+				compilerBackend,
+				dst,
+				imagePush,
+				force,
+				fromRepo,
+				LuetCfg)
+			helpers.CheckErr(err)
 
 		} else {
 			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
-				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force)
-		}
-
-		if err != nil {
-			Fatal("Error: " + err.Error())
+				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force, fromRepo, LuetCfg)
+			helpers.CheckErr(err)
 		}
 
 		if treetype != "" {
-			treeFile.SetCompressionType(compiler.CompressionImplementation(treetype))
+			treeFile.SetCompressionType(compression.Implementation(treetype))
 		}
 
 		if treeName != "" {
@@ -138,7 +144,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		}
 
 		if metatype != "" {
-			metaFile.SetCompressionType(compiler.CompressionImplementation(metatype))
+			metaFile.SetCompressionType(compression.Implementation(metatype))
 		}
 
 		if metaName != "" {
@@ -149,17 +155,15 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
 		err = repo.Write(dst, reset, true)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
+		helpers.CheckErr(err)
+
 	},
 }
 
 func init() {
 	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
+	helpers.CheckErr(err)
+
 	createrepoCmd.Flags().String("packages", filepath.Join(path, "build"), "Packages folder (output from build)")
 	createrepoCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the source trees to use.")
 	createrepoCmd.Flags().String("output", filepath.Join(path, "build"), "Destination for generated archives. With 'docker' repository type, it should be an image reference (e.g 'foo/bar')")
@@ -178,6 +182,7 @@ func init() {
 	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
 	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
+	createrepoCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

cmd/database.go

@@ -36,6 +36,7 @@ func init() {
 
 	databaseGroupCmd.AddCommand(
 		NewDatabaseCreateCommand(),
+		NewDatabaseGetCommand(),
 		NewDatabaseRemoveCommand(),
 	)
 }

cmd/database/create.go

@@ -18,7 +18,9 @@ package cmd_database
 import (
 	"io/ioutil"
 
-	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/cmd/util"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -44,12 +46,11 @@ The yaml must contain the package definition, and the file list at least.
 For reference, inspect a "metadata.yaml" file generated while running "luet build"`,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 
+			util.SetSystemConfig()
 			systemDB := LuetCfg.GetSystemDB()
 
 			for _, a := range args {
@@ -57,25 +58,35 @@ For reference, inspect a "metadata.yaml" file generated while running "luet buil
 				if err != nil {
 					Fatal("Failed reading ", a, ": ", err.Error())
 				}
-				art, err := compiler.NewPackageArtifactFromYaml(dat)
+				art, err := artifact.NewPackageArtifactFromYaml(dat)
 				if err != nil {
 					Fatal("Failed reading yaml ", a, ": ", err.Error())
 				}
 
-				files := art.GetFiles()
+				files := art.Files
 
-				if _, err := systemDB.CreatePackage(art.GetCompileSpec().GetPackage()); err != nil {
+				// Check if the package is already present
+				if p, err := systemDB.FindPackage(art.CompileSpec.GetPackage()); err == nil && p.GetName() != "" {
+					Fatal("Package", art.CompileSpec.GetPackage().HumanReadableString(),
+						" already present.")
+				}
+
+				if _, err := systemDB.CreatePackage(art.CompileSpec.GetPackage()); err != nil {
 					Fatal("Failed to create ", a, ": ", err.Error())
 				}
-				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.GetCompileSpec().GetPackage().GetFingerPrint(), Files: files}); err != nil {
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.CompileSpec.GetPackage().GetFingerPrint(), Files: files}); err != nil {
 					Fatal("Failed setting package files for ", a, ": ", err.Error())
 				}
 
-				Info(art.GetCompileSpec().GetPackage().HumanReadableString(), " created")
+				Info(art.CompileSpec.GetPackage().HumanReadableString(), " created")
 			}
 
 		},
 	}
 
+	ans.Flags().String("system-dbpath", "", "System db path")
+	ans.Flags().String("system-target", "", "System rootpath")
+	ans.Flags().String("system-engine", "", "System DB engine")
+
 	return ans
 }

cmd/database/get.go

@@ -0,0 +1,87 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"fmt"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	"gopkg.in/yaml.v2"
+
+	. "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseGetCommand() *cobra.Command {
+	var c = &cobra.Command{
+		Use:   "get <package>",
+		Short: "Get a package in the system DB as yaml",
+		Long: `Get a package in the system database in the YAML format:
+
+		$ luet database get system/foo
+
+To return also files:
+		$ luet database get --files system/foo`,
+		Args: cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			util.BindSystemFlags(cmd)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			showFiles, _ := cmd.Flags().GetBool("files")
+			util.SetSystemConfig()
+
+			systemDB := LuetCfg.GetSystemDB()
+
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					continue
+				}
+
+				ps, err := systemDB.FindPackages(pack)
+				if err != nil {
+					continue
+				}
+				for _, p := range ps {
+					y, err := p.Yaml()
+					if err != nil {
+						continue
+					}
+					fmt.Println(string(y))
+					if showFiles {
+						files, err := systemDB.GetPackageFiles(p)
+						if err != nil {
+							continue
+						}
+						b, err := yaml.Marshal(files)
+						if err != nil {
+							continue
+						}
+						fmt.Println("files:\n" + string(b))
+					}
+				}
+			}
+		},
+	}
+	c.Flags().Bool("files", false, "Show package files.")
+	c.Flags().String("system-dbpath", "", "System db path")
+	c.Flags().String("system-target", "", "System rootpath")
+	c.Flags().String("system-engine", "", "System DB engine")
+
+	return c
+}

cmd/database/remove.go

@@ -19,6 +19,7 @@ import (
 	. "github.com/mudler/luet/pkg/logger"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 
 	"github.com/spf13/cobra"
@@ -36,11 +37,11 @@ This commands takes multiple packages as arguments and prunes their entries from
 `,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
+			util.SetSystemConfig()
+
 			systemDB := LuetCfg.GetSystemDB()
 
 			for _, a := range args {
@@ -60,6 +61,9 @@ This commands takes multiple packages as arguments and prunes their entries from
 
 		},
 	}
+	ans.Flags().String("system-dbpath", "", "System db path")
+	ans.Flags().String("system-target", "", "System rootpath")
+	ans.Flags().String("system-engine", "", "System DB engine")
 
 	return ans
 }

cmd/helpers/cli.go

@@ -22,6 +22,8 @@ import (
 	"regexp"
 	"strings"
 
+	. "github.com/mudler/luet/pkg/logger"
+
 	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	pkg "github.com/mudler/luet/pkg/package"
 	version "github.com/mudler/luet/pkg/versioner"
@@ -56,7 +58,8 @@ func packageData(p string) (string, string) {
 }
 func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
 
-	if !strings.HasPrefix(p, "=") {
+	if !(strings.HasPrefix(p, "=") || strings.HasPrefix(p, ">") ||
+		strings.HasPrefix(p, "<")) {
 		ver := ">=0"
 		cat := ""
 		name := ""
@@ -111,3 +114,9 @@ func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
 
 	return pack, nil
 }
+
+func CheckErr(err error) {
+	if err != nil {
+		Fatal(err)
+	}
+}

cmd/helpers/cli_test.go

@@ -66,5 +66,37 @@ var _ = Describe("CLI Helpers", func() {
 			Expect(pack.GetCategory()).To(Equal("cat"))
 			Expect(pack.GetVersion()).To(Equal("1.2"))
 		})
+
+		It("accept gentoo regex parsing with with condition", func() {
+			pack, err := ParsePackageStr(">=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition2", func() {
+			pack, err := ParsePackageStr("<cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("<1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition3", func() {
+			pack, err := ParsePackageStr(">cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition4", func() {
+			pack, err := ParsePackageStr("<=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("<=1.2"))
+		})
 	})
 })

cmd/install.go

@@ -15,12 +15,11 @@
 package cmd
 
 import (
-	"os"
-
 	installer "github.com/mudler/luet/pkg/installer"
 	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -49,12 +48,8 @@ To force install a package:
 `,
 	Aliases: []string{"i"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
@@ -71,30 +66,15 @@ To force install a package:
 			toInstall = append(toInstall, pack)
 		}
 
-		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
-		repos := installer.Repositories{}
-		for _, repo := range LuetCfg.SystemRepositories {
-			if !repo.Enable {
-				continue
-			}
-			r := installer.NewSystemRepository(repo)
-			repos = append(repos, r)
-		}
-
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps := LuetCfg.Viper.GetBool("nodeps")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		util.SetSystemConfig()
+		util.SetSolverConfig()
 
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
@@ -103,6 +83,7 @@ To force install a package:
 		}
 
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		repos := installer.SystemRepositories(LuetCfg)
 
 		// Load config protect configs
 		installer.LoadConfigProtectConfs(LuetCfg)
@@ -114,6 +95,7 @@ To force install a package:
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
 			PreserveSystemEssentialData: true,
+			DownloadOnly:                downloadOnly,
 			Ask:                         !yes,
 		})
 		inst.Repositories(repos)
@@ -127,12 +109,10 @@ To force install a package:
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	installCmd.Flags().String("system-dbpath", path, "System db path")
-	installCmd.Flags().String("system-target", path, "System rootpath")
+	installCmd.Flags().String("system-dbpath", "", "System db path")
+	installCmd.Flags().String("system-target", "", "System rootpath")
+	installCmd.Flags().String("system-engine", "", "System DB engine")
+
 	installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
 	installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
@@ -142,6 +122,7 @@ func init() {
 	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
 	installCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	installCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	installCmd.Flags().Bool("download-only", false, "Download only")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/pack.go

@@ -20,7 +20,9 @@ import (
 	"time"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
-	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -64,21 +66,21 @@ Afterwards, you can use the content generated and associate it with a tree and a
 			Fatal("Invalid package string ", packageName, ": ", err.Error())
 		}
 
-		spec := &compiler.LuetCompilationSpec{Package: p}
-		artifact := compiler.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
-		artifact.SetCompressionType(compiler.CompressionImplementation(compressionType))
-		err = artifact.Compress(sourcePath, concurrency)
+		spec := &compilerspec.LuetCompilationSpec{Package: p}
+		a := artifact.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
+		a.CompressionType = compression.Implementation(compressionType)
+		err = a.Compress(sourcePath, concurrency)
 		if err != nil {
 			Fatal("failed compressing ", packageName, ": ", err.Error())
 		}
-		artifact.SetCompileSpec(spec)
-		filelist, err := artifact.FileList()
+		a.CompileSpec = spec
+		filelist, err := a.FileList()
 		if err != nil {
 			Fatal("failed generating file list for ", packageName, ": ", err.Error())
 		}
-		artifact.SetFiles(filelist)
-		artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
-		err = artifact.WriteYaml(dst)
+		a.Files = filelist
+		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
+		err = a.WriteYaml(dst)
 		if err != nil {
 			Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
 		}

cmd/reclaim.go

@@ -15,8 +15,7 @@
 package cmd
 
 import (
-	"os"
-
+	"github.com/mudler/luet/cmd/util"
 	installer "github.com/mudler/luet/pkg/installer"
 
 	. "github.com/mudler/luet/pkg/config"
@@ -29,8 +28,7 @@ var reclaimCmd = &cobra.Command{
 	Use:   "reclaim",
 	Short: "Reclaim packages to Luet database from available repositories",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		util.BindSystemFlags(cmd)
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Long: `Reclaim tries to find association between packages in the online repositories and the system one.
@@ -40,6 +38,7 @@ var reclaimCmd = &cobra.Command{
 It scans the target file system, and if finds a match with a package available in the repositories, it marks as installed in the system database.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
+		util.SetSystemConfig()
 
 		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
 		repos := installer.Repositories{}
@@ -71,12 +70,11 @@ It scans the target file system, and if finds a match with a package available i
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	reclaimCmd.Flags().String("system-dbpath", path, "System db path")
-	reclaimCmd.Flags().String("system-target", path, "System rootpath")
+
+	reclaimCmd.Flags().String("system-dbpath", "", "System db path")
+	reclaimCmd.Flags().String("system-target", "", "System rootpath")
+	reclaimCmd.Flags().String("system-engine", "", "System DB engine")
+
 	reclaimCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
 
 	RootCmd.AddCommand(reclaimCmd)

cmd/reinstall.go

@@ -0,0 +1,128 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+var reinstallCmd = &cobra.Command{
+	Use:   "reinstall <pkg1> <pkg2> <pkg3>",
+	Short: "reinstall a set of packages",
+	Long: `Reinstall a group of packages in the system:
+
+	$ luet reinstall -y system/busybox shells/bash system/coreutils ...
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("for", cmd.Flags().Lookup("for"))
+
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var toUninstall pkg.Packages
+		var toAdd pkg.Packages
+
+		force := LuetCfg.Viper.GetBool("force")
+		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
+
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		util.SetSystemConfig()
+
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toUninstall = append(toUninstall, pack)
+			toAdd = append(toAdd, pack)
+		}
+
+		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		util.SetSolverConfig()
+
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      true,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
+		})
+		inst.Repositories(repos)
+
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Swap(toUninstall, toAdd, system)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+
+	reinstallCmd.Flags().String("system-dbpath", "", "System db path")
+	reinstallCmd.Flags().String("system-target", "", "System rootpath")
+	reinstallCmd.Flags().String("system-engine", "", "System DB engine")
+
+	reinstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	reinstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	reinstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	reinstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	reinstallCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	reinstallCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	reinstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	reinstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	reinstallCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(reinstallCmd)
+}

cmd/replace.go

@@ -15,12 +15,11 @@
 package cmd
 
 import (
-	"os"
-
 	installer "github.com/mudler/luet/pkg/installer"
 	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -37,12 +36,8 @@ var replaceCmd = &cobra.Command{
 	$ luet replace -y system/busybox ... --for shells/bash --for system/coreutils ...
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
@@ -55,16 +50,15 @@ var replaceCmd = &cobra.Command{
 		var toAdd pkg.Packages
 
 		f := LuetCfg.Viper.GetStringSlice("for")
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps := LuetCfg.Viper.GetBool("nodeps")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
+		util.SetSystemConfig()
+		util.SetSolverConfig()
 		for _, a := range args {
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
@@ -91,11 +85,6 @@ var replaceCmd = &cobra.Command{
 			repos = append(repos, r)
 		}
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
 		} else {
@@ -115,6 +104,7 @@ var replaceCmd = &cobra.Command{
 			OnlyDeps:                    onlydeps,
 			PreserveSystemEssentialData: true,
 			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
 		})
 		inst.Repositories(repos)
 
@@ -127,12 +117,11 @@ var replaceCmd = &cobra.Command{
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	replaceCmd.Flags().String("system-dbpath", path, "System db path")
-	replaceCmd.Flags().String("system-target", path, "System rootpath")
+
+	replaceCmd.Flags().String("system-dbpath", "", "System db path")
+	replaceCmd.Flags().String("system-target", "", "System rootpath")
+	replaceCmd.Flags().String("system-engine", "", "System DB engine")
+
 	replaceCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
 	replaceCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	replaceCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
@@ -143,6 +132,7 @@ func init() {
 	replaceCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	replaceCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 	replaceCmd.Flags().StringSlice("for", []string{}, "Packages that has to be installed in place of others")
+	replaceCmd.Flags().Bool("download-only", false, "Download only")
 
 	RootCmd.AddCommand(replaceCmd)
 }

cmd/repo/list.go

@@ -72,8 +72,8 @@ func NewRepoListCommand() *cobra.Command {
 					if repo.Cached {
 
 						r := installer.NewSystemRepository(repo)
-						localRepo, _ := r.(*installer.LuetSystemRepository).ReadSpecFile(filepath.Join(repobasedir,
-							installer.REPOSITORY_SPECFILE), false)
+						localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir,
+							installer.REPOSITORY_SPECFILE))
 						if localRepo != nil {
 							tsec, _ := strconv.ParseInt(localRepo.GetLastUpdate(), 10, 64)
 							repoRevision = Bold(Red(localRepo.GetRevision())).String() +

cmd/root.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/marcsauter/single"
 	bus "github.com/mudler/luet/pkg/bus"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	extensions "github.com/mudler/cobra-extensions"
 	config "github.com/mudler/luet/pkg/config"
@@ -40,8 +41,14 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.10.2"
+	LuetCLIVersion = "0.17.4"
 	LuetEnvPrefix  = "LUET"
+	license        = `
+	Luet Copyright (C) 2019-2021 Ettore Di Giacinto
+	This program comes with ABSOLUTELY NO WARRANTY.
+	This is free software, and you are welcome to redistribute it
+	under certain conditions.
+	`
 )
 
 // Build time and commit information.
@@ -52,6 +59,47 @@ var (
 	BuildCommit string
 )
 
+func version() string {
+	return fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime)
+}
+
+var noBannerCommands = []string{"search", "exec", "tree", "database", "box", "cleanup"}
+
+func displayVersionBanner() {
+	display := true
+	if len(os.Args) > 1 {
+		for _, c := range noBannerCommands {
+			if os.Args[1] == c {
+				display = false
+			}
+		}
+	}
+	if display {
+		Info("Luet version", version())
+		Info(license)
+	}
+}
+
+func handleLock() {
+	if os.Getenv("LUET_NOLOCK") != "true" {
+		if len(os.Args) > 1 {
+			for _, lockedCmd := range LockedCommands {
+				if os.Args[1] == lockedCmd {
+					s := single.New("luet")
+					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
+						Fatal("another instance of the app is already running, exiting")
+					} else if err != nil {
+						// Another error occurred, might be worth handling it as well
+						Fatal("failed to acquire exclusive app lock:", err.Error())
+					}
+					defer s.TryUnlock()
+					break
+				}
+			}
+		}
+	}
+}
+
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
 	Use:   "luet",
@@ -79,8 +127,9 @@ To build a package, from a tree definition:
 	$ luet build --tree tree/path package
 	
 `,
-	Version: fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime),
+	Version: version(),
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+
 		err := LoadConfig(config.LuetCfg)
 		if err != nil {
 			Fatal("failed to load configuration:", err.Error())
@@ -97,7 +146,7 @@ To build a package, from a tree definition:
 
 		plugin := viper.GetStringSlice("plugin")
 
-		bus.Manager.Load(plugin...).Register()
+		bus.Manager.Initialize(plugin...)
 		if len(bus.Manager.Plugins) != 0 {
 			Info(":lollipop:Enabled plugins:")
 			for _, p := range bus.Manager.Plugins {
@@ -154,23 +203,8 @@ func LoadConfig(c *config.LuetConfig) error {
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
 
-	if os.Getenv("LUET_NOLOCK") != "true" {
-		if len(os.Args) > 1 {
-			for _, lockedCmd := range LockedCommands {
-				if os.Args[1] == lockedCmd {
-					s := single.New("luet")
-					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
-						Fatal("another instance of the app is already running, exiting")
-					} else if err != nil {
-						// Another error occurred, might be worth handling it as well
-						Fatal("failed to acquire exclusive app lock:", err.Error())
-					}
-					defer s.TryUnlock()
-					break
-				}
-			}
-		}
-	}
+	handleLock()
+	displayVersionBanner()
 
 	if err := RootCmd.Execute(); err != nil {
 		fmt.Println(err)
@@ -252,7 +286,7 @@ func initConfig() {
 		}
 		homeDir := helpers.GetHomeDir()
 
-		if helpers.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && helpers.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
+		if fileHelper.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && fileHelper.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
 			viper.AddConfigPath(".")
 			if homeDir != "" {
 				viper.AddConfigPath(homeDir)

cmd/search.go

@@ -16,12 +16,12 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 	"strings"
 
 	"github.com/ghodss/yaml"
 	"github.com/jedib0t/go-pretty/table"
 	"github.com/jedib0t/go-pretty/v6/list"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -30,12 +30,13 @@ import (
 )
 
 type PackageResult struct {
-	Name       string `json:"name"`
-	Category   string `json:"category"`
-	Version    string `json:"version"`
-	Repository string `json:"repository"`
-	Target     string `json:"target"`
-	Hidden     bool   `json:"hidden"`
+	Name       string   `json:"name"`
+	Category   string   `json:"category"`
+	Version    string   `json:"version"`
+	Repository string   `json:"repository"`
+	Target     string   `json:"target"`
+	Hidden     bool     `json:"hidden"`
+	Files      []string `json:"files"`
 }
 
 type Results struct {
@@ -64,6 +65,210 @@ func packageToList(l list.Writer, repo string, p pkg.Package) {
 	l.UnIndent()
 }
 
+func searchLocally(term string, l list.Writer, t table.Writer, label, labelMatch, revdeps, hidden bool) Results {
+	var results Results
+
+	system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+
+	var err error
+	iMatches := pkg.Packages{}
+	if label {
+		iMatches, err = system.Database.FindPackageLabel(term)
+	} else if labelMatch {
+		iMatches, err = system.Database.FindPackageLabelMatch(term)
+	} else {
+		iMatches, err = system.Database.FindPackageMatch(term)
+	}
+
+	if err != nil {
+		Fatal("Error: " + err.Error())
+	}
+
+	for _, pack := range iMatches {
+		if !revdeps {
+			if !pack.IsHidden() || pack.IsHidden() && hidden {
+
+				t.AppendRow(packageToRow("system", pack))
+				packageToList(l, "system", pack)
+				f, _ := system.Database.GetPackageFiles(pack)
+				results.Packages = append(results.Packages,
+					PackageResult{
+						Name:       pack.GetName(),
+						Version:    pack.GetVersion(),
+						Category:   pack.GetCategory(),
+						Repository: "system",
+						Hidden:     pack.IsHidden(),
+						Files:      f,
+					})
+			}
+		} else {
+
+			packs, _ := system.Database.GetRevdeps(pack)
+			for _, revdep := range packs {
+				if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+					t.AppendRow(packageToRow("system", pack))
+					packageToList(l, "system", pack)
+					f, _ := system.Database.GetPackageFiles(revdep)
+					results.Packages = append(results.Packages,
+						PackageResult{
+							Name:       revdep.GetName(),
+							Version:    revdep.GetVersion(),
+							Category:   revdep.GetCategory(),
+							Repository: "system",
+							Hidden:     revdep.IsHidden(),
+							Files:      f,
+						})
+				}
+			}
+		}
+	}
+	return results
+
+}
+func searchOnline(term string, l list.Writer, t table.Writer, label, labelMatch, revdeps, hidden bool) Results {
+	var results Results
+
+	repos := installer.Repositories{}
+	for _, repo := range LuetCfg.SystemRepositories {
+		if !repo.Enable {
+			continue
+		}
+		r := installer.NewSystemRepository(repo)
+		repos = append(repos, r)
+	}
+
+	inst := installer.NewLuetInstaller(
+		installer.LuetInstallerOptions{
+			Concurrency:   LuetCfg.GetGeneral().Concurrency,
+			SolverOptions: *LuetCfg.GetSolverOptions(),
+		},
+	)
+	inst.Repositories(repos)
+	synced, err := inst.SyncRepositories(false)
+	if err != nil {
+		Fatal("Error: " + err.Error())
+	}
+
+	Info("--- Search results (" + term + "): ---")
+
+	matches := []installer.PackageMatch{}
+	if label {
+		matches = synced.SearchLabel(term)
+	} else if labelMatch {
+		matches = synced.SearchLabelMatch(term)
+	} else {
+		matches = synced.Search(term)
+	}
+
+	for _, m := range matches {
+		if !revdeps {
+			if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
+				t.AppendRow(packageToRow(m.Repo.GetName(), m.Package))
+				packageToList(l, m.Repo.GetName(), m.Package)
+				r := &PackageResult{
+					Name:       m.Package.GetName(),
+					Version:    m.Package.GetVersion(),
+					Category:   m.Package.GetCategory(),
+					Repository: m.Repo.GetName(),
+					Hidden:     m.Package.IsHidden(),
+				}
+				if m.Artifact != nil {
+					r.Files = m.Artifact.Files
+				}
+				results.Packages = append(results.Packages, *r)
+			}
+		} else {
+			packs, _ := m.Repo.GetTree().GetDatabase().GetRevdeps(m.Package)
+			for _, revdep := range packs {
+				if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+					t.AppendRow(packageToRow(m.Repo.GetName(), revdep))
+					packageToList(l, m.Repo.GetName(), revdep)
+					r := &PackageResult{
+						Name:       revdep.GetName(),
+						Version:    revdep.GetVersion(),
+						Category:   revdep.GetCategory(),
+						Repository: m.Repo.GetName(),
+						Hidden:     revdep.IsHidden(),
+					}
+					if m.Artifact != nil {
+						r.Files = m.Artifact.Files
+					}
+					results.Packages = append(results.Packages, *r)
+				}
+			}
+		}
+	}
+	return results
+}
+func searchLocalFiles(term string, l list.Writer, t table.Writer) Results {
+	var results Results
+	Info("--- Search results (" + term + "): ---")
+
+	matches, _ := LuetCfg.GetSystemDB().FindPackageByFile(term)
+	for _, pack := range matches {
+		t.AppendRow(packageToRow("system", pack))
+		packageToList(l, "system", pack)
+		f, _ := LuetCfg.GetSystemDB().GetPackageFiles(pack)
+		results.Packages = append(results.Packages,
+			PackageResult{
+				Name:       pack.GetName(),
+				Version:    pack.GetVersion(),
+				Category:   pack.GetCategory(),
+				Repository: "system",
+				Hidden:     pack.IsHidden(),
+				Files:      f,
+			})
+	}
+
+	return results
+}
+
+func searchFiles(term string, l list.Writer, t table.Writer) Results {
+	var results Results
+
+	repos := installer.Repositories{}
+	for _, repo := range LuetCfg.SystemRepositories {
+		if !repo.Enable {
+			continue
+		}
+		r := installer.NewSystemRepository(repo)
+		repos = append(repos, r)
+	}
+
+	inst := installer.NewLuetInstaller(
+		installer.LuetInstallerOptions{
+			Concurrency:   LuetCfg.GetGeneral().Concurrency,
+			SolverOptions: *LuetCfg.GetSolverOptions(),
+		},
+	)
+	inst.Repositories(repos)
+	synced, err := inst.SyncRepositories(false)
+	if err != nil {
+		Fatal("Error: " + err.Error())
+	}
+
+	Info("--- Search results (" + term + "): ---")
+
+	matches := []installer.PackageMatch{}
+
+	matches = synced.SearchPackages(term, installer.FileSearch)
+
+	for _, m := range matches {
+		t.AppendRow(packageToRow(m.Repo.GetName(), m.Package))
+		packageToList(l, m.Repo.GetName(), m.Package)
+		results.Packages = append(results.Packages,
+			PackageResult{
+				Name:       m.Package.GetName(),
+				Version:    m.Package.GetVersion(),
+				Category:   m.Package.GetCategory(),
+				Repository: m.Repo.GetName(),
+				Hidden:     m.Package.IsHidden(),
+				Files:      m.Artifact.Files,
+			})
+	}
+	return results
+}
+
 var searchCmd = &cobra.Command{
 	Use:   "search <term>",
 	Short: "Search packages",
@@ -102,13 +307,9 @@ Search can also return results in the terminal in different ways: as terminal ou
 `,
 	Aliases: []string{"s"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var results Results
@@ -120,145 +321,34 @@ Search can also return results in the terminal in different ways: as terminal ou
 		hidden, _ := cmd.Flags().GetBool("hidden")
 
 		installed := LuetCfg.Viper.GetBool("installed")
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		searchWithLabel, _ := cmd.Flags().GetBool("by-label")
 		searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex")
 		revdeps, _ := cmd.Flags().GetBool("revdeps")
 		tableMode, _ := cmd.Flags().GetBool("table")
+		files, _ := cmd.Flags().GetBool("files")
+
+		util.SetSystemConfig()
+		util.SetSolverConfig()
 
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
 			LuetCfg.GetLogging().SetLogLevel("error")
 		}
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-
 		l := list.NewWriter()
 		t := table.NewWriter()
 		t.AppendHeader(rows)
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
-		if !installed {
-
-			repos := installer.Repositories{}
-			for _, repo := range LuetCfg.SystemRepositories {
-				if !repo.Enable {
-					continue
-				}
-				r := installer.NewSystemRepository(repo)
-				repos = append(repos, r)
-			}
-
-			inst := installer.NewLuetInstaller(
-				installer.LuetInstallerOptions{
-					Concurrency:   LuetCfg.GetGeneral().Concurrency,
-					SolverOptions: *LuetCfg.GetSolverOptions(),
-				},
-			)
-			inst.Repositories(repos)
-			synced, err := inst.SyncRepositories(false)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-
-			Info("--- Search results (" + args[0] + "): ---")
-
-			matches := []installer.PackageMatch{}
-			if searchWithLabel {
-				matches = synced.SearchLabel(args[0])
-			} else if searchWithLabelMatch {
-				matches = synced.SearchLabelMatch(args[0])
-			} else {
-				matches = synced.Search(args[0])
-			}
-			for _, m := range matches {
-				if !revdeps {
-					if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
-						t.AppendRow(packageToRow(m.Repo.GetName(), m.Package))
-						packageToList(l, m.Repo.GetName(), m.Package)
-						results.Packages = append(results.Packages,
-							PackageResult{
-								Name:       m.Package.GetName(),
-								Version:    m.Package.GetVersion(),
-								Category:   m.Package.GetCategory(),
-								Repository: m.Repo.GetName(),
-								Hidden:     m.Package.IsHidden(),
-							})
-					}
-				} else {
-					packs, _ := m.Repo.GetTree().GetDatabase().GetRevdeps(m.Package)
-					for _, revdep := range packs {
-						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
-							t.AppendRow(packageToRow(m.Repo.GetName(), revdep))
-							packageToList(l, m.Repo.GetName(), revdep)
-							results.Packages = append(results.Packages,
-								PackageResult{
-									Name:       revdep.GetName(),
-									Version:    revdep.GetVersion(),
-									Category:   revdep.GetCategory(),
-									Repository: m.Repo.GetName(),
-									Hidden:     revdep.IsHidden(),
-								})
-						}
-					}
-				}
-			}
-		} else {
-
-			system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
-
-			var err error
-			iMatches := pkg.Packages{}
-			if searchWithLabel {
-				iMatches, err = system.Database.FindPackageLabel(args[0])
-			} else if searchWithLabelMatch {
-				iMatches, err = system.Database.FindPackageLabelMatch(args[0])
-			} else {
-				iMatches, err = system.Database.FindPackageMatch(args[0])
-			}
-
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-
-			for _, pack := range iMatches {
-				if !revdeps {
-					if !pack.IsHidden() || pack.IsHidden() && hidden {
-						t.AppendRow(packageToRow("system", pack))
-						packageToList(l, "system", pack)
-						results.Packages = append(results.Packages,
-							PackageResult{
-								Name:       pack.GetName(),
-								Version:    pack.GetVersion(),
-								Category:   pack.GetCategory(),
-								Repository: "system",
-								Hidden:     pack.IsHidden(),
-							})
-					}
-				} else {
-					packs, _ := system.Database.GetRevdeps(pack)
-					for _, revdep := range packs {
-						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
-							t.AppendRow(packageToRow("system", pack))
-							packageToList(l, "system", pack)
-							results.Packages = append(results.Packages,
-								PackageResult{
-									Name:       revdep.GetName(),
-									Version:    revdep.GetVersion(),
-									Category:   revdep.GetCategory(),
-									Repository: "system",
-									Hidden:     revdep.IsHidden(),
-								})
-						}
-					}
-				}
-			}
+		switch {
+		case files && installed:
+			results = searchLocalFiles(args[0], l, t)
+		case files && !installed:
+			results = searchFiles(args[0], l, t)
+		case !installed:
+			results = searchOnline(args[0], l, t, searchWithLabel, searchWithLabelMatch, revdeps, hidden)
+		default:
+			results = searchLocally(args[0], l, t, searchWithLabel, searchWithLabelMatch, revdeps, hidden)
 		}
 
 		t.AppendFooter(rows)
@@ -292,12 +382,10 @@ Search can also return results in the terminal in different ways: as terminal ou
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	searchCmd.Flags().String("system-dbpath", path, "System db path")
-	searchCmd.Flags().String("system-target", path, "System rootpath")
+	searchCmd.Flags().String("system-dbpath", "", "System db path")
+	searchCmd.Flags().String("system-target", "", "System rootpath")
+	searchCmd.Flags().String("system-engine", "", "System DB engine")
+
 	searchCmd.Flags().Bool("installed", false, "Search between system packages")
 	searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
 	searchCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
@@ -309,6 +397,7 @@ func init() {
 	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
 	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
 	searchCmd.Flags().Bool("table", false, "show output in a table (wider screens)")
+	searchCmd.Flags().Bool("files", false, "Search between packages files")
 
 	RootCmd.AddCommand(searchCmd)
 }

cmd/tree/images.go

@@ -25,6 +25,7 @@ import (
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/options"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -57,6 +58,7 @@ func NewTreeImageCommand() *cobra.Command {
 
 			treePath, _ := cmd.Flags().GetStringArray("tree")
 			imageRepository := viper.GetString("image-repository")
+			pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
 			out, _ := cmd.Flags().GetString("output")
 			if out != "terminal" {
@@ -73,12 +75,15 @@ func NewTreeImageCommand() *cobra.Command {
 			}
 			compilerBackend := backend.NewSimpleDockerBackend()
 
-			opts := compiler.NewDefaultCompilerOptions()
-			opts.SolverOptions = *LuetCfg.GetSolverOptions()
-			opts.ImageRepository = imageRepository
-
-			solverOpts := solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
-			luetCompiler := compiler.NewLuetCompiler(compilerBackend, reciper.GetDatabase(), opts, solverOpts)
+			opts := *LuetCfg.GetSolverOptions()
+			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
+			luetCompiler := compiler.NewLuetCompiler(
+				compilerBackend,
+				reciper.GetDatabase(),
+				options.WithPushRepository(imageRepository),
+				options.WithPullRepositories(pullRepo),
+				options.WithSolverOptions(opts),
+			)
 
 			a := args[0]
 
@@ -91,9 +96,14 @@ func NewTreeImageCommand() *cobra.Command {
 			if err != nil {
 				Fatal("Error: " + err.Error())
 			}
-			asserts, err := luetCompiler.ComputeDepTree(spec)
 
-			for _, assertion := range asserts { //highly dependent on the order
+			ht := compiler.NewHashTree(reciper.GetDatabase())
+			hashtree, err := ht.Query(luetCompiler, spec)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
+			for _, assertion := range hashtree.Solution { //highly dependent on the order
 
 				//buildImageHash := imageRepository + ":" + assertion.Hash.BuildHash
 				currentPackageImageHash := imageRepository + ":" + assertion.Hash.PackageHash
@@ -135,6 +145,7 @@ func NewTreeImageCommand() *cobra.Command {
 	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
 	ans.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+	ans.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")
 
 	return ans
 }

cmd/tree/validate.go

@@ -244,10 +244,33 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 				Spinner(32)
 				solution, err := depSolver.Install(pkg.Packages{r})
 				ass := solution.SearchByName(r.GetPackageName())
-				if err == nil {
-					_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
-				}
 				SpinnerStop()
+				if err == nil {
+					if ass == nil {
+
+						ans = errors.New(
+							fmt.Sprintf("[%9s] %s/%s-%s: solution doesn't retrieve package %s/%s-%s.",
+								checkType,
+								p.GetCategory(), p.GetName(), p.GetVersion(),
+								r.GetCategory(), r.GetName(), r.GetVersion(),
+							))
+
+						if LuetCfg.GetGeneral().Debug {
+							for idx, pa := range solution {
+								fmt.Println(fmt.Sprintf("[%9s] %s/%s-%s: solution %d: %s",
+									checkType,
+									p.GetCategory(), p.GetName(), p.GetVersion(), idx,
+									pa.Package.GetPackageName()))
+							}
+						}
+
+						Error(ans.Error())
+						opts.IncrBrokenDeps()
+						validpkg = false
+					} else {
+						_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+					}
+				}
 
 				if err != nil {
 

cmd/uninstall.go

@@ -15,9 +15,8 @@
 package cmd
 
 import (
-	"os"
-
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -33,12 +32,8 @@ var uninstallCmd = &cobra.Command{
 	Long:    `Uninstall packages`,
 	Aliases: []string{"rm", "un"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
@@ -54,10 +49,6 @@ var uninstallCmd = &cobra.Command{
 			toRemove = append(toRemove, pack)
 		}
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps, _ := cmd.Flags().GetBool("nodeps")
 		full, _ := cmd.Flags().GetBool("full")
@@ -65,11 +56,12 @@ var uninstallCmd = &cobra.Command{
 		fullClean, _ := cmd.Flags().GetBool("full-clean")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
+		keepProtected, _ := cmd.Flags().GetBool("keep-protected-files")
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		util.SetSystemConfig()
+		util.SetSolverConfig()
+
+		LuetCfg.ConfigProtectSkip = !keepProtected
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
 		} else {
@@ -101,12 +93,11 @@ var uninstallCmd = &cobra.Command{
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	uninstallCmd.Flags().String("system-dbpath", path, "System db path")
-	uninstallCmd.Flags().String("system-target", path, "System rootpath")
+
+	uninstallCmd.Flags().String("system-dbpath", "", "System db path")
+	uninstallCmd.Flags().String("system-target", "", "System rootpath")
+	uninstallCmd.Flags().String("system-engine", "", "System DB engine")
+
 	uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
 	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
@@ -118,6 +109,7 @@ func init() {
 	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
 	uninstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	uninstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	uninstallCmd.Flags().BoolP("keep-protected-files", "k", false, "Keep package protected files around")
 
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -15,8 +15,7 @@
 package cmd
 
 import (
-	"os"
-
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -30,12 +29,8 @@ var upgradeCmd = &cobra.Command{
 	Short:   "Upgrades the system",
 	Aliases: []string{"u"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", installCmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", installCmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
@@ -52,10 +47,6 @@ var upgradeCmd = &cobra.Command{
 			repos = append(repos, r)
 		}
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps, _ := cmd.Flags().GetBool("nodeps")
 		full, _ := cmd.Flags().GetBool("full")
@@ -64,11 +55,11 @@ var upgradeCmd = &cobra.Command{
 		sync, _ := cmd.Flags().GetBool("sync")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		util.SetSystemConfig()
+		util.SetSolverConfig()
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
 		} else {
@@ -91,6 +82,7 @@ var upgradeCmd = &cobra.Command{
 			UpgradeNewRevisions:         sync,
 			PreserveSystemEssentialData: true,
 			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
 		})
 		inst.Repositories(repos)
 
@@ -102,12 +94,10 @@ var upgradeCmd = &cobra.Command{
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	upgradeCmd.Flags().String("system-dbpath", path, "System db path")
-	upgradeCmd.Flags().String("system-target", path, "System rootpath")
+	upgradeCmd.Flags().String("system-dbpath", "", "System db path")
+	upgradeCmd.Flags().String("system-target", "", "System rootpath")
+	upgradeCmd.Flags().String("system-engine", "", "System DB engine")
+
 	upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
 	upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
@@ -120,6 +110,7 @@ func init() {
 	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
 	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	upgradeCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	upgradeCmd.Flags().Bool("download-only", false, "Download only")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

cmd/util.go

@@ -0,0 +1,36 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/util"
+
+	"github.com/spf13/cobra"
+)
+
+var utilGroup = &cobra.Command{
+	Use:   "util [command] [OPTIONS]",
+	Short: "General luet internal utilities exposed",
+}
+
+func init() {
+	RootCmd.AddCommand(utilGroup)
+
+	utilGroup.AddCommand(
+		NewUnpackCommand(),
+	)
+}

cmd/util/cli.go

@@ -0,0 +1,65 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/config"
+)
+
+func BindSystemFlags(cmd *cobra.Command) {
+	LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+	LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+	LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+}
+
+func BindSolverFlags(cmd *cobra.Command) {
+	LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+	LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+	LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+	LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+}
+
+func SetSystemConfig() {
+	dbpath := LuetCfg.Viper.GetString("system.database_path")
+	rootfs := LuetCfg.Viper.GetString("system.rootfs")
+	engine := LuetCfg.Viper.GetString("system.database_engine")
+
+	LuetCfg.System.DatabaseEngine = engine
+	LuetCfg.System.DatabasePath = dbpath
+	LuetCfg.System.SetRootFS(rootfs)
+}
+
+func SetSolverConfig() (c *config.LuetSolverOptions) {
+	stype := LuetCfg.Viper.GetString("solver.type")
+	discount := LuetCfg.Viper.GetFloat64("solver.discount")
+	rate := LuetCfg.Viper.GetFloat64("solver.rate")
+	attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+
+	LuetCfg.GetSolverOptions().Type = stype
+	LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+	LuetCfg.GetSolverOptions().Discount = float32(discount)
+	LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+	return &config.LuetSolverOptions{
+		Type:        stype,
+		LearnRate:   float32(rate),
+		Discount:    float32(discount),
+		MaxAttempts: attempts,
+	}
+}

cmd/util/unpack.go

@@ -0,0 +1,98 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/go-units"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers/docker"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+func NewUnpackCommand() *cobra.Command {
+
+	c := &cobra.Command{
+		Use:   "unpack image path",
+		Short: "Unpack a docker image natively",
+		Long: `unpack doesn't need the docker daemon to run, and unpacks a docker image in the specified directory:
+		
+	luet util unpack golang:alpine /alpine
+`,
+		PreRun: func(cmd *cobra.Command, args []string) {
+
+			if len(args) != 2 {
+				Fatal("Expects an image and a path")
+			}
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			image := args[0]
+			destination, err := filepath.Abs(args[1])
+			if err != nil {
+				Error("Invalid path %s", destination)
+				os.Exit(1)
+			}
+
+			verify, _ := cmd.Flags().GetBool("verify")
+			user, _ := cmd.Flags().GetString("auth-username")
+			pass, _ := cmd.Flags().GetString("auth-password")
+			authType, _ := cmd.Flags().GetString("auth-type")
+			server, _ := cmd.Flags().GetString("auth-server-address")
+			identity, _ := cmd.Flags().GetString("auth-identity-token")
+			registryToken, _ := cmd.Flags().GetString("auth-registry-token")
+
+			temp, err := config.LuetCfg.GetSystem().TempDir("contentstore")
+			if err != nil {
+				Fatal("Cannot create a tempdir", err.Error())
+			}
+
+			Info("Downloading", image, "to", destination)
+			auth := &types.AuthConfig{
+				Username:      user,
+				Password:      pass,
+				ServerAddress: server,
+				Auth:          authType,
+				IdentityToken: identity,
+				RegistryToken: registryToken,
+			}
+
+			info, err := docker.DownloadAndExtractDockerImage(temp, image, destination, auth, verify)
+			if err != nil {
+				Error(err.Error())
+				os.Exit(1)
+			}
+			Info(fmt.Sprintf("Pulled: %s %s", info.Target.Digest, info.Name))
+			Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
+		},
+	}
+
+	c.Flags().String("auth-username", "", "Username to authenticate to registry/notary")
+	c.Flags().String("auth-password", "", "Password to authenticate to registry")
+	c.Flags().String("auth-type", "", "Auth type")
+	c.Flags().String("auth-server-address", "", "Authentication server address")
+	c.Flags().String("auth-identity-token", "", "Authentication identity token")
+	c.Flags().String("auth-registry-token", "", "Authentication registry token")
+	c.Flags().Bool("verify", false, "Verify signed images to notary before to pull")
+	return c
+}

contrib/config/get_luet_root.sh

@@ -7,7 +7,7 @@ fi
 set -ex
 export LUET_NOLOCK=true
 
-LUET_VERSION=$(curl -s https://api.github.com/repos/mudler/luet/releases/latest | ( grep -oP '"tag_name": "\K(.*)(?=")' || echo "0.9.24" ))
+LUET_VERSION=$(curl -s https://api.github.com/repos/mudler/luet/releases/latest | grep tag_name | awk '{ print $2 }' | sed -e 's/\"//g' -e 's/,//g' || echo "0.9.24" )
 LUET_ROOTFS=${LUET_ROOTFS:-/}
 LUET_DATABASE_PATH=${LUET_DATABASE_PATH:-/var/luet/db}
 LUET_DATABASE_ENGINE=${LUET_DATABASE_ENGINE:-boltdb}

go.mod

@@ -4,22 +4,26 @@ go 1.14
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/Sabayon/pkgs-checker v0.7.2
+	github.com/Sabayon/pkgs-checker v0.8.1
+	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
-	github.com/briandowns/spinner v1.7.0
+	github.com/briandowns/spinner v1.12.1-0.20201220203425-e201aaea0a31
 	github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec
 	github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc
 	github.com/crillab/gophersat v1.3.2-0.20201023142334-3fc2ac466765
+	github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
+	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
-	github.com/fsouza/go-dockerclient v1.6.4
 	github.com/genuinetools/img v0.5.11
 	github.com/ghodss/yaml v1.0.0
 	github.com/google/go-containerregistry v0.2.1
+	github.com/google/renameio v1.0.0
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/hashicorp/go-version v1.2.0
+	github.com/hashicorp/go-version v1.2.1
+	github.com/imdario/mergo v0.3.9
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/jedib0t/go-pretty/v6 v6.0.5
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
@@ -29,27 +33,33 @@ require (
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
+	github.com/mitchellh/hashstructure/v2 v2.0.1
 	github.com/moby/buildkit v0.7.2
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+	github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
 	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
 	github.com/pkg/errors v0.9.1
 	github.com/schollz/progressbar/v3 v3.7.1
-	github.com/sirupsen/logrus v1.6.0
-	github.com/spf13/cobra v1.0.0
-	github.com/spf13/viper v1.7.0
+	github.com/sirupsen/logrus v1.7.0
+	github.com/spf13/cobra v1.1.1
+	github.com/spf13/viper v1.7.1
+	github.com/theupdateframework/notary v0.7.0
 	go.etcd.io/bbolt v1.3.5
 	go.uber.org/atomic v1.5.1 // indirect
-	go.uber.org/multierr v1.4.0 // indirect
+	go.uber.org/multierr v1.4.0
 	go.uber.org/zap v1.13.0
+	golang.org/x/mod v0.4.2
+	google.golang.org/grpc v1.29.1
 	gopkg.in/yaml.v2 v2.3.0
+	gotest.tools/v3 v3.0.2 // indirect
 	helm.sh/helm/v3 v3.3.4
 
 )

pkg/box/exec.go

@@ -23,9 +23,9 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/pkg/errors"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
-	helpers "github.com/mudler/luet/pkg/helpers"
+	"github.com/pkg/errors"
 )
 
 type Box interface {
@@ -107,7 +107,7 @@ func (b *DefaultBox) Exec() error {
 
 func (b *DefaultBox) Run() error {
 
-	if !helpers.Exists(b.Root) {
+	if !fileHelper.Exists(b.Root) {
 		return errors.New(b.Root + " does not exist")
 	}
 

pkg/bus/events.go

@@ -2,6 +2,7 @@ package bus
 
 import (
 	"github.com/mudler/go-pluggable"
+	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -44,24 +45,59 @@ var (
 	EventRepositoryPreBuild pluggable.EventType = "repository.pre.build"
 	// EventRepositoryPostBuild is the event fired after a repository was built
 	EventRepositoryPostBuild pluggable.EventType = "repository.post.build"
+
+	// Image unpack
+
+	// EventImagePreUnPack is the event fired before unpacking an image to a local dir
+	EventImagePreUnPack pluggable.EventType = "image.pre.unpack"
+	// EventImagePostUnPack is the event fired after unpacking an image to a local dir
+	EventImagePostUnPack pluggable.EventType = "image.post.unpack"
 )
 
 // Manager is the bus instance manager, which subscribes plugins to events emitted by Luet
-var Manager *pluggable.Manager = pluggable.NewManager(
-	[]pluggable.EventType{
-		EventPackageInstall,
-		EventPackageUnInstall,
-		EventPackagePreBuild,
-		EventPackagePreBuildArtifact,
-		EventPackagePostBuildArtifact,
-		EventPackagePostBuild,
-		EventRepositoryPreBuild,
-		EventRepositoryPostBuild,
-		EventImagePreBuild,
-		EventImagePrePull,
-		EventImagePrePush,
-		EventImagePostBuild,
-		EventImagePostPull,
-		EventImagePostPush,
-	},
-)
+var Manager *Bus = &Bus{
+	Manager: pluggable.NewManager(
+		[]pluggable.EventType{
+			EventPackageInstall,
+			EventPackageUnInstall,
+			EventPackagePreBuild,
+			EventPackagePreBuildArtifact,
+			EventPackagePostBuildArtifact,
+			EventPackagePostBuild,
+			EventRepositoryPreBuild,
+			EventRepositoryPostBuild,
+			EventImagePreBuild,
+			EventImagePrePull,
+			EventImagePrePush,
+			EventImagePostBuild,
+			EventImagePostPull,
+			EventImagePostPush,
+			EventImagePreUnPack,
+			EventImagePostUnPack,
+		},
+	),
+}
+
+type Bus struct {
+	*pluggable.Manager
+}
+
+func (b *Bus) Initialize(plugin ...string) {
+	b.Manager.Load(plugin...).Register()
+
+	for _, e := range b.Manager.Events {
+		b.Manager.Response(e, func(p *pluggable.Plugin, r *pluggable.EventResponse) {
+			if r.Errored() {
+				logrus.Fatal("Plugin", p.Name, "at", p.Executable, "Error", r.Error)
+			}
+			logrus.Debug(
+				"plugin_event",
+				"received from",
+				p.Name,
+				"at",
+				p.Executable,
+				r,
+			)
+		})
+	}
+}

pkg/compiler/backend.go

@@ -1,33 +1,52 @@
-// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package backend
+package compiler
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
-	. "github.com/mudler/luet/pkg/logger"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 
-	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/pkg/errors"
+
+	. "github.com/mudler/luet/pkg/logger"
 )
 
+func NewBackend(s string) (CompilerBackend, error) {
+	var compilerBackend CompilerBackend
+
+	switch s {
+	case backend.ImgBackend:
+		compilerBackend = backend.NewSimpleImgBackend()
+	case backend.DockerBackend:
+		compilerBackend = backend.NewSimpleDockerBackend()
+	default:
+		return nil, errors.New("invalid backend. Unsupported")
+	}
+
+	return compilerBackend, nil
+}
+
+type CompilerBackend interface {
+	BuildImage(backend.Options) error
+	ExportImage(backend.Options) error
+	RemoveImage(backend.Options) error
+	ImageDefinitionToTar(backend.Options) error
+	ExtractRootfs(opts backend.Options, keepPerms bool) error
+
+	CopyImage(string, string) error
+	DownloadImage(opts backend.Options) error
+
+	Push(opts backend.Options) error
+	ImageAvailable(string) bool
+
+	ImageExists(string) bool
+}
+
 // GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
 // example of json return: [
 //   {
@@ -54,46 +73,46 @@ import (
 //     }
 //   }
 // ]
-func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
+func GenerateChanges(b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) {
 
-	res := compiler.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
+	res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
 
 	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
 	defer os.RemoveAll(tmpdiffs) // clean up
 
 	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
 	defer os.RemoveAll(srcRootFS) // clean up
 
 	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
 	defer os.RemoveAll(dstRootFS) // clean up
 
-	srcImageExtract := compiler.CompilerBackendOptions{
+	srcImageExtract := backend.Options{
 		ImageName:   fromImage.ImageName,
 		Destination: srcRootFS,
 	}
 	Debug("Extracting source image", fromImage.ImageName)
 	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
 	}
 
-	dstImageExtract := compiler.CompilerBackendOptions{
+	dstImageExtract := backend.Options{
 		ImageName:   toImage.ImageName,
 		Destination: dstRootFS,
 	}
 	Debug("Extracting destination image", toImage.ImageName)
 	err = b.ExtractRootfs(dstImageExtract, false)
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
 	}
 
 	// Get Additions/Changes. dst -> src
@@ -114,7 +133,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
 
 			if sizeA != sizeB {
 				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
-				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
+				res.Diffs.Changes = append(res.Diffs.Changes, artifact.ArtifactNode{
 					Name: filepath.Join("/", realpath),
 					Size: int(sizeB),
 				})
@@ -127,7 +146,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
 			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
 				sizeB = s.Size()
 			}
-			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
+			res.Diffs.Additions = append(res.Diffs.Additions, artifact.ArtifactNode{
 				Name: filepath.Join("/", realpath),
 				Size: int(sizeB),
 			})
@@ -138,7 +157,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
 		return nil
 	})
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
 	}
 
 	// Get deletions. src -> dst
@@ -150,7 +169,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
 		realpath := strings.Replace(path, srcRootFS, "", -1)
 		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
 			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
-			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
+			res.Diffs.Deletions = append(res.Diffs.Deletions, artifact.ArtifactNode{
 				Name: filepath.Join("/", realpath),
 			})
 		}
@@ -158,8 +177,71 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
 		return nil
 	})
 	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
 	}
 
-	return []compiler.ArtifactLayer{res}, nil
+	diffs := []artifact.ArtifactLayer{res}
+
+	if config.LuetCfg.GetGeneral().Debug {
+		summary := ComputeArtifactLayerSummary(diffs)
+		for _, l := range summary.Layers {
+			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
+				l.FromImage, l.ToImage,
+				l.AddFiles, l.AddSizes,
+				l.DelFiles, l.DelSizes,
+				l.ChangeFiles, l.ChangeSizes))
+		}
+	}
+
+	return diffs, nil
+}
+
+type ArtifactLayerSummary struct {
+	FromImage   string `json:"image1"`
+	ToImage     string `json:"image2"`
+	AddFiles    int    `json:"add_files"`
+	AddSizes    int64  `json:"add_sizes"`
+	DelFiles    int    `json:"del_files"`
+	DelSizes    int64  `json:"del_sizes"`
+	ChangeFiles int    `json:"change_files"`
+	ChangeSizes int64  `json:"change_sizes"`
+}
+
+type ArtifactLayersSummary struct {
+	Layers []ArtifactLayerSummary `json:"summary"`
+}
+
+func ComputeArtifactLayerSummary(diffs []artifact.ArtifactLayer) ArtifactLayersSummary {
+
+	ans := ArtifactLayersSummary{
+		Layers: make([]ArtifactLayerSummary, 0),
+	}
+
+	for _, layer := range diffs {
+		sum := ArtifactLayerSummary{
+			FromImage:   layer.FromImage,
+			ToImage:     layer.ToImage,
+			AddFiles:    0,
+			AddSizes:    0,
+			DelFiles:    0,
+			DelSizes:    0,
+			ChangeFiles: 0,
+			ChangeSizes: 0,
+		}
+		for _, a := range layer.Diffs.Additions {
+			sum.AddFiles++
+			sum.AddSizes += int64(a.Size)
+		}
+		for _, d := range layer.Diffs.Deletions {
+			sum.DelFiles++
+			sum.DelSizes += int64(d.Size)
+		}
+		for _, c := range layer.Diffs.Changes {
+			sum.ChangeFiles++
+			sum.ChangeSizes += int64(c.Size)
+		}
+		ans.Layers = append(ans.Layers, sum)
+	}
+
+	return ans
 }

pkg/compiler/backend/common.go

@@ -16,8 +16,13 @@
 package backend
 
 import (
+	"os/exec"
+
+	"github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+
 	"github.com/google/go-containerregistry/pkg/crane"
-	"github.com/mudler/luet/pkg/compiler"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -30,14 +35,48 @@ func imageAvailable(image string) bool {
 	return err == nil
 }
 
-func NewBackend(s string) compiler.CompilerBackend {
-	var compilerBackend compiler.CompilerBackend
-
-	switch s {
-	case ImgBackend:
-		compilerBackend = NewSimpleImgBackend()
-	case DockerBackend:
-		compilerBackend = NewSimpleDockerBackend()
-	}
-	return compilerBackend
+type Options struct {
+	ImageName      string
+	SourcePath     string
+	DockerFileName string
+	Destination    string
+	Context        string
+	BackendArgs    []string
+}
+
+func runCommand(cmd *exec.Cmd) error {
+	output := ""
+	buffered := !config.LuetCfg.GetGeneral().ShowBuildOutput
+	writer := NewBackendWriter(buffered)
+
+	cmd.Stdout = writer
+	cmd.Stderr = writer
+
+	if buffered {
+		Spinner(22)
+		defer SpinnerStop()
+	}
+
+	err := cmd.Start()
+	if err != nil {
+		return errors.Wrap(err, "Failed starting command")
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		output = writer.GetCombinedOutput()
+		return errors.Wrapf(err, "Failed running command: %s", output)
+	}
+
+	return nil
+}
+
+func genBuildCommand(opts Options) []string {
+	context := opts.Context
+
+	if context == "" {
+		context = "."
+	}
+	buildarg := append(opts.BackendArgs, "-f", opts.DockerFileName, "-t", opts.ImageName, context)
+	return append([]string{"build"}, buildarg...)
 }

pkg/compiler/backend/simpledocker.go

@@ -17,18 +17,17 @@ package backend
 
 import (
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 
-	docker "github.com/fsouza/go-dockerclient"
+	bus "github.com/mudler/luet/pkg/bus"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	capi "github.com/mudler/docker-companion/api"
 
-	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -37,51 +36,27 @@ import (
 
 type SimpleDocker struct{}
 
-func NewSimpleDockerBackend() compiler.CompilerBackend {
+func NewSimpleDockerBackend() *SimpleDocker {
 	return &SimpleDocker{}
 }
 
 // TODO: Missing still: labels, and build args expansion
-func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) BuildImage(opts Options) error {
 	name := opts.ImageName
-	path := opts.SourcePath
-	dockerfileName := opts.DockerFileName
-	context := opts.Context
-
-	if context == "" {
-		context = "."
-	}
-	buildarg := []string{"build", "-f", dockerfileName, "-t", name, context}
+	bus.Manager.Publish(bus.EventImagePreBuild, opts)
 
+	buildarg := genBuildCommand(opts)
 	Info(":whale2: Building image " + name)
 	cmd := exec.Command("docker", buildarg...)
-	cmd.Dir = path
-	out, err := cmd.CombinedOutput()
+	cmd.Dir = opts.SourcePath
+	err := runCommand(cmd)
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return err
 	}
+
 	Info(":whale: Building image " + name + " done")
 
-	if os.Getenv("DOCKER_SQUASH") == "true" {
-		Info(":whale: Squashing image " + name)
-		var client *docker.Client
-
-		client, err = docker.NewClientFromEnv()
-		if err != nil {
-			return errors.Wrap(err, "could not connect to the Docker daemon")
-		}
-		err = capi.Squash(client, name, name)
-		if err != nil {
-			return errors.Wrap(err, "Failed squashing image")
-		}
-		Info(":whale: Squashing image " + name + " done")
-	}
-
-	if config.LuetCfg.GetGeneral().ShowBuildOutput {
-		Info(string(out))
-	} else {
-		Debug(string(out))
-	}
+	bus.Manager.Publish(bus.EventImagePostBuild, opts)
 
 	return nil
 }
@@ -97,16 +72,25 @@ func (*SimpleDocker) CopyImage(src, dst string) error {
 	return nil
 }
 
-func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) DownloadImage(opts Options) error {
 	name := opts.ImageName
+	bus.Manager.Publish(bus.EventImagePrePull, opts)
+
 	buildarg := []string{"pull", name}
 	Debug(":whale: Downloading image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	cmd := exec.Command("docker", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed pulling image: "+string(out))
 	}
+
 	Info(":whale: Downloaded image:", name)
+	bus.Manager.Publish(bus.EventImagePostPull, opts)
+
 	return nil
 }
 
@@ -127,7 +111,7 @@ func (*SimpleDocker) ImageAvailable(imagename string) bool {
 	return imageAvailable(imagename)
 }
 
-func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) RemoveImage(opts Options) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
 	out, err := exec.Command("docker", buildarg...).CombinedOutput()
@@ -139,19 +123,26 @@ func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
-func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) Push(opts Options) error {
 	name := opts.ImageName
 	pusharg := []string{"push", name}
+	bus.Manager.Publish(bus.EventImagePrePush, opts)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("docker", pusharg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed pushing image: "+string(out))
 	}
 	Info(":whale: Pushed image:", name)
+	bus.Manager.Publish(bus.EventImagePostPush, opts)
+
 	//Info(string(out))
 	return nil
 }
 
-func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
 	}
@@ -164,12 +155,16 @@ func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions
 	return nil
 }
 
-func (*SimpleDocker) ExportImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) ExportImage(opts Options) error {
 	name := opts.ImageName
 	path := opts.Destination
 
 	buildarg := []string{"save", name, "-o", path}
 	Debug(":whale: Saving image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("docker", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed exporting image: "+string(out))
@@ -183,10 +178,16 @@ type ManifestEntry struct {
 	Layers []string `json:"Layers"`
 }
 
-func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
+func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
 	name := opts.ImageName
 	dst := opts.Destination
 
+	if !b.ImageExists(name) {
+		if err := b.DownloadImage(opts); err != nil {
+			return errors.Wrap(err, "failed pulling image "+name+" during extraction")
+		}
+	}
+
 	tempexport, err := ioutil.TempDir(dst, "tmprootfs")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
@@ -194,10 +195,16 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
 	defer os.RemoveAll(tempexport) // clean up
 
 	imageExport := filepath.Join(tempexport, "image.tar")
-	if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: name, Destination: imageExport}); err != nil {
+
+	Spinner(22)
+	defer SpinnerStop()
+
+	if err := b.ExportImage(Options{ImageName: name, Destination: imageExport}); err != nil {
 		return errors.Wrap(err, "failed while extracting rootfs for "+name)
 	}
 
+	SpinnerStop()
+
 	src := imageExport
 
 	if src == "" && opts.ImageName != "" {
@@ -207,7 +214,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
 		}
 		defer os.RemoveAll(tempUnpack) // clean up
 		imageExport := filepath.Join(tempUnpack, "image.tar")
-		if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
+		if err := b.ExportImage(Options{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
 			return errors.Wrap(err, "while exporting image before extraction")
 		}
 		src = imageExport
@@ -231,7 +238,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
 		return errors.Wrap(err, "Error met while unpacking rootfs")
 	}
 
-	manifest, err := helpers.Read(filepath.Join(rootfs, "manifest.json"))
+	manifest, err := fileHelper.Read(filepath.Join(rootfs, "manifest.json"))
 	if err != nil {
 		return errors.Wrap(err, "Error met while reading image manifest")
 	}
@@ -259,7 +266,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
 		return err
 	}
 
-	err = export.UnPackLayers(layers_sha, dst, "")
+	err = export.UnPackLayers(layers_sha, dst, "containerd")
 	if err != nil {
 		return err
 	}
@@ -271,21 +278,3 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
 
 	return nil
 }
-
-// Changes retrieves changes between image layers
-func (d *SimpleDocker) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
-	diffs, err := GenerateChanges(d, fromImage, toImage)
-
-	if config.LuetCfg.GetGeneral().Debug {
-		summary := compiler.ComputeArtifactLayerSummary(diffs)
-		for _, l := range summary.Layers {
-			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
-				l.FromImage, l.ToImage,
-				l.AddFiles, l.AddSizes,
-				l.DelFiles, l.DelSizes,
-				l.ChangeFiles, l.ChangeSizes))
-		}
-	}
-
-	return diffs, err
-}

pkg/compiler/backend/simpledocker_test.go

@@ -16,15 +16,17 @@
 package backend_test
 
 import (
+	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
-	"github.com/mudler/luet/pkg/solver"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
-	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -41,13 +43,10 @@ var _ = Describe("Docker backend", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
 			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 			Expect(lspec.Image).To(Equal("luet/base"))
 			Expect(lspec.Seed).To(Equal("alpine"))
@@ -61,7 +60,7 @@ var _ = Describe("Docker backend", func() {
 
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -71,7 +70,7 @@ ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin`))
 			b := NewSimpleDockerBackend()
-			opts := CompilerBackendOptions{
+			opts := backend.Options{
 				ImageName:      "luet/base",
 				SourcePath:     tmpdir,
 				DockerFileName: "Dockerfile",
@@ -80,11 +79,11 @@ ENV PACKAGE_CATEGORY=app-admin`))
 
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
@@ -95,7 +94,7 @@ ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
-			opts2 := CompilerBackendOptions{
+			opts2 := backend.Options{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
@@ -104,28 +103,28 @@ RUN echo bar > /test2`))
 
 			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
-			artifacts := []ArtifactNode{{
+			artifacts := []artifact.ArtifactNode{{
 				Name: "/luetbuild/LuetDockerfile",
 				Size: 175,
 			}}
 			if os.Getenv("DOCKER_BUILDKIT") == "1" {
-				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+				artifacts = append(artifacts, artifact.ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
 			}
-			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
-			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4})
 
-			Expect(b.Changes(opts, opts2)).To(Equal(
-				[]ArtifactLayer{{
+			Expect(compiler.GenerateChanges(b, opts, opts2)).To(Equal(
+				[]artifact.ArtifactLayer{{
 					FromImage: "luet/base",
 					ToImage:   "test",
-					Diffs: ArtifactDiffs{
+					Diffs: artifact.ArtifactDiffs{
 						Additions: artifacts,
 					},
 				}}))
 
-			opts2 = CompilerBackendOptions{
+			opts2 = backend.Options{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
@@ -133,7 +132,7 @@ RUN echo bar > /test2`))
 			}
 
 			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
 			Expect(b.ImageExists(opts2.ImageName)).To(BeFalse())
 		})
 

pkg/compiler/backend/simpleimg.go

@@ -20,7 +20,8 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/mudler/luet/pkg/compiler"
+	bus "github.com/mudler/luet/pkg/bus"
+
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/pkg/errors"
@@ -28,37 +29,33 @@ import (
 
 type SimpleImg struct{}
 
-func NewSimpleImgBackend() compiler.CompilerBackend {
+func NewSimpleImgBackend() *SimpleImg {
 	return &SimpleImg{}
 }
 
 // TODO: Missing still: labels, and build args expansion
-func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) BuildImage(opts Options) error {
 	name := opts.ImageName
-	path := opts.SourcePath
-	context := opts.Context
+	bus.Manager.Publish(bus.EventImagePreBuild, opts)
 
-	if context == "" {
-		context = "."
-	}
-	dockerfileName := opts.DockerFileName
+	buildarg := genBuildCommand(opts)
 
-	buildarg := []string{"build", "-f", dockerfileName, "-t", name, context}
-	Spinner(22)
-	defer SpinnerStop()
 	Info(":tea: Building image " + name)
-	cmd := exec.Command("img", buildarg...)
-	cmd.Dir = path
-	out, err := cmd.CombinedOutput()
 
+	cmd := exec.Command("img", buildarg...)
+	cmd.Dir = opts.SourcePath
+	err := runCommand(cmd)
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return err
 	}
+	bus.Manager.Publish(bus.EventImagePostBuild, opts)
+
 	Info(":tea: Building image " + name + " done")
+
 	return nil
 }
 
-func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) RemoveImage(opts Options) error {
 	name := opts.ImageName
 	buildarg := []string{"rm", name}
 	Spinner(22)
@@ -72,12 +69,16 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
-func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
-
+func (*SimpleImg) DownloadImage(opts Options) error {
 	name := opts.ImageName
-	buildarg := []string{"pull", name}
+	bus.Manager.Publish(bus.EventImagePrePull, opts)
 
+	buildarg := []string{"pull", name}
 	Debug(":tea: Downloading image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	cmd := exec.Command("img", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
@@ -85,6 +86,7 @@ func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
 	}
 
 	Info(":tea: Image " + name + " downloaded")
+	bus.Manager.Publish(bus.EventImagePostPull, opts)
 
 	return nil
 }
@@ -120,7 +122,7 @@ func (*SimpleImg) ImageExists(imagename string) bool {
 	return false
 }
 
-func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleImg) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
 	}
@@ -133,11 +135,15 @@ func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) e
 	return nil
 }
 
-func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) ExportImage(opts Options) error {
 	name := opts.ImageName
 	path := opts.Destination
 	buildarg := []string{"save", "-o", path, name}
 	Debug(":tea: Saving image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed exporting image: "+string(out))
@@ -147,7 +153,7 @@ func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
 }
 
 // ExtractRootfs extracts the docker image content inside the destination
-func (s *SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
+func (s *SimpleImg) ExtractRootfs(opts Options, keepPerms bool) error {
 	name := opts.ImageName
 	path := opts.Destination
 
@@ -158,8 +164,13 @@ func (s *SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerm
 	}
 
 	os.RemoveAll(path)
+
 	buildarg := []string{"unpack", "-o", path, name}
 	Debug(":tea: Extracting image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed extracting image: "+string(out))
@@ -168,20 +179,18 @@ func (s *SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerm
 	return nil
 }
 
-// TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (i *SimpleImg) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
-	return GenerateChanges(i, fromImage, toImage)
-}
-
-func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) Push(opts Options) error {
 	name := opts.ImageName
+	bus.Manager.Publish(bus.EventImagePrePush, opts)
+
 	pusharg := []string{"push", name}
 	out, err := exec.Command("img", pusharg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed pushing image: "+string(out))
 	}
 	Info(":tea: Pushed image:", name)
+	bus.Manager.Publish(bus.EventImagePostPush, opts)
+
 	//Info(string(out))
 	return nil
 }

pkg/compiler/backend/writer.go

@@ -0,0 +1,48 @@
+// Copyright © 2021 Daniele Rondina <geaaru@sabayonlinux.org>
+//                  Ettore Di Giacinto <mudler@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"bytes"
+
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+type BackendWriter struct {
+	BufferedOutput bool
+	Buffer         *bytes.Buffer
+}
+
+func NewBackendWriter(buffered bool) *BackendWriter {
+	return &BackendWriter{
+		BufferedOutput: buffered,
+		Buffer:         &bytes.Buffer{},
+	}
+}
+
+func (b *BackendWriter) Write(p []byte) (int, error) {
+	if b.BufferedOutput {
+		return b.Buffer.Write(p)
+	}
+
+	Msg("info", false, false, (string(p)))
+
+	return len(p), nil
+}
+
+func (b *BackendWriter) Close() error              { return nil }
+func (b *BackendWriter) GetCombinedOutput() string { return b.Buffer.String() }

pkg/compiler/backend_test.go

@@ -13,10 +13,10 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package backend_test
+package compiler_test
 
 import (
-	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
 
 	. "github.com/onsi/ginkgo"
@@ -24,7 +24,7 @@ import (
 )
 
 var _ = Describe("Docker image diffs", func() {
-	var b compiler.CompilerBackend
+	var b CompilerBackend
 
 	BeforeEach(func() {
 		b = NewSimpleDockerBackend()
@@ -32,7 +32,7 @@ var _ = Describe("Docker image diffs", func() {
 
 	Context("Generate diffs from docker images", func() {
 		It("Detect no changes", func() {
-			opts := compiler.CompilerBackendOptions{
+			opts := Options{
 				ImageName: "alpine:latest",
 			}
 			err := b.DownloadImage(opts)
@@ -47,18 +47,18 @@ var _ = Describe("Docker image diffs", func() {
 		})
 
 		It("Detects additions and changed files", func() {
-			err := b.DownloadImage(compiler.CompilerBackendOptions{
+			err := b.DownloadImage(Options{
 				ImageName: "quay.io/mocaccino/micro",
 			})
 			Expect(err).ToNot(HaveOccurred())
-			err = b.DownloadImage(compiler.CompilerBackendOptions{
+			err = b.DownloadImage(Options{
 				ImageName: "quay.io/mocaccino/extra",
 			})
 			Expect(err).ToNot(HaveOccurred())
 
-			layers, err := GenerateChanges(b, compiler.CompilerBackendOptions{
+			layers, err := GenerateChanges(b, Options{
 				ImageName: "quay.io/mocaccino/micro",
-			}, compiler.CompilerBackendOptions{
+			}, Options{
 				ImageName: "quay.io/mocaccino/extra",
 			})
 			Expect(err).ToNot(HaveOccurred())

pkg/compiler/compiler_test.go

@@ -21,9 +21,12 @@ import (
 
 	. "github.com/mudler/luet/pkg/compiler"
 	sd "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	helpers "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -39,7 +42,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -54,19 +57,18 @@ var _ = Describe("Compiler", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
@@ -74,6 +76,68 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Copy and Join", func() {
+		It("Compiles it correctly with Copy", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/copy")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(spec.Rel("result"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("bina/busybox"))).To(BeTrue())
+		})
+
+		It("Compiles it correctly with Join", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/join")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(spec.Rel("newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+		})
+	})
+
 	Context("Simple package build definition", func() {
 		It("Compiles it in parallel", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -83,7 +147,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -98,12 +162,11 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 		})
@@ -118,7 +181,7 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/templates")
 			Expect(err).ToNot(HaveOccurred())
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 			pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -142,7 +205,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -159,30 +222,29 @@ var _ = Describe("Compiler", func() {
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2, spec3))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(3))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
-			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("c"))
+			content1, err := fileHelper.Read(spec.Rel("c"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("cd"))
+			content2, err := fileHelper.Read(spec.Rel("cd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("c\n"))
 			Expect(content2).To(Equal("c\n"))
 
-			content1, err = helpers.Read(spec.Rel("d"))
+			content1, err = fileHelper.Read(spec.Rel("d"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err = helpers.Read(spec.Rel("dd"))
+			content2, err = fileHelper.Read(spec.Rel("dd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("s\n"))
 			Expect(content2).To(Equal("dd\n"))
@@ -199,7 +261,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -207,27 +269,26 @@ var _ = Describe("Compiler", func() {
 			Expect(err).ToNot(HaveOccurred())
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
-			artifacts2, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec2))
+			artifacts2, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts2)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			for _, artifact := range artifacts2 {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
-			Expect(helpers.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test1"))).To(BeTrue())
 		})
 
 		It("Compiles and includes ony wanted files", func() {
@@ -241,7 +302,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -250,19 +311,18 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
 		It("Compiles and excludes files", func() {
@@ -276,7 +336,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -285,20 +345,19 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 		})
 
 		It("Compiles includes and excludes files", func() {
@@ -312,7 +371,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -321,20 +380,19 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
 		It("Compiles and excludes ony wanted files also from unpacked packages", func() {
@@ -348,7 +406,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -357,18 +415,17 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 		})
 
 		It("Compiles includes and excludes ony wanted files also from unpacked packages", func() {
@@ -382,7 +439,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -391,18 +448,17 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 		})
 
 		It("Compiles and includes ony wanted files also from unpacked packages", func() {
@@ -416,7 +472,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -425,22 +481,21 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test2"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("lib/firmware"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test2"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("lib/firmware"))).ToNot(BeTrue())
 		})
 
 		It("Compiles a more complex tree", func() {
@@ -454,7 +509,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -463,25 +518,24 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("extra-layer"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("base-layer-0.1.metadata.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.metadata.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
 		It("Compiles with provides support", func() {
@@ -495,7 +549,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -504,27 +558,26 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("dd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("cd"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
 		})
 
 		It("Compiles with provides and selectors support", func() {
@@ -539,7 +592,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -548,27 +601,26 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("dd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("cd"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
 		})
 		It("Compiles revdeps", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -581,7 +633,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -591,21 +643,21 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(2))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("extra-layer"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
 		It("Compiles complex dependencies trees with best matches", func() {
@@ -619,7 +671,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
 			Expect(err).ToNot(HaveOccurred())
@@ -629,22 +681,22 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(6))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(6))
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("build-sabayon-overlays-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("linux-sabayon-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("sabayon-sources-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("vhba"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("build-sabayon-overlays-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("linux-sabayon-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("sabayon-sources-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("vhba"))).To(BeTrue())
 		})
 
 		It("Compiles revdeps with seeds", func() {
@@ -658,42 +710,42 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(4))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			// A deps on B, so A artifacts are here:
-			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
 
 			// B
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("artifact42"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("artifact42"))).To(BeTrue())
 
 			// C depends on B, so B is here
-			content1, err := helpers.Read(spec.Rel("c"))
+			content1, err := fileHelper.Read(spec.Rel("c"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("cd"))
+			content2, err := fileHelper.Read(spec.Rel("cd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("c\n"))
 			Expect(content2).To(Equal("c\n"))
 
 			// D is here as it requires C, and C was recompiled
-			content1, err = helpers.Read(spec.Rel("d"))
+			content1, err = fileHelper.Read(spec.Rel("d"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err = helpers.Read(spec.Rel("dd"))
+			content2, err = fileHelper.Read(spec.Rel("dd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("s\n"))
 			Expect(content2).To(Equal("dd\n"))
@@ -710,7 +762,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -722,24 +774,23 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-				for _, d := range artifact.GetDependencies() {
-					Expect(helpers.Exists(d.GetPath())).To(BeTrue())
-					Expect(helpers.Untar(d.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				for _, d := range artifact.Dependencies {
+					Expect(fileHelper.Exists(d.Path)).To(BeTrue())
+					Expect(helpers.Untar(d.Path, tmpdir, false)).ToNot(HaveOccurred())
 				}
 			}
 
-			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
 		})
 	})
@@ -753,7 +804,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -765,15 +816,14 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 			Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
 	})
 
@@ -786,7 +836,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{
 				Name:     "dironly",
@@ -814,21 +864,19 @@ var _ = Describe("Compiler", func() {
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir2)
 
-			compiler.SetConcurrency(1)
-
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(2))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(0))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(0))
 
 			Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test2"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test1"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test2"))).To(BeTrue())
 
 			Expect(helpers.Untar(spec2.Rel("dironly_filter-test-1.0.package.tar"), tmpdir2, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec2.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
 		})
 	})
 
@@ -841,11 +889,11 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
-			compiler.SetCompressionType(GZip)
+			compiler.Options.CompressionType = compression.GZip
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
 
 			tmpdir, err := ioutil.TempDir("", "tree")
@@ -853,18 +901,17 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
-			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
+			Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
 			Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred())
 			//	Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
 	})
 
@@ -877,7 +924,7 @@ var _ = Describe("Compiler", func() {
 			err := generalRecipe.Load("../../tests/fixtures/includeimage")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
 			Expect(err).ToNot(HaveOccurred())
@@ -896,11 +943,11 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
-			compiler.SetCompressionType(GZip)
+			compiler.Options.CompressionType = compression.GZip
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
 
 			tmpdir, err := ioutil.TempDir("", "tree")
@@ -908,20 +955,19 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
-			Expect(artifacts[0].GetFiles()).To(ContainElement("bin/busybox"))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
+			Expect(artifacts[0].Files).To(ContainElement("bin/busybox"))
 
-			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
 
 			art, err := LoadArtifactFromYaml(spec)
 			Expect(err).ToNot(HaveOccurred())
 
-			files := art.GetFiles()
+			files := art.Files
 			Expect(files).To(ContainElement("bin/busybox"))
 		})
 	})

pkg/compiler/imagehashtree.go

@@ -0,0 +1,161 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"fmt"
+
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	"github.com/mudler/luet/pkg/config"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	"github.com/pkg/errors"
+)
+
+// ImageHashTree is holding the Database
+// and the options to resolve PackageImageHashTrees
+// for a given specfile
+// It is responsible of returning a concrete result
+// which identifies a Package in a HashTree
+type ImageHashTree struct {
+	Database      pkg.PackageDatabase
+	SolverOptions config.LuetSolverOptions
+}
+
+// PackageImageHashTree represent the Package into a given image hash tree
+// The hash tree is constructed by a set of images representing
+// the package during its build stage. A Hash is assigned to each image
+// from the package fingerprint, plus the SAT solver assertion result (which is hashed as well)
+// and the specfile signatures. This guarantees that each image of the build stage
+// is unique and can be identified later on.
+type PackageImageHashTree struct {
+	Target                       *solver.PackageAssert
+	Dependencies                 solver.PackagesAssertions
+	Solution                     solver.PackagesAssertions
+	dependencyBuilderImageHashes map[string]string
+	SourceHash                   string
+	BuilderImageHash             string
+}
+
+func NewHashTree(db pkg.PackageDatabase) *ImageHashTree {
+	return &ImageHashTree{
+		Database: db,
+	}
+}
+
+func (ht *PackageImageHashTree) DependencyBuildImage(p pkg.Package) (string, error) {
+	found, ok := ht.dependencyBuilderImageHashes[p.GetFingerPrint()]
+	if !ok {
+		return "", errors.New("package hash not found")
+	}
+	return found, nil
+}
+
+func (ht *PackageImageHashTree) String() string {
+	return fmt.Sprintf(
+		"Target buildhash: %s\nTarget packagehash: %s\nBuilder Imagehash: %s\nSource Imagehash: %s\n",
+		ht.Target.Hash.BuildHash,
+		ht.Target.Hash.PackageHash,
+		ht.BuilderImageHash,
+		ht.SourceHash,
+	)
+}
+
+// Query takes a compiler and a compilation spec and returns a PackageImageHashTree tied to it.
+// PackageImageHashTree contains all the informations to resolve the spec build images in order to
+// reproducibly re-build images from packages
+func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) {
+	assertions, err := ht.resolve(cs, p)
+	if err != nil {
+		return nil, err
+	}
+	targetAssertion := assertions.Search(p.GetPackage().GetFingerPrint())
+
+	dependencies := assertions.Drop(p.GetPackage())
+	var sourceHash string
+	imageHashes := map[string]string{}
+	for _, assertion := range dependencies {
+		var depbuildImageTag string
+		compileSpec, err := cs.FromPackage(assertion.Package)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+		}
+		if compileSpec.GetImage() != "" {
+			depbuildImageTag = assertion.Hash.BuildHash
+		} else {
+			depbuildImageTag = ht.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash)
+		}
+		imageHashes[assertion.Package.GetFingerPrint()] = depbuildImageTag
+		sourceHash = assertion.Hash.PackageHash
+	}
+
+	return &PackageImageHashTree{
+		Dependencies:                 dependencies,
+		Target:                       targetAssertion,
+		SourceHash:                   sourceHash,
+		BuilderImageHash:             ht.genBuilderImageTag(p, targetAssertion.Hash.PackageHash),
+		dependencyBuilderImageHashes: imageHashes,
+		Solution:                     assertions,
+	}, nil
+}
+
+func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(packageImage))
+}
+
+// resolve computes the dependency tree of a compilation spec and returns solver assertions
+// in order to be able to compile the spec.
+func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
+	dependencies, err := cs.ComputeDepTree(p)
+	if err != nil {
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
+	}
+
+	// Get hash from buildpsecs
+	salts := map[string]string{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			spec, err := cs.FromPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "while computing hash buildspecs")
+			}
+			hash, err := spec.Hash()
+			if err != nil {
+				return nil, errors.Wrap(err, "failed computing hash")
+			}
+			salts[assertion.Package.GetFingerPrint()] = hash
+		}
+	}
+
+	assertions := solver.PackagesAssertions{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			nthsolution := dependencies.Cut(assertion.Package)
+			assertion.Hash = solver.PackageHash{
+				BuildHash:   nthsolution.SaltedHashFrom(assertion.Package, salts),
+				PackageHash: nthsolution.SaltedAssertionHash(salts),
+			}
+			assertion.Package.SetTreeDir(p.Package.GetTreeDir())
+			assertions = append(assertions, assertion)
+		}
+	}
+
+	return assertions, nil
+}

pkg/compiler/imagehashtree_test.go

@@ -0,0 +1,150 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler_test
+
+import (
+	. "github.com/mudler/luet/pkg/compiler"
+	sd "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ImageHashTree", func() {
+	generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+	compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+	hashtree := NewHashTree(generalRecipe.GetDatabase())
+	Context("Simple package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			err := generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+
+		It("Calculates the hash correctly", func() {
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(packageHash.Target.Hash.BuildHash).To(Equal("53993e5a02da4c21ad845371c872f5836fe45ff3a4e3c5ccb6296d0faee2b107"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("a786d3fd29d0b8bdfe5f304c8bf8be909d5c764cd7059c0e63294a8bff17f3ef"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-0cd3c0d07fc9be568377b3bf1b699e06"))
+		})
+	})
+
+	expectedPackageHash := "0d568ac04c4ca528a4e5b67978f2ad3a75d31d443ab20f9d7683b9608cc0d494"
+
+	Context("complex package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(expectedPackageHash))
+			Expect(packageHash.SourceHash).To(Equal(expectedPackageHash))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-0f45c345f59103e84fc8bebbf02f2e2b"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("2e8159583ac825acada763358290cfbea919a33873a926cab84f4f1a67ecf111"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+			Expect(assertionA.Hash.PackageHash).To(Equal(expectedPackageHash))
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+
+			Expect(assertionB.Hash.PackageHash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hashB).To(Equal("315075265aeb2e3c04c5428d31911f53c194ec9fa3db1421e8478f44b1e0def8"))
+		})
+	})
+
+	Context("complex package definition, with small change in build.yaml", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			//Definition of A here is slightly changed in the steps build.yaml file (1 character only)
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision_content_changed")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).ToNot(Equal(expectedPackageHash))
+			sourceHash := "66ec001fe72052d0e605ca96f607ae39ea4f8b53f0b7f762e377622d9c654de3"
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(sourceHash))
+			Expect(packageHash.SourceHash).To(Equal(sourceHash))
+
+			Expect(packageHash.SourceHash).ToNot(Equal(expectedPackageHash))
+
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-ffc02fd8aaa916d0e17249885b3226b1"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("b9c0286ebf6d28be831926ec7da9cb3cda6b489722d656aefc363ebd7173f937"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+
+			Expect(assertionA.Hash.PackageHash).To(Equal("66ec001fe72052d0e605ca96f607ae39ea4f8b53f0b7f762e377622d9c654de3"))
+			Expect(assertionA.Hash.PackageHash).ToNot(Equal(expectedPackageHash))
+
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+
+			Expect(assertionB.Hash.PackageHash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hashB).To(Equal("315075265aeb2e3c04c5428d31911f53c194ec9fa3db1421e8478f44b1e0def8"))
+		})
+	})
+
+})

pkg/compiler/interface.go

@@ -1,197 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler
-
-import (
-	"runtime"
-
-	"github.com/mudler/luet/pkg/config"
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
-)
-
-type Compiler interface {
-	Compile(bool, CompilationSpec) (Artifact, error)
-	CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
-	ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error)
-	SetConcurrency(i int)
-	FromPackage(pkg.Package) (CompilationSpec, error)
-	FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error)
-	SetBackend(CompilerBackend)
-	GetBackend() CompilerBackend
-	SetCompressionType(t CompressionImplementation)
-}
-
-type CompilerBackendOptions struct {
-	ImageName      string
-	SourcePath     string
-	DockerFileName string
-	Destination    string
-	Context        string
-}
-
-type CompilerOptions struct {
-	ImageRepository          string
-	PullFirst, KeepImg, Push bool
-	Concurrency              int
-	CompressionType          CompressionImplementation
-
-	Wait            bool
-	OnlyDeps        bool
-	NoDeps          bool
-	SolverOptions   config.LuetSolverOptions
-	BuildValuesFile string
-
-	PackageTargetOnly bool
-}
-
-func NewDefaultCompilerOptions() *CompilerOptions {
-	return &CompilerOptions{
-		ImageRepository: "luet/cache",
-		PullFirst:       false,
-		Push:            false,
-		CompressionType: None,
-		KeepImg:         true,
-		Concurrency:     runtime.NumCPU(),
-		OnlyDeps:        false,
-		NoDeps:          false,
-	}
-}
-
-type CompilerBackend interface {
-	BuildImage(CompilerBackendOptions) error
-	ExportImage(CompilerBackendOptions) error
-	RemoveImage(CompilerBackendOptions) error
-	Changes(fromImage, toImage CompilerBackendOptions) ([]ArtifactLayer, error)
-	ImageDefinitionToTar(CompilerBackendOptions) error
-	ExtractRootfs(opts CompilerBackendOptions, keepPerms bool) error
-
-	CopyImage(string, string) error
-	DownloadImage(opts CompilerBackendOptions) error
-
-	Push(opts CompilerBackendOptions) error
-	ImageAvailable(string) bool
-
-	ImageExists(string) bool
-}
-
-type Artifact interface {
-	GetPath() string
-	SetPath(string)
-	GetDependencies() []Artifact
-	SetDependencies(d []Artifact)
-	GetSourceAssertion() solver.PackagesAssertions
-	SetSourceAssertion(as solver.PackagesAssertions)
-
-	SetCompileSpec(as CompilationSpec)
-	GetCompileSpec() CompilationSpec
-	WriteYaml(dst string) error
-	Unpack(dst string, keepPerms bool) error
-	Compress(src string, concurrency int) error
-	SetCompressionType(t CompressionImplementation)
-	FileList() ([]string, error)
-	Hash() error
-	Verify() error
-
-	SetFiles(f []string)
-	GetFiles() []string
-	GetFileName() string
-
-	GetChecksums() Checksums
-	SetChecksums(c Checksums)
-
-	GenerateFinalImage(string, CompilerBackend, bool) (CompilerBackendOptions, error)
-	GetUncompressedName() string
-}
-
-type ArtifactNode struct {
-	Name string `json:"Name"`
-	Size int    `json:"Size"`
-}
-type ArtifactDiffs struct {
-	Additions []ArtifactNode `json:"Adds"`
-	Deletions []ArtifactNode `json:"Dels"`
-	Changes   []ArtifactNode `json:"Mods"`
-}
-type ArtifactLayer struct {
-	FromImage string        `json:"Image1"`
-	ToImage   string        `json:"Image2"`
-	Diffs     ArtifactDiffs `json:"Diff"`
-}
-type ArtifactLayerSummary struct {
-	FromImage   string `json:"image1"`
-	ToImage     string `json:"image2"`
-	AddFiles    int    `json:"add_files"`
-	AddSizes    int64  `json:"add_sizes"`
-	DelFiles    int    `json:"del_files"`
-	DelSizes    int64  `json:"del_sizes"`
-	ChangeFiles int    `json:"change_files"`
-	ChangeSizes int64  `json:"change_sizes"`
-}
-type ArtifactLayersSummary struct {
-	Layers []ArtifactLayerSummary `json:"summary"`
-}
-
-// CompilationSpec represent a compilation specification derived from a package
-type CompilationSpec interface {
-	ImageUnpack() bool // tells if the definition is just an image
-	GetIncludes() []string
-	GetExcludes() []string
-
-	RenderBuildImage() (string, error)
-	WriteBuildImageDefinition(string) error
-
-	RenderStepImage(image string) (string, error)
-	WriteStepImageDefinition(fromimage, path string) error
-
-	GetPackage() pkg.Package
-	BuildSteps() []string
-
-	GetSeedImage() string
-	SetSeedImage(string)
-
-	GetImage() string
-	SetImage(string)
-
-	SetOutputPath(string)
-	GetOutputPath() string
-	Rel(string) string
-
-	GetPreBuildSteps() []string
-
-	GetSourceAssertion() solver.PackagesAssertions
-	SetSourceAssertion(as solver.PackagesAssertions)
-
-	GetRetrieve() []string
-	CopyRetrieves(dest string) error
-
-	SetPackageDir(string)
-	GetPackageDir() string
-
-	EmptyPackage() bool
-	UnpackedPackage() bool
-	HasImageSource() bool
-}
-
-type CompilationSpecs interface {
-	Unique() CompilationSpecs
-	Len() int
-	All() []CompilationSpec
-	Add(CompilationSpec)
-	Remove(s CompilationSpecs) CompilationSpecs
-}

pkg/compiler/types/artifact/artifact.go

@@ -13,12 +13,14 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler
+package artifact
 
 import (
 	"archive/tar"
 	"bufio"
 	"bytes"
+	"crypto/sha1"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -27,7 +29,6 @@ import (
 	"path/filepath"
 	"regexp"
 
-	system "github.com/docker/docker/pkg/system"
 	zstd "github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 
@@ -36,8 +37,12 @@ import (
 	"sync"
 
 	bus "github.com/mudler/luet/pkg/bus"
+	backend "github.com/mudler/luet/pkg/compiler/backend"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
@@ -45,54 +50,31 @@ import (
 	yaml "gopkg.in/yaml.v2"
 )
 
-type CompressionImplementation string
-
-const (
-	None      CompressionImplementation = "none" // e.g. tar for standard packages
-	GZip      CompressionImplementation = "gzip"
-	Zstandard CompressionImplementation = "zstd"
-)
-
-type ArtifactIndex []Artifact
-
-func (i ArtifactIndex) CleanPath() ArtifactIndex {
-	newIndex := ArtifactIndex{}
-	for _, n := range i {
-		art := n.(*PackageArtifact)
-		// FIXME: This is a dup and makes difficult to add attributes to artifacts
-		newIndex = append(newIndex, &PackageArtifact{
-			Path:            path.Base(n.GetPath()),
-			SourceAssertion: art.SourceAssertion,
-			CompileSpec:     art.CompileSpec,
-			Dependencies:    art.Dependencies,
-			CompressionType: art.CompressionType,
-			Checksums:       art.Checksums,
-			Files:           art.Files,
-		})
-	}
-	return newIndex
-	//Update if exists, otherwise just create
-}
-
 //  When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository
 // which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact.
 // In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages.
 type PackageArtifact struct {
 	Path string `json:"path"`
 
-	Dependencies    []*PackageArtifact        `json:"dependencies"`
-	CompileSpec     *LuetCompilationSpec      `json:"compilationspec"`
-	Checksums       Checksums                 `json:"checksums"`
-	SourceAssertion solver.PackagesAssertions `json:"-"`
-	CompressionType CompressionImplementation `json:"compressiontype"`
-	Files           []string                  `json:"files"`
+	Dependencies      []*PackageArtifact                `json:"dependencies"`
+	CompileSpec       *compilerspec.LuetCompilationSpec `json:"compilationspec"`
+	Checksums         Checksums                         `json:"checksums"`
+	SourceAssertion   solver.PackagesAssertions         `json:"-"`
+	CompressionType   compression.Implementation        `json:"compressiontype"`
+	Files             []string                          `json:"files"`
+	PackageCacheImage string                            `json:"package_cacheimage"`
 }
 
-func NewPackageArtifact(path string) Artifact {
-	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: None}
+func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
+	copy := *p
+	return &copy
 }
 
-func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
+func NewPackageArtifact(path string) *PackageArtifact {
+	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: compression.None}
+}
+
+func NewPackageArtifactFromYaml(data []byte) (*PackageArtifact, error) {
 	p := &PackageArtifact{Checksums: Checksums{}}
 	err := yaml.Unmarshal(data, &p)
 	if err != nil {
@@ -102,42 +84,6 @@ func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
 	return p, err
 }
 
-func LoadArtifactFromYaml(spec CompilationSpec) (Artifact, error) {
-
-	metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml"
-	dat, err := ioutil.ReadFile(spec.Rel(metaFile))
-	if err != nil {
-		return nil, errors.Wrap(err, "Error reading file "+metaFile)
-	}
-	art, err := NewPackageArtifactFromYaml(dat)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error writing file "+metaFile)
-	}
-	// It is relative, set it back to abs
-	art.SetPath(spec.Rel(art.GetPath()))
-	return art, nil
-}
-
-func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) {
-	a.CompressionType = t
-}
-
-func (a *PackageArtifact) GetChecksums() Checksums {
-	return a.Checksums
-}
-
-func (a *PackageArtifact) SetChecksums(c Checksums) {
-	a.Checksums = c
-}
-
-func (a *PackageArtifact) SetFiles(f []string) {
-	a.Files = f
-}
-
-func (a *PackageArtifact) GetFiles() []string {
-	return a.Files
-}
-
 func (a *PackageArtifact) Hash() error {
 	return a.Checksums.Generate(a)
 }
@@ -181,8 +127,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 	}
 	//p := a.CompileSpec.GetPackage().GetPath()
 
-	mangle.GetCompileSpec().GetPackage().SetPath("")
-	for _, ass := range mangle.GetCompileSpec().GetSourceAssertion() {
+	mangle.CompileSpec.GetPackage().SetPath("")
+	for _, ass := range mangle.CompileSpec.GetSourceAssertion() {
 		ass.Package.SetPath("")
 	}
 
@@ -191,7 +137,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
 	}
 
-	err = ioutil.WriteFile(filepath.Join(dst, a.GetCompileSpec().GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm)
+	err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetMetadataFilePath()), data, os.ModePerm)
 	if err != nil {
 		return errors.Wrap(err, "While writing PackageArtifact YAML")
 	}
@@ -201,66 +147,32 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 	return nil
 }
 
-func (a *PackageArtifact) GetSourceAssertion() solver.PackagesAssertions {
-	return a.SourceAssertion
-}
-
-func (a *PackageArtifact) SetCompileSpec(as CompilationSpec) {
-	a.CompileSpec = as.(*LuetCompilationSpec)
-}
-
-func (a *PackageArtifact) GetCompileSpec() CompilationSpec {
-	return a.CompileSpec
-}
-
-func (a *PackageArtifact) SetSourceAssertion(as solver.PackagesAssertions) {
-	a.SourceAssertion = as
-}
-
-func (a *PackageArtifact) GetDependencies() []Artifact {
-	ret := []Artifact{}
-	for _, d := range a.Dependencies {
-		ret = append(ret, d)
-	}
-	return ret
-}
-
-func (a *PackageArtifact) SetDependencies(d []Artifact) {
-	ret := []*PackageArtifact{}
-	for _, dd := range d {
-		ret = append(ret, dd.(*PackageArtifact))
-	}
-	a.Dependencies = ret
-}
-
-func (a *PackageArtifact) GetPath() string {
-	return a.Path
-}
-
 func (a *PackageArtifact) GetFileName() string {
-	return path.Base(a.GetPath())
-}
-
-func (a *PackageArtifact) SetPath(p string) {
-	a.Path = p
+	return path.Base(a.Path)
 }
 
 func (a *PackageArtifact) genDockerfile() string {
 	return `
 FROM scratch
-COPY * /`
+COPY . /`
 }
 
 // CreateArtifactForFile creates a new artifact from the given file
 func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
-
+	if _, err := os.Stat(s); os.IsNotExist(err) {
+		return nil, errors.Wrap(err, "artifact path doesn't exist")
+	}
 	fileName := path.Base(s)
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
 		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
 	}
 	defer os.RemoveAll(archive) // clean up
-	helpers.CopyFile(s, filepath.Join(archive, fileName))
+	dst := filepath.Join(archive, fileName)
+	if err := fileHelper.CopyFile(s, dst); err != nil {
+		return nil, errors.Wrapf(err, "error while copying %s to %s", s, dst)
+	}
+
 	artifact, err := LuetCfg.GetSystem().TempDir("artifact")
 	if err != nil {
 		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
@@ -274,9 +186,13 @@ func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageAr
 	return a, a.Compress(archive, 1)
 }
 
+type ImageBuilder interface {
+	BuildImage(backend.Options) error
+}
+
 // GenerateFinalImage takes an artifact and builds a Docker image with its content
-func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend, keepPerms bool) (CompilerBackendOptions, error) {
-	builderOpts := CompilerBackendOptions{}
+func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) {
+	builderOpts := backend.Options{}
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
 		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
@@ -290,16 +206,28 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend
 		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
 	}
 
+	if err := a.Unpack(uncompressedFiles, keepPerms); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
+	}
+
+	empty, err := fileHelper.DirectoryIsEmpty(uncompressedFiles)
+	if err != nil {
+		return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles)
+	}
+
+	// See https://github.com/moby/moby/issues/38039.
+	// We can't generate FROM scratch empty images. Docker will refuse to export them
+	// workaround: Inject a .virtual empty file
+	if empty {
+		fileHelper.Touch(filepath.Join(uncompressedFiles, ".virtual"))
+	}
+
 	data := a.genDockerfile()
 	if err := ioutil.WriteFile(dockerFile, []byte(data), 0644); err != nil {
 		return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path)
 	}
 
-	if err := a.Unpack(uncompressedFiles, keepPerms); err != nil {
-		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
-	}
-
-	builderOpts = CompilerBackendOptions{
+	builderOpts = backend.Options{
 		ImageName:      imageName,
 		SourcePath:     archive,
 		DockerFileName: dockerFile,
@@ -314,7 +242,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend
 func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	switch a.CompressionType {
 
-	case Zstandard:
+	case compression.Zstandard:
 		err := helpers.Tar(src, a.Path)
 		if err != nil {
 			return err
@@ -352,7 +280,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 
 		a.Path = zstdFile
 		return nil
-	case GZip:
+	case compression.GZip:
 		err := helpers.Tar(src, a.Path)
 		if err != nil {
 			return err
@@ -392,15 +320,14 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	default:
 		return helpers.Tar(src, a.getCompressedName())
 	}
-	return errors.New("Compression type must be supplied")
 }
 
 func (a *PackageArtifact) getCompressedName() string {
 	switch a.CompressionType {
-	case Zstandard:
+	case compression.Zstandard:
 		return a.Path + ".zst"
 
-	case GZip:
+	case compression.GZip:
 		return a.Path + ".gz"
 	}
 	return a.Path
@@ -409,12 +336,34 @@ func (a *PackageArtifact) getCompressedName() string {
 // GetUncompressedName returns the artifact path without the extension suffix
 func (a *PackageArtifact) GetUncompressedName() string {
 	switch a.CompressionType {
-	case Zstandard, GZip:
+	case compression.Zstandard, compression.GZip:
 		return strings.TrimSuffix(a.Path, filepath.Ext(a.Path))
 	}
 	return a.Path
 }
 
+func hashContent(bv []byte) string {
+	hasher := sha1.New()
+	hasher.Write(bv)
+	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	return sha
+}
+
+func hashFileContent(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	h := sha1.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+
+	return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil
+}
+
 func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
 	// If the destination path already exists I rename target file name with postfix.
 	var destPath string
@@ -426,6 +375,7 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return nil, nil, err
 		}
 	}
+	tarHash := hashContent(buffer.Bytes())
 
 	// If file is not present on archive but is defined on mods
 	// I receive the callback. Prevent nil exception.
@@ -438,13 +388,26 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return header, buffer.Bytes(), nil
 		}
 
+		existingHash := ""
+		f, err := os.Lstat(destPath)
+		if err == nil {
+			Debug("File exists already, computing hash for", destPath)
+			hash, herr := hashFileContent(destPath)
+			if herr == nil {
+				existingHash = hash
+			}
+		}
+
+		Debug("Existing file hash: ", existingHash, "Tar file hashsum: ", tarHash)
+		// We want to protect file only if the hash of the files are differing OR the file size are
+		differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
 		// Check if exists
-		if helpers.Exists(destPath) {
+		if fileHelper.Exists(destPath) && differs {
 			for i := 1; i < 1000; i++ {
 				name := filepath.Join(filepath.Join(filepath.Dir(path),
 					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
 
-				if helpers.Exists(name) {
+				if fileHelper.Exists(name) {
 					continue
 				}
 				Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
@@ -490,6 +453,9 @@ func (a *PackageArtifact) GetProtectFiles() []string {
 
 // Unpack Untar and decompress (TODO) to the given path
 func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+	if !strings.HasPrefix(dst, "/") {
+		return errors.New("destination must be an absolute path")
+	}
 
 	// Create
 	protectedFiles := a.GetProtectFiles()
@@ -497,13 +463,13 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
 
 	switch a.CompressionType {
-	case Zstandard:
+	case compression.Zstandard:
 		// Create the uncompressed archive
-		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		archive, err := os.Create(a.Path + ".uncompressed")
 		if err != nil {
 			return err
 		}
-		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer os.RemoveAll(a.Path + ".uncompressed")
 		defer archive.Close()
 
 		original, err := os.Open(a.Path)
@@ -522,22 +488,22 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 
 		_, err = io.Copy(archive, d)
 		if err != nil {
-			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
+			return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
 		}
 
-		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
 			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
 		}
 		return nil
-	case GZip:
+	case compression.GZip:
 		// Create the uncompressed archive
-		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		archive, err := os.Create(a.Path + ".uncompressed")
 		if err != nil {
 			return err
 		}
-		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer os.RemoveAll(a.Path + ".uncompressed")
 		defer archive.Close()
 
 		original, err := os.Open(a.Path)
@@ -555,10 +521,10 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 
 		_, err = io.Copy(archive, r)
 		if err != nil {
-			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
+			return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
 		}
 
-		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
 			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
@@ -566,7 +532,7 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 		return nil
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.UntarProtect(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner,
+		return helpers.UntarProtect(a.Path, dst, LuetCfg.GetGeneral().SameOwner,
 			protectedFiles, tarModifier)
 	}
 	return errors.New("Compression type must be supplied")
@@ -576,12 +542,12 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 func (a *PackageArtifact) FileList() ([]string, error) {
 	var tr *tar.Reader
 	switch a.CompressionType {
-	case Zstandard:
-		archive, err := os.Create(a.GetPath() + ".uncompressed")
+	case compression.Zstandard:
+		archive, err := os.Create(a.Path + ".uncompressed")
 		if err != nil {
 			return []string{}, err
 		}
-		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer os.RemoveAll(a.Path + ".uncompressed")
 		defer archive.Close()
 
 		original, err := os.Open(a.Path)
@@ -597,13 +563,13 @@ func (a *PackageArtifact) FileList() ([]string, error) {
 		}
 		defer r.Close()
 		tr = tar.NewReader(r)
-	case GZip:
+	case compression.GZip:
 		// Create the uncompressed archive
-		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		archive, err := os.Create(a.Path + ".uncompressed")
 		if err != nil {
 			return []string{}, err
 		}
-		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer os.RemoveAll(a.Path + ".uncompressed")
 		defer archive.Close()
 
 		original, err := os.Open(a.Path)
@@ -622,7 +588,7 @@ func (a *PackageArtifact) FileList() ([]string, error) {
 
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		tarFile, err := os.Open(a.GetPath())
+		tarFile, err := os.Open(a.Path)
 		if err != nil {
 			return []string{}, errors.Wrap(err, "Could not open package archive")
 		}
@@ -659,47 +625,16 @@ type CopyJob struct {
 	Artifact string
 }
 
-func copyXattr(srcPath, dstPath, attr string) error {
-	data, err := system.Lgetxattr(srcPath, attr)
-	if err != nil {
-		return err
-	}
-	if data != nil {
-		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func doCopyXattrs(srcPath, dstPath string) error {
-	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
-		return err
-	}
-
-	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
-}
-
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
 	for job := range s {
-		//Info("#"+strconv.Itoa(i), "copying", job.Src, "to", job.Dst)
-		// if dir, err := helpers.IsDirectory(job.Src); err == nil && dir {
-		// 	err = helpers.CopyDir(job.Src, job.Dst)
-		// 	if err != nil {
-		// 		Warning("Error copying dir", job, err)
-		// 	}
-		// 	continue
-		// }
-
 		_, err := os.Lstat(job.Dst)
 		if err != nil {
 			Debug("Copying ", job.Src)
-			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
+			if err := fileHelper.DeepCopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
-			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
@@ -717,8 +652,24 @@ func compileRegexes(regexes []string) []*regexp.Regexp {
 	return result
 }
 
+type ArtifactNode struct {
+	Name string `json:"Name"`
+	Size int    `json:"Size"`
+}
+type ArtifactDiffs struct {
+	Additions []ArtifactNode `json:"Adds"`
+	Deletions []ArtifactNode `json:"Dels"`
+	Changes   []ArtifactNode `json:"Mods"`
+}
+
+type ArtifactLayer struct {
+	FromImage string        `json:"Image1"`
+	ToImage   string        `json:"Image2"`
+	Diffs     ArtifactDiffs `json:"Diff"`
+}
+
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t CompressionImplementation) (Artifact, error) {
+func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) {
 
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
@@ -840,45 +791,10 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 	wg.Wait()
 
 	a := NewPackageArtifact(dst)
-	a.SetCompressionType(t)
+	a.CompressionType = t
 	err = a.Compress(archive, concurrency)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}
 	return a, nil
 }
-
-func ComputeArtifactLayerSummary(diffs []ArtifactLayer) ArtifactLayersSummary {
-
-	ans := ArtifactLayersSummary{
-		Layers: make([]ArtifactLayerSummary, 0),
-	}
-
-	for _, layer := range diffs {
-		sum := ArtifactLayerSummary{
-			FromImage:   layer.FromImage,
-			ToImage:     layer.ToImage,
-			AddFiles:    0,
-			AddSizes:    0,
-			DelFiles:    0,
-			DelSizes:    0,
-			ChangeFiles: 0,
-			ChangeSizes: 0,
-		}
-		for _, a := range layer.Diffs.Additions {
-			sum.AddFiles++
-			sum.AddSizes += int64(a.Size)
-		}
-		for _, d := range layer.Diffs.Deletions {
-			sum.DelFiles++
-			sum.DelSizes += int64(d.Size)
-		}
-		for _, c := range layer.Diffs.Changes {
-			sum.ChangeFiles++
-			sum.ChangeSizes += int64(c.Size)
-		}
-		ans.Layers = append(ans.Layers, sum)
-	}
-
-	return ans
-}

pkg/compiler/types/artifact/artifact_suite_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestArtifact(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Artifact Suite")
+}

pkg/compiler/types/artifact/artifact_test.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler_test
+package artifact_test
 
 import (
 	"io/ioutil"
@@ -22,10 +22,14 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
-	"github.com/mudler/luet/pkg/solver"
+	backend "github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/compiler/types/artifact"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -38,18 +42,15 @@ var _ = Describe("Artifact", func() {
 
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
+			err := generalRecipe.Load("../../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
 			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 			Expect(lspec.Image).To(Equal("luet/base"))
 			Expect(lspec.Seed).To(Equal("alpine"))
@@ -71,7 +72,7 @@ var _ = Describe("Artifact", func() {
 
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -81,7 +82,7 @@ ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin`))
 			b := NewSimpleDockerBackend()
-			opts := CompilerBackendOptions{
+			opts := backend.Options{
 				ImageName:      "luet/base",
 				SourcePath:     tmpdir,
 				DockerFileName: "Dockerfile",
@@ -89,12 +90,12 @@ ENV PACKAGE_CATEGORY=app-admin`))
 			}
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
@@ -105,7 +106,7 @@ ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
-			opts2 := CompilerBackendOptions{
+			opts2 := backend.Options{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
@@ -113,8 +114,8 @@ RUN echo bar > /test2`))
 			}
 			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
-			diffs, err := b.Changes(opts, opts2)
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
+			diffs, err := compiler.GenerateChanges(b, opts, opts2)
 			Expect(err).ToNot(HaveOccurred())
 
 			artifacts := []ArtifactNode{{
@@ -135,30 +136,30 @@ RUN echo bar > /test2`))
 						Additions: artifacts,
 					},
 				}}))
-			err = b.ExtractRootfs(CompilerBackendOptions{ImageName: "test", Destination: rootfs}, false)
+			err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, None)
+			a, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
-			err = helpers.Untar(artifact.GetPath(), unpacked, false)
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
+			err = helpers.Untar(a.Path, unpacked, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
-			content1, err := helpers.Read(filepath.Join(unpacked, "test"))
+			Expect(fileHelper.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
+			content1, err := fileHelper.Read(filepath.Join(unpacked, "test"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("foo\n"))
-			content2, err := helpers.Read(filepath.Join(unpacked, "test2"))
+			content2, err := fileHelper.Read(filepath.Join(unpacked, "test2"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content2).To(Equal("bar\n"))
 
-			err = artifact.Hash()
+			err = a.Hash()
 			Expect(err).ToNot(HaveOccurred())
-			err = artifact.Verify()
+			err = a.Verify()
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
+			Expect(fileHelper.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
 
-			err = artifact.Verify()
+			err = a.Verify()
 			Expect(err).To(HaveOccurred())
 		})
 
@@ -175,16 +176,21 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpWork) // clean up
 
+			Expect(os.MkdirAll(filepath.Join(tmpdir, "foo", "bar"), os.ModePerm)).ToNot(HaveOccurred())
+
 			err = ioutil.WriteFile(filepath.Join(tmpdir, "test"), testString, 0644)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
-			artifact.SetCompileSpec(&LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}})
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "foo", "bar", "test"), testString, 0644)
+			Expect(err).ToNot(HaveOccurred())
 
-			err = artifact.Compress(tmpdir, 1)
+			a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}
+
+			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := artifact.GenerateFinalImage(resultingImage, b, false)
+			opts, err := a.GenerateFinalImage(resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(opts.ImageName).To(Equal(resultingImage))
 
@@ -194,26 +200,69 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(result) // clean up
 
-			err = b.ExtractRootfs(CompilerBackendOptions{ImageName: resultingImage, Destination: result}, false)
+			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			content, err := ioutil.ReadFile(filepath.Join(result, "test"))
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(content).To(Equal(testString))
+
+			content, err = ioutil.ReadFile(filepath.Join(result, "foo", "bar", "test"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(content).To(Equal(testString))
+		})
+
+		It("Generates empty packages images", func() {
+			b := NewSimpleDockerBackend()
+			imageprefix := "foo/"
+
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "artifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpWork, err := ioutil.TempDir(os.TempDir(), "artifact2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpWork) // clean up
+
+			a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}
+
+			err = a.Compress(tmpdir, 1)
+			Expect(err).ToNot(HaveOccurred())
+			resultingImage := imageprefix + "foo--1.0"
+			opts, err := a.GenerateFinalImage(resultingImage, b, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(opts.ImageName).To(Equal(resultingImage))
+
+			Expect(b.ImageExists(resultingImage)).To(BeTrue())
+
+			result, err := ioutil.TempDir(os.TempDir(), "result")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(result) // clean up
+
+			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.DirectoryIsEmpty(result)).To(BeFalse())
+			content, err := ioutil.ReadFile(filepath.Join(result, ".virtual"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(content)).To(Equal(""))
 		})
 
 		It("Retrieves uncompressed name", func() {
 			a := NewPackageArtifact("foo.tar.gz")
-			a.SetCompressionType(compiler.GZip)
+			a.CompressionType = (compression.GZip)
 			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
 
 			a = NewPackageArtifact("foo.tar.zst")
-			a.SetCompressionType(compiler.Zstandard)
+			a.CompressionType = compression.Zstandard
 			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
 
 			a = NewPackageArtifact("foo.tar")
-			a.SetCompressionType(compiler.None)
+			a.CompressionType = compression.None
 			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
 		})
 	})

pkg/compiler/types/artifact/checksum.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler
+package artifact
 
 import (
 
@@ -43,7 +43,7 @@ type HashOptions struct {
 }
 
 // Generate generates all Checksums supported for the artifact
-func (c *Checksums) Generate(a Artifact) error {
+func (c *Checksums) Generate(a *PackageArtifact) error {
 	return c.generateSHA256(a)
 }
 
@@ -56,13 +56,13 @@ func (c Checksums) Compare(d Checksums) error {
 	return nil
 }
 
-func (c *Checksums) generateSHA256(a Artifact) error {
+func (c *Checksums) generateSHA256(a *PackageArtifact) error {
 	return c.generateSum(a, HashOptions{Hasher: sha256.New(), Type: SHA256})
 }
 
-func (c *Checksums) generateSum(a Artifact, opts HashOptions) error {
+func (c *Checksums) generateSum(a *PackageArtifact, opts HashOptions) error {
 
-	f, err := os.Open(a.GetPath())
+	f, err := os.Open(a.Path)
 	if err != nil {
 		return err
 	}

pkg/compiler/types/artifact/checksum_test.go

@@ -13,13 +13,13 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler_test
+package artifact_test
 
 import (
 	"io/ioutil"
 	"os"
 
-	. "github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/types/artifact"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -40,13 +40,13 @@ var _ = Describe("Checksum", func() {
 			Expect(len(definitionsum)).To(Equal(0))
 			Expect(len(definitionsum2)).To(Equal(0))
 
-			err = buildsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/build.yaml"))
+			err = buildsum.Generate(NewPackageArtifact("../../../../tests/fixtures/layers/alpine/build.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = definitionsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			err = definitionsum.Generate(NewPackageArtifact("../../../../tests/fixtures/layers/alpine/definition.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = definitionsum2.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			err = definitionsum2.Generate(NewPackageArtifact("../../../../tests/fixtures/layers/alpine/definition.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(buildsum)).To(Equal(1))

pkg/compiler/types/compression/compression.go

@@ -0,0 +1,9 @@
+package compression
+
+type Implementation string
+
+const (
+	None      Implementation = "none" // e.g. tar for standard packages
+	GZip      Implementation = "gzip"
+	Zstandard Implementation = "zstd"
+)

pkg/compiler/types/options/compiler_options.go

@@ -0,0 +1,199 @@
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package options
+
+import (
+	"runtime"
+
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/solver"
+)
+
+type Compiler struct {
+	PushImageRepository      string
+	PullImageRepository      []string
+	PullFirst, KeepImg, Push bool
+	Concurrency              int
+	CompressionType          compression.Implementation
+
+	Wait            bool
+	OnlyDeps        bool
+	NoDeps          bool
+	SolverOptions   config.LuetSolverOptions
+	BuildValuesFile []string
+	BuildValues     []map[string]interface{}
+
+	PackageTargetOnly bool
+	Rebuild           bool
+
+	BackendArgs []string
+
+	BackendType string
+}
+
+func NewDefaultCompiler() *Compiler {
+	return &Compiler{
+		PushImageRepository: "luet/cache",
+		PullFirst:           false,
+		Push:                false,
+		CompressionType:     compression.None,
+		KeepImg:             true,
+		Concurrency:         runtime.NumCPU(),
+		OnlyDeps:            false,
+		NoDeps:              false,
+		SolverOptions:       config.LuetSolverOptions{Options: solver.Options{Concurrency: 1, Type: solver.SingleCoreSimple}},
+	}
+}
+
+type Option func(cfg *Compiler) error
+
+func (cfg *Compiler) Apply(opts ...Option) error {
+	for _, opt := range opts {
+		if opt == nil {
+			continue
+		}
+		if err := opt(cfg); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func WithOptions(opt *Compiler) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg = opt
+		return nil
+	}
+}
+
+func WithBackendType(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BackendType = r
+		return nil
+	}
+}
+
+func WithBuildValues(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BuildValuesFile = r
+		return nil
+	}
+}
+
+func WithPullRepositories(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PullImageRepository = r
+		return nil
+	}
+}
+
+func WithPushRepository(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		if len(cfg.PullImageRepository) == 0 {
+			cfg.PullImageRepository = []string{cfg.PushImageRepository}
+		}
+		cfg.PushImageRepository = r
+		return nil
+	}
+}
+
+func BackendArgs(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BackendArgs = r
+		return nil
+	}
+}
+
+func PullFirst(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PullFirst = b
+		return nil
+	}
+}
+
+func KeepImg(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.KeepImg = b
+		return nil
+	}
+}
+
+func Rebuild(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Rebuild = b
+		return nil
+	}
+}
+
+func PushImages(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Push = b
+		return nil
+	}
+}
+
+func Wait(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Wait = b
+		return nil
+	}
+}
+
+func OnlyDeps(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.OnlyDeps = b
+		return nil
+	}
+}
+
+func OnlyTarget(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PackageTargetOnly = b
+		return nil
+	}
+}
+
+func NoDeps(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.NoDeps = b
+		return nil
+	}
+}
+
+func Concurrency(i int) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		if i == 0 {
+			i = runtime.NumCPU()
+		}
+		cfg.Concurrency = i
+		return nil
+	}
+}
+
+func WithCompressionType(t compression.Implementation) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.CompressionType = t
+		return nil
+	}
+}
+
+func WithSolverOptions(c config.LuetSolverOptions) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.SolverOptions = c
+		return nil
+	}
+}

pkg/compiler/types/spec/spec.go

@@ -13,21 +13,26 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler
+package compilerspec
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
 
+	"github.com/mitchellh/hashstructure/v2"
+	options "github.com/mudler/luet/pkg/compiler/types/options"
+
+	"github.com/ghodss/yaml"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/otiai10/copy"
-	yaml "gopkg.in/yaml.v2"
+	dirhash "golang.org/x/mod/sumdb/dirhash"
 )
 
 type LuetCompilationspecs []LuetCompilationSpec
 
-func NewLuetCompilationspecs(s ...CompilationSpec) CompilationSpecs {
+func NewLuetCompilationspecs(s ...*LuetCompilationSpec) *LuetCompilationspecs {
 	all := LuetCompilationspecs{}
 
 	for _, spec := range s {
@@ -40,7 +45,7 @@ func (specs LuetCompilationspecs) Len() int {
 	return len(specs)
 }
 
-func (specs *LuetCompilationspecs) Remove(s CompilationSpecs) CompilationSpecs {
+func (specs *LuetCompilationspecs) Remove(s *LuetCompilationspecs) *LuetCompilationspecs {
 	newSpecs := LuetCompilationspecs{}
 SPECS:
 	for _, spec := range specs.All() {
@@ -54,16 +59,12 @@ SPECS:
 	return &newSpecs
 }
 
-func (specs *LuetCompilationspecs) Add(s CompilationSpec) {
-	c, ok := s.(*LuetCompilationSpec)
-	if !ok {
-		panic("LuetCompilationspecs supports only []LuetCompilationSpec")
-	}
-	*specs = append(*specs, *c)
+func (specs *LuetCompilationspecs) Add(s *LuetCompilationSpec) {
+	*specs = append(*specs, *s)
 }
 
-func (specs *LuetCompilationspecs) All() []CompilationSpec {
-	var cspecs []CompilationSpec
+func (specs *LuetCompilationspecs) All() []*LuetCompilationSpec {
+	var cspecs []*LuetCompilationSpec
 	for i, _ := range *specs {
 		f := (*specs)[i]
 		cspecs = append(cspecs, &f)
@@ -72,7 +73,7 @@ func (specs *LuetCompilationspecs) All() []CompilationSpec {
 	return cspecs
 }
 
-func (specs *LuetCompilationspecs) Unique() CompilationSpecs {
+func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
 	newSpecs := LuetCompilationspecs{}
 	seen := map[string]bool{}
 
@@ -87,6 +88,13 @@ func (specs *LuetCompilationspecs) Unique() CompilationSpecs {
 	return &newSpecs
 }
 
+type CopyField struct {
+	Package     *pkg.DefaultPackage `json:"package"`
+	Image       string              `json:"image"`
+	Source      string              `json:"source"`
+	Destination string              `json:"destination"`
+}
+
 type LuetCompilationSpec struct {
 	Steps           []string                  `json:"steps"` // Are run inside a container and the result layer diff is saved
 	Env             []string                  `json:"env"`
@@ -103,23 +111,89 @@ type LuetCompilationSpec struct {
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
 	Excludes   []string `json:"excludes"`
+
+	BuildOptions *options.Compiler `json:"build_options"`
+
+	Copy []CopyField `json:"copy"`
+
+	Join                pkg.DefaultPackages `json:"join"`
+	RequiresFinalImages bool                `json:"requires_final_images" yaml:"requires_final_images"`
 }
 
-func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
+// Signature is a portion of the spec that yields a signature for the hash
+type Signature struct {
+	Image               string
+	Steps               []string
+	PackageDir          string
+	Prelude             []string
+	Seed                string
+	Env                 []string
+	Retrieve            []string
+	Unpack              bool
+	Includes            []string
+	Excludes            []string
+	Copy                []CopyField
+	Join                pkg.DefaultPackages
+	RequiresFinalImages bool
+}
+
+func (cs *LuetCompilationSpec) signature() Signature {
+	return Signature{
+		Image:               cs.Image,
+		Steps:               cs.Steps,
+		PackageDir:          cs.PackageDir,
+		Prelude:             cs.Prelude,
+		Seed:                cs.Seed,
+		Env:                 cs.Env,
+		Retrieve:            cs.Retrieve,
+		Unpack:              cs.Unpack,
+		Includes:            cs.Includes,
+		Excludes:            cs.Excludes,
+		Copy:                cs.Copy,
+		Join:                cs.Join,
+		RequiresFinalImages: cs.RequiresFinalImages,
+	}
+}
+
+func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
 	var spec LuetCompilationSpec
+	var packageDefinition pkg.DefaultPackage
 	err := yaml.Unmarshal(b, &spec)
 	if err != nil {
 		return &spec, err
 	}
-	spec.Package = p.(*pkg.DefaultPackage)
+	err = yaml.Unmarshal(b, &packageDefinition)
+	if err != nil {
+		return &spec, err
+	}
+
+	// Update requires/conflict/provides
+	// When we have been passed a bytes slice, parse it as a package
+	// and updates requires/conflicts/provides.
+	// This is required in order to allow manipulation of such fields with templating
+	copy := *p.(*pkg.DefaultPackage)
+	spec.Package = &copy
+	if len(packageDefinition.GetRequires()) != 0 {
+		spec.Package.Requires(packageDefinition.GetRequires())
+	}
+	if len(packageDefinition.GetConflicts()) != 0 {
+		spec.Package.Conflicts(packageDefinition.GetConflicts())
+	}
+	if len(packageDefinition.GetProvides()) != 0 {
+		spec.Package.SetProvides(packageDefinition.GetProvides())
+	}
 	return &spec, nil
 }
-func (a *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
-	return a.SourceAssertion
+func (cs *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
+	return cs.SourceAssertion
 }
 
-func (a *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) {
-	a.SourceAssertion = as
+func (cs *LuetCompilationSpec) SetBuildOptions(b options.Compiler) {
+	cs.BuildOptions = &b
+}
+
+func (cs *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) {
+	cs.SourceAssertion = as
 }
 func (cs *LuetCompilationSpec) GetPackage() pkg.Package {
 	return cs.Package
@@ -157,6 +231,12 @@ func (cs *LuetCompilationSpec) GetRetrieve() []string {
 	return cs.Retrieve
 }
 
+// IsVirtual returns true if the spec is virtual.
+// A spec is virtual if the package is empty, and it has no image source to unpack from.
+func (cs *LuetCompilationSpec) IsVirtual() bool {
+	return cs.EmptyPackage() && !cs.HasImageSource()
+}
+
 func (cs *LuetCompilationSpec) GetSeedImage() string {
 	return cs.Seed
 }
@@ -186,7 +266,7 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 }
 
 func (cs *LuetCompilationSpec) EmptyPackage() bool {
-	return len(cs.BuildSteps()) == 0 && len(cs.GetPreBuildSteps()) == 0 && !cs.UnpackedPackage()
+	return len(cs.BuildSteps()) == 0 && !cs.UnpackedPackage()
 }
 
 func (cs *LuetCompilationSpec) UnpackedPackage() bool {
@@ -199,8 +279,25 @@ func (cs *LuetCompilationSpec) UnpackedPackage() bool {
 	return unpack
 }
 
+// HasImageSource returns true when the compilation spec has an image source.
+// a compilation spec has an image source when it depends on other packages or have a source image
+// explictly supplied
 func (cs *LuetCompilationSpec) HasImageSource() bool {
-	return len(cs.GetPackage().GetRequires()) != 0 || cs.GetImage() != ""
+	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != "" || len(cs.Join) != 0
+}
+
+func (cs *LuetCompilationSpec) Hash() (string, error) {
+	// build a signature, we want to be part of the hash only the fields that are relevant for build purposes
+	signature := cs.signature()
+	h, err := hashstructure.Hash(signature, hashstructure.FormatV2, nil)
+	if err != nil {
+		return "", err
+	}
+	sum, err := dirhash.HashDir(cs.Package.Path, "", dirhash.DefaultHash)
+	if err != nil {
+		return fmt.Sprint(h), err
+	}
+	return fmt.Sprint(h, sum), err
 }
 
 func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
@@ -243,6 +340,13 @@ ADD ` + s + ` /luetbuild/`
 		}
 	}
 
+	for _, c := range cs.Copy {
+		if c.Image != "" {
+			copyLine := fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
+			spec = spec + copyLine
+		}
+	}
+
 	for _, s := range cs.Env {
 		spec = spec + `
 ENV ` + s

pkg/compiler/types/spec/spec_suite_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compilerspec_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSpec(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Spec Suite")
+}

pkg/compiler/types/spec/spec_test.go

@@ -13,17 +13,19 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler_test
+package compilerspec_test
 
 import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
+	options "github.com/mudler/luet/pkg/compiler/types/options"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	. "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -32,43 +34,116 @@ import (
 var _ = Describe("Spec", func() {
 	Context("Luet specs", func() {
 		It("Allows normal operations", func() {
-			testSpec := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
-			testSpec2 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}}
-			testSpec3 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}}
-			testSpec4 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
+			testSpec := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
+			testSpec2 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}}
+			testSpec3 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}}
+			testSpec4 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
 
-			specs := NewLuetCompilationspecs(testSpec, testSpec2)
+			specs := compilerspec.NewLuetCompilationspecs(testSpec, testSpec2)
 			Expect(specs.Len()).To(Equal(2))
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2}))
+			Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2}))
 			specs.Add(testSpec3)
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
+			Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
 			specs.Add(testSpec4)
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
+			Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
 			newSpec := specs.Unique()
-			Expect(newSpec.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
+			Expect(newSpec.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
 
-			newSpec2 := specs.Remove(NewLuetCompilationspecs(testSpec, testSpec2))
-			Expect(newSpec2.All()).To(Equal([]CompilationSpec{testSpec3}))
+			newSpec2 := specs.Remove(compilerspec.NewLuetCompilationspecs(testSpec, testSpec2))
+			Expect(newSpec2.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec3}))
 
 		})
+		Context("virtuals", func() {
+			When("is empty", func() {
+				It("is virtual", func() {
+					spec := &compilerspec.LuetCompilationSpec{}
+					Expect(spec.IsVirtual()).To(BeTrue())
+				})
+			})
+			When("has defined steps", func() {
+				It("is not a virtual", func() {
+					spec := &compilerspec.LuetCompilationSpec{Steps: []string{"foo"}}
+					Expect(spec.IsVirtual()).To(BeFalse())
+				})
+			})
+			When("has defined image", func() {
+				It("is not a virtual", func() {
+					spec := &compilerspec.LuetCompilationSpec{Image: "foo"}
+					Expect(spec.IsVirtual()).To(BeFalse())
+				})
+			})
+		})
+	})
+
+	Context("Image hashing", func() {
+		It("is stable", func() {
+			spec1 := &compilerspec.LuetCompilationSpec{
+				Image:        "foo",
+				BuildOptions: &options.Compiler{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
+
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			spec2 := &compilerspec.LuetCompilationSpec{
+				Image:        "foo",
+				BuildOptions: &options.Compiler{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			spec3 := &compilerspec.LuetCompilationSpec{
+				Image: "foo",
+				Steps: []string{"foo"},
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			hash, err := spec1.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			hash2, err := spec2.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			hash3, err := spec3.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal(hash2))
+			hashagain, err := spec2.Hash()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hash).ToNot(Equal(hash3))
+			Expect(hash).To(Equal(hashagain))
+		})
 	})
 
 	Context("Simple package build definition", func() {
 		It("Loads it correctly", func() {
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
+			err := generalRecipe.Load("../../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
 			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 			Expect(lspec.Image).To(Equal("luet/base"))
 			Expect(lspec.Seed).To(Equal("alpine"))
@@ -79,7 +154,7 @@ var _ = Describe("Spec", func() {
 			lspec.Env = []string{"test=1"}
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -92,7 +167,7 @@ ENV test=1`))
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
@@ -112,18 +187,15 @@ RUN echo bar > /test2`))
 	It("Renders retrieve and env fields", func() {
 		generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-		err := generalRecipe.Load("../../tests/fixtures/retrieve")
+		err := generalRecipe.Load("../../../../tests/fixtures/retrieve")
 		Expect(err).ToNot(HaveOccurred())
 
 		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-		spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+		lspec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
 		Expect(err).ToNot(HaveOccurred())
 
-		lspec, ok := spec.(*LuetCompilationSpec)
-		Expect(ok).To(BeTrue())
-
 		Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 		Expect(lspec.Image).To(Equal("luet/base"))
 		Expect(lspec.Seed).To(Equal("alpine"))
@@ -133,7 +205,7 @@ RUN echo bar > /test2`))
 
 		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
-		dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
 		Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -150,7 +222,7 @@ ENV test=1`))
 
 		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
-		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
 		Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -165,7 +237,7 @@ ENV test=1`))
 
 		err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
-		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
 
 		Expect(dockerfile).To(Equal(`

pkg/config/config.go

@@ -27,7 +27,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
 	solver "github.com/mudler/luet/pkg/solver"
 
@@ -65,6 +64,7 @@ type LuetGeneralConfig struct {
 }
 
 type LuetSolverOptions struct {
+	solver.Options
 	Type           string            `mapstructure:"type"`
 	LearnRate      float32           `mapstructure:"rate"`
 	Discount       float32           `mapstructure:"discount"`
@@ -107,6 +107,19 @@ type LuetSystemConfig struct {
 	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
 }
 
+func (s *LuetSystemConfig) SetRootFS(path string) error {
+	pathToSet := path
+	if !filepath.IsAbs(path) {
+		abs, err := filepath.Abs(path)
+		if err != nil {
+			return err
+		}
+		pathToSet = abs
+	}
+	s.Rootfs = pathToSet
+	return nil
+}
+
 func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
 	dbpath = filepath.Join(dbpath, "repos/"+name)
@@ -159,8 +172,9 @@ type LuetRepository struct {
 	Enable         bool              `json:"enable" yaml:"enable" mapstructure:"enable"`
 	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
 	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
-	TreePath       string            `json:"tree_path,omitempty" yaml:"tree_path,omitempty" mapstructure:"tree_path"`
-	MetaPath       string            `json:"meta_path,omitempty" yaml:"meta_path,omitempty" mapstructure:"meta_path"`
+	TreePath       string            `json:"treepath,omitempty" yaml:"treepath,omitempty" mapstructure:"treepath"`
+	MetaPath       string            `json:"metapath,omitempty" yaml:"metapath,omitempty" mapstructure:"metapath"`
+	Verify         bool              `json:"verify,omitempty" yaml:"verify,omitempty" mapstructure:"verify"`
 
 	// Serialized options not used in repository configuration
 
@@ -404,8 +418,13 @@ system:
 }
 
 func (c *LuetSystemConfig) InitTmpDir() error {
-	if !helpers.Exists(c.TmpDirBase) {
-		return os.MkdirAll(c.TmpDirBase, os.ModePerm)
+	if _, err := os.Stat(c.TmpDirBase); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(c.TmpDirBase, os.ModePerm)
+			if err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }

pkg/config/config_test.go

@@ -22,7 +22,7 @@ import (
 	"strings"
 
 	config "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -38,7 +38,7 @@ var _ = Describe("Config", func() {
 			tmpDir, err := config.LuetCfg.GetSystem().TempDir("test1")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(strings.HasPrefix(tmpDir, filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
-			Expect(helpers.Exists(tmpDir)).To(BeTrue())
+			Expect(fileHelper.Exists(tmpDir)).To(BeTrue())
 
 			defer os.RemoveAll(tmpDir)
 		})
@@ -49,7 +49,7 @@ var _ = Describe("Config", func() {
 			tmpFile, err := config.LuetCfg.GetSystem().TempFile("testfile1")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(strings.HasPrefix(tmpFile.Name(), filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
-			Expect(helpers.Exists(tmpFile.Name())).To(BeTrue())
+			Expect(fileHelper.Exists(tmpFile.Name())).To(BeTrue())
 
 			defer os.Remove(tmpFile.Name())
 		})

pkg/helpers/archive.go

@@ -90,14 +90,11 @@ func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modi
 	}
 
 	if sameOwner {
-		// PRE: i have root privileged.
-
+		// we do have root permissions, so we can extract keeping the same permissions.
 		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
 
 		opts := &archive.TarOptions{
-			// NOTE: NoLchown boolean is used for chmod of the symlink
-			// Probably it's needed set this always to true.
-			NoLchown:        true,
+			NoLchown:        false,
 			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
 			ContinueOnError: true,
 		}
@@ -201,12 +198,8 @@ func Untar(src, dest string, sameOwner bool) error {
 	defer in.Close()
 
 	if sameOwner {
-		// PRE: i have root privileged.
-
 		opts := &archive.TarOptions{
-			// NOTE: NoLchown boolean is used for chmod of the symlink
-			// Probably it's needed set this always to true.
-			NoLchown:        true,
+			NoLchown:        false,
 			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
 			ContinueOnError: true,
 		}

pkg/helpers/archive_test.go

@@ -25,6 +25,8 @@ import (
 	"os"
 	"path/filepath"
 
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	"github.com/docker/docker/pkg/archive"
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
@@ -128,7 +130,7 @@ var _ = Describe("Helpers Archive", func() {
 			err = archive.Untar(replacerArchive, targetDir, opts)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
+			Expect(fileHelper.Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
 		})
 	})
 })

pkg/helpers/docker/docker.go

@@ -0,0 +1,261 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package docker
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/containerd/containerd/images"
+	"github.com/mudler/luet/pkg/helpers/imgworker"
+
+	continerdarchive "github.com/containerd/containerd/archive"
+	"github.com/docker/cli/cli/trust"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/registry"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/mudler/luet/pkg/bus"
+	"github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"github.com/theupdateframework/notary/tuf/data"
+)
+
+// See also https://github.com/docker/cli/blob/88c6089300a82d3373892adf6845a4fed1a4ba8d/cli/command/image/trust.go#L171
+
+func verifyImage(image string, authConfig *types.AuthConfig) (string, error) {
+	ref, err := reference.ParseAnyReference(image)
+	if err != nil {
+		return "", errors.Wrapf(err, "invalid reference %s", image)
+	}
+
+	// only check if image ref doesn't contain hashes
+	if _, ok := ref.(reference.Digested); !ok {
+		namedRef, ok := ref.(reference.Named)
+		if !ok {
+			return "", errors.New("failed to resolve image digest using content trust: reference is not named")
+		}
+		namedRef = reference.TagNameOnly(namedRef)
+		taggedRef, ok := namedRef.(reference.NamedTagged)
+		if !ok {
+			return "", errors.New("failed to resolve image digest using content trust: reference is not tagged")
+		}
+
+		resolvedImage, err := trustedResolveDigest(context.Background(), taggedRef, authConfig, "luet")
+		if err != nil {
+			return "", errors.Wrap(err, "failed to resolve image digest using content trust")
+		}
+		resolvedFamiliar := reference.FamiliarString(resolvedImage)
+		return resolvedFamiliar, nil
+	}
+
+	return "", nil
+}
+
+func trustedResolveDigest(ctx context.Context, ref reference.NamedTagged, authConfig *types.AuthConfig, useragent string) (reference.Canonical, error) {
+	repoInfo, err := registry.ParseRepositoryInfo(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	notaryRepo, err := trust.GetNotaryRepository(os.Stdin, os.Stdout, useragent, repoInfo, authConfig, "pull")
+	if err != nil {
+		return nil, errors.Wrap(err, "error establishing connection to trust repository")
+	}
+
+	t, err := notaryRepo.GetTargetByName(ref.Tag(), trust.ReleasesRole, data.CanonicalTargetsRole)
+	if err != nil {
+		return nil, trust.NotaryError(repoInfo.Name.Name(), err)
+	}
+	// Only get the tag if it's in the top level targets role or the releases delegation role
+	// ignore it if it's in any other delegation roles
+	if t.Role != trust.ReleasesRole && t.Role != data.CanonicalTargetsRole {
+		return nil, trust.NotaryError(repoInfo.Name.Name(), errors.Errorf("No trust data for %s", reference.FamiliarString(ref)))
+	}
+
+	h, ok := t.Hashes["sha256"]
+	if !ok {
+		return nil, errors.New("no valid hash, expecting sha256")
+	}
+
+	dgst := digest.NewDigestFromHex("sha256", hex.EncodeToString(h))
+
+	// Allow returning canonical reference with tag
+	return reference.WithDigest(ref, dgst)
+}
+
+type staticAuth struct {
+	auth *types.AuthConfig
+}
+
+func (s staticAuth) Authorization() (*authn.AuthConfig, error) {
+	if s.auth == nil {
+		return nil, nil
+	}
+	return &authn.AuthConfig{
+		Username:      s.auth.Username,
+		Password:      s.auth.Password,
+		Auth:          s.auth.Auth,
+		IdentityToken: s.auth.IdentityToken,
+		RegistryToken: s.auth.RegistryToken,
+	}, nil
+}
+
+// UnpackEventData is the data structure to pass for the bus events
+type UnpackEventData struct {
+	Image string
+	Dest  string
+}
+
+// privilegedExtractImage uses the imgworker (which requires privileges) to extract a container image
+func privilegedExtractImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) {
+	defer os.RemoveAll(temp)
+	c, err := imgworker.New(temp, auth)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed creating client")
+	}
+	defer c.Close()
+
+	listedImage, err := c.Pull(image)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed listing images")
+	}
+
+	os.RemoveAll(dest)
+
+	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	err = c.Unpack(image, dest)
+
+	bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	return listedImage, err
+}
+
+// UnarchiveLayers extract layers with archive.Untar from docker instead of containerd
+func UnarchiveLayers(temp string, img v1.Image, image, dest string, auth *types.AuthConfig, verify bool) (int64, error) {
+	layers, err := img.Layers()
+	if err != nil {
+		return 0, fmt.Errorf("reading layers from '%s' image failed: %v", image, err)
+	}
+	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	var size int64
+	for _, l := range layers {
+		s, err := l.Size()
+		if err != nil {
+			return 0, fmt.Errorf("reading layer size from '%s' image failed: %v", image, err)
+		}
+		size += s
+
+		layerReader, err := l.Uncompressed()
+		if err != nil {
+			return 0, fmt.Errorf("reading uncompressed layer from '%s' image failed: %v", image, err)
+		}
+		defer layerReader.Close()
+
+		// Unpack the tarfile to the rootfs path.
+		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
+		if err := archive.Untar(layerReader, dest, &archive.TarOptions{
+			NoLchown:        false,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+		}); err != nil {
+			return 0, fmt.Errorf("extracting '%s' image to directory %s failed: %v", image, dest, err)
+		}
+	}
+	bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	return size, nil
+}
+
+// DownloadAndExtractDockerImage extracts a container image natively. It supports privileged/unprivileged mode
+func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) {
+	if verify {
+		img, err := verifyImage(image, auth)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed verifying image")
+		}
+		image = img
+	}
+
+	if os.Getenv("LUET_PRIVILEGED_EXTRACT") == "true" {
+		return privilegedExtractImage(temp, image, dest, auth, verify)
+	}
+
+	ref, err := name.ParseReference(image)
+	if err != nil {
+		return nil, err
+	}
+
+	img, err := remote.Image(ref, remote.WithAuth(staticAuth{auth}))
+	if err != nil {
+		return nil, err
+	}
+
+	m, err := img.Manifest()
+	if err != nil {
+		return nil, err
+	}
+
+	mt, err := img.MediaType()
+	if err != nil {
+		return nil, err
+	}
+
+	d, err := img.Digest()
+	if err != nil {
+		return nil, err
+	}
+
+	reader := mutate.Extract(img)
+	defer reader.Close()
+	defer os.RemoveAll(temp)
+
+	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	c, err := continerdarchive.Apply(context.TODO(), dest, reader)
+	if err != nil {
+		return nil, err
+	}
+
+	bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	return &imgworker.ListedImage{
+		Image: images.Image{
+			Name:   image,
+			Labels: m.Annotations,
+			Target: specs.Descriptor{
+				MediaType: string(mt),
+				Digest:    digest.Digest(d.String()),
+				Size:      c,
+			},
+		},
+		ContentSize: c,
+	}, nil
+}
+
+func StripInvalidStringsFromImage(s string) string {
+	return strings.ReplaceAll(s, "+", "-")
+}

pkg/helpers/docker/docker_test.go

@@ -0,0 +1,30 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package docker_test
+
+import (
+	"github.com/mudler/luet/pkg/helpers/docker"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("StripInvalidStringsFromImage", func() {
+	Context("Image names", func() {
+		It("strips invalid chars", func() {
+			Expect(docker.StripInvalidStringsFromImage("foo+bar")).To(Equal("foo-bar"))
+		})
+	})
+})

pkg/helpers/file/file.go

@@ -13,10 +13,13 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package file
 
 import (
+	"fmt"
+	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"sort"
@@ -24,10 +27,42 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/docker/docker/pkg/system"
+	"github.com/google/renameio"
 	copy "github.com/otiai10/copy"
 	"github.com/pkg/errors"
 )
 
+var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+func RandStringRunes(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letterRunes[rand.Intn(len(letterRunes))]
+	}
+	return string(b)
+}
+
+func Move(src, dst string) error {
+	f, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	t, err := renameio.TempFile("", dst)
+	if err != nil {
+		return err
+	}
+	defer t.Cleanup()
+
+	_, err = io.Copy(t, f)
+	if err != nil {
+		return err
+	}
+	return t.CloseAtomicallyReplace()
+}
+
 func OrderFiles(target string, files []string) ([]string, []string) {
 
 	var newFiles []string
@@ -82,6 +117,20 @@ func ListDir(dir string) ([]string, error) {
 	return content, err
 }
 
+// DirectoryIsEmpty Checks wether the directory is empty or not
+func DirectoryIsEmpty(dir string) (bool, error) {
+	f, err := os.Open(dir)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	if _, err = f.Readdirnames(1); err == io.EOF {
+		return true, nil
+	}
+	return false, nil
+}
+
 // Touch creates an empty file
 func Touch(f string) error {
 	_, err := os.Stat(f)
@@ -119,9 +168,27 @@ func Read(file string) (string, error) {
 	return string(dat), nil
 }
 
+func EnsureDirPerm(src, dst string) {
+	if info, err := os.Lstat(filepath.Dir(src)); err == nil {
+		if _, err := os.Lstat(filepath.Dir(dst)); os.IsNotExist(err) {
+			err := os.MkdirAll(filepath.Dir(dst), info.Mode().Perm())
+			if err != nil {
+				fmt.Println("warning: failed creating", filepath.Dir(dst), err.Error())
+			}
+			if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+				if err := os.Lchown(filepath.Dir(dst), int(stat.Uid), int(stat.Gid)); err != nil {
+					fmt.Println("warning: failed chowning", filepath.Dir(dst), err.Error())
+				}
+			}
+		}
+	} else {
+		EnsureDir(dst)
+	}
+}
+
 func EnsureDir(fileName string) error {
 	dirName := filepath.Dir(fileName)
-	if _, serr := os.Stat(dirName); serr != nil {
+	if _, serr := os.Stat(dirName); os.IsNotExist(serr) {
 		merr := os.MkdirAll(dirName, os.ModePerm) // FIXME: It should preserve permissions from src to dst instead
 		if merr != nil {
 			return merr
@@ -130,12 +197,39 @@ func EnsureDir(fileName string) error {
 	return nil
 }
 
-// CopyFile copies the contents of the file named src to the file named
+func CopyFile(src, dst string) (err error) {
+	return copy.Copy(src, dst, copy.Options{
+		Sync:      true,
+		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
+}
+
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
+// DeepCopyFile copies the contents of the file named src to the file named
 // by dst. The file will be created if it does not already exist. If the
 // destination file exists, all it's contents will be replaced by the contents
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
-func CopyFile(src, dst string) (err error) {
+func DeepCopyFile(src, dst string) (err error) {
 	// Workaround for https://github.com/otiai10/copy/issues/47
 	fi, err := os.Lstat(src)
 	if err != nil {
@@ -145,7 +239,7 @@ func CopyFile(src, dst string) (err error) {
 	fm := fi.Mode()
 	switch {
 	case fm&os.ModeNamedPipe != 0:
-		EnsureDir(dst)
+		EnsureDirPerm(src, dst)
 		if err := syscall.Mkfifo(dst, uint32(fi.Mode())); err != nil {
 			return errors.Wrap(err, "failed creating pipe")
 		}
@@ -157,9 +251,22 @@ func CopyFile(src, dst string) (err error) {
 		return nil
 	}
 
-	return copy.Copy(src, dst, copy.Options{
+	//filepath.Dir(src)
+	EnsureDirPerm(src, dst)
+
+	err = copy.Copy(src, dst, copy.Options{
 		Sync:      true,
 		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
+	if err != nil {
+		return err
+	}
+	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
+		if err := os.Lchown(dst, int(stat.Uid), int(stat.Gid)); err != nil {
+			fmt.Println("warning: failed chowning", dst, err.Error())
+		}
+	}
+
+	return doCopyXattrs(src, dst)
 }
 
 func IsDirectory(path string) (bool, error) {

pkg/helpers/file/file_test.go

@@ -13,14 +13,15 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers_test
+package file_test
 
 import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
-	. "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -28,8 +29,25 @@ import (
 var _ = Describe("Helpers", func() {
 	Context("Exists", func() {
 		It("Detect existing and not-existing files", func() {
-			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml")).To(BeTrue())
-			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
+			Expect(fileHelper.Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml")).To(BeTrue())
+			Expect(fileHelper.Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
+		})
+	})
+
+	Context("DirectoryIsEmpty", func() {
+		It("Detects empty directory", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+			Expect(fileHelper.DirectoryIsEmpty(testDir)).To(BeTrue())
+		})
+		It("Detects directory with files", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+			err = fileHelper.Touch(filepath.Join(testDir, "foo"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.DirectoryIsEmpty(testDir)).To(BeFalse())
 		})
 	})
 
@@ -55,7 +73,7 @@ var _ = Describe("Helpers", func() {
 			err = ioutil.WriteFile(filepath.Join(testDir, "baz2", "foo"), []byte("test\n"), 0644)
 			Expect(err).ToNot(HaveOccurred())
 
-			ordered, notExisting := OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
+			ordered, notExisting := fileHelper.OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
 
 			Expect(ordered).To(Equal([]string{"baz", "bar/foo", "foo", "baz2/foo", "bar", "baz2"}))
 			Expect(notExisting).To(Equal([]string{"notexisting"}))
@@ -79,7 +97,7 @@ var _ = Describe("Helpers", func() {
 			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz", "fa"), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			ordered, _ := OrderFiles(testDir, []string{"foo", "foo/bar", "bar", "foo/baz/fa", "foo/baz"})
+			ordered, _ := fileHelper.OrderFiles(testDir, []string{"foo", "foo/bar", "bar", "foo/baz/fa", "foo/baz"})
 			Expect(ordered).To(Equal([]string{"foo/baz/fa", "foo/bar", "foo/baz", "foo", "bar"}))
 		})
 	})

pkg/helpers/helm.go

@@ -3,6 +3,9 @@ package helpers
 import (
 	"io/ioutil"
 
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
+	"github.com/imdario/mergo"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 	"helm.sh/helm/v3/pkg/chart"
@@ -37,13 +40,44 @@ func RenderHelm(template string, values, d map[string]interface{}) (string, erro
 
 type templatedata map[string]interface{}
 
-func RenderFiles(toTemplate, valuesFile string, defaultFile string) (string, error) {
+// UnMarshalValues unmarshal values files and joins them into a unique templatedata
+// the join happens from right to left, so any rightmost value file overwrites the content of the ones before it.
+func UnMarshalValues(values []string) (templatedata, error) {
+	dst := templatedata{}
+	if len(values) > 0 {
+		for _, bv := range reverse(values) {
+			current := templatedata{}
+
+			defBuild, err := ioutil.ReadFile(bv)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+bv)
+			}
+			err = yaml.Unmarshal(defBuild, &current)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+bv)
+			}
+			if err := mergo.Merge(&dst, current); err != nil {
+				return nil, errors.Wrap(err, "merging values file "+bv)
+			}
+		}
+	}
+	return dst, nil
+}
+
+func reverse(s []string) []string {
+	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+		s[i], s[j] = s[j], s[i]
+	}
+	return s
+}
+
+func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string, error) {
 	raw, err := ioutil.ReadFile(toTemplate)
 	if err != nil {
 		return "", errors.Wrap(err, "reading file "+toTemplate)
 	}
 
-	if !Exists(valuesFile) {
+	if !fileHelper.Exists(valuesFile) {
 		return "", errors.Wrap(err, "file not existing "+valuesFile)
 	}
 	val, err := ioutil.ReadFile(valuesFile)
@@ -52,20 +86,14 @@ func RenderFiles(toTemplate, valuesFile string, defaultFile string) (string, err
 	}
 
 	var values templatedata
-	d := templatedata{}
-	if len(defaultFile) > 0 {
-		def, err := ioutil.ReadFile(defaultFile)
-		if err != nil {
-			return "", errors.Wrap(err, "reading file "+valuesFile)
-		}
-		if err = yaml.Unmarshal(def, &d); err != nil {
-			return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
-		}
-	}
-
 	if err = yaml.Unmarshal(val, &values); err != nil {
 		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
 	}
 
-	return RenderHelm(string(raw), values, d)
+	dst, err := UnMarshalValues(defaultFile)
+	if err != nil {
+		return "", errors.Wrap(err, "unmarshalling values")
+	}
+
+	return RenderHelm(string(raw), values, dst)
 }

pkg/helpers/helm_test.go

@@ -114,11 +114,46 @@ foo: "bar"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, "")
+			res, err := RenderFiles(toTemplate, values)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bar"))
 		})
 
+		It("Render files merging defaults", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+			d2 := filepath.Join(testDir, "default2.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}{{.Values.bar}}{{.Values.b}}`)
+			writeFile(values, `
+foo: "bar"
+b: "f"
+`)
+			writeFile(d, `
+foo: "baz"
+`)
+
+			writeFile(d2, `
+foo: "do"
+bar: "nei"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d2, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bazneif"))
+
+			res, err = RenderFiles(toTemplate, values, d, d2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("doneif"))
+		})
+
 		It("doesn't interpolate if no one provides the values", func() {
 			testDir, err := ioutil.TempDir(os.TempDir(), "test")
 			Expect(err).ToNot(HaveOccurred())

pkg/helpers/imgworker/auth.go

@@ -0,0 +1,36 @@
+package imgworker
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types"
+
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/session/auth"
+	"google.golang.org/grpc"
+)
+
+func NewDockerAuthProvider(auth *types.AuthConfig) session.Attachable {
+	return &authProvider{
+		config: auth,
+	}
+}
+
+type authProvider struct {
+	config *types.AuthConfig
+}
+
+func (ap *authProvider) Register(server *grpc.Server) {
+	// no-op
+}
+
+func (ap *authProvider) Credentials(ctx context.Context, req *auth.CredentialsRequest) (*auth.CredentialsResponse, error) {
+	res := &auth.CredentialsResponse{}
+	if ap.config.IdentityToken != "" {
+		res.Secret = ap.config.IdentityToken
+	} else {
+		res.Username = ap.config.Username
+		res.Secret = ap.config.Password
+	}
+	return res, nil
+}

pkg/helpers/imgworker/client.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 
 	"github.com/containerd/containerd/namespaces"
+	dockertypes "github.com/docker/docker/api/types"
 	"github.com/genuinetools/img/types"
 	"github.com/moby/buildkit/control"
 	"github.com/moby/buildkit/session"
@@ -29,10 +30,11 @@ type Client struct {
 
 	sess *session.Session
 	ctx  context.Context
+	auth *dockertypes.AuthConfig
 }
 
 // New returns a new client for communicating with the buildkit controller.
-func New(root string) (*Client, error) {
+func New(root string, auth *dockertypes.AuthConfig) (*Client, error) {
 	// Native backend is fine, our images have just one layer. No need to depend on anything
 	backend := types.NativeBackend
 
@@ -45,6 +47,7 @@ func New(root string) (*Client, error) {
 		backend:   types.NativeBackend,
 		root:      root,
 		localDirs: nil,
+		auth:      auth,
 	}
 
 	if err := c.prepare(); err != nil {

pkg/helpers/imgworker/pull.go

@@ -31,6 +31,7 @@ func (c *Client) Pull(image string) (*ListedImage, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	// Parse the image name and tag.
 	named, err := reference.ParseNormalizedNamed(image)
 	if err != nil {
@@ -114,7 +115,6 @@ func (c *Client) Pull(image string) (*ListedImage, error) {
 	if _, err := e.Export(ctx, exporter.Source{Ref: ref}); err != nil {
 		return nil, err
 	}
-
 	// Get the image.
 	img, err := opt.ImageStore.Get(ctx, image)
 	if err != nil {

pkg/helpers/imgworker/session.go

@@ -4,10 +4,8 @@ package imgworker
 
 import (
 	"context"
-	"os"
 
 	"github.com/moby/buildkit/session"
-	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/session/testutil"
 	"github.com/pkg/errors"
@@ -31,7 +29,7 @@ func (c *Client) Session(ctx context.Context) (*session.Session, session.Dialer,
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to create session manager")
 	}
-	sessionName := "img"
+	sessionName := "luet"
 	s, err := session.NewSession(ctx, sessionName, "")
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to create session")
@@ -41,7 +39,7 @@ func (c *Client) Session(ctx context.Context) (*session.Session, session.Dialer,
 		syncedDirs = append(syncedDirs, filesync.SyncedDir{Name: name, Dir: d})
 	}
 	s.Allow(filesync.NewFSSyncProvider(syncedDirs))
-	s.Allow(authprovider.NewDockerAuthProvider(os.Stderr))
+	s.Allow(NewDockerAuthProvider(c.auth))
 	return s, sessionDialer(s, m), err
 }
 

pkg/helpers/imgworker/unpack.go

@@ -5,6 +5,7 @@ package imgworker
 import (
 	"errors"
 	"fmt"
+	"github.com/mudler/luet/pkg/bus"
 	"os"
 
 	"github.com/containerd/containerd/content"
@@ -18,6 +19,12 @@ import (
 // TODO: this requires root permissions to mount/unmount layers, althrought it shouldn't be required.
 // See how backends are unpacking images without asking for root permissions.
 
+// UnpackEventData is the data structure to pass for the bus events
+type UnpackEventData struct {
+	Image string
+	Dest  string
+}
+
 // Unpack exports an image to a rootfs destination directory.
 func (c *Client) Unpack(image, dest string) error {
 
@@ -59,6 +66,8 @@ func (c *Client) Unpack(image, dest string) error {
 		return fmt.Errorf("getting image manifest failed: %v", err)
 	}
 
+	_,_ = bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
+
 	for _, desc := range manifest.Layers {
 		logrus.Debugf("Unpacking layer %s", desc.Digest.String())
 
@@ -71,12 +80,14 @@ func (c *Client) Unpack(image, dest string) error {
 		// Unpack the tarfile to the rootfs path.
 		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
 		if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{
-			NoLchown:        true,
+			NoLchown:        false,
 			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
 		}); err != nil {
 			return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err)
 		}
 	}
 
+	_, _ = bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
+
 	return nil
 }

pkg/helpers/match/match.go

@@ -14,12 +14,21 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package match
 
 import (
+	"reflect"
 	"regexp"
 )
 
+func ReverseAny(s interface{}) {
+	n := reflect.ValueOf(s).Len()
+	swap := reflect.Swapper(s)
+	for i, j := 0, n-1; i < j; i, j = i+1, j-1 {
+		swap(i, j)
+	}
+}
+
 func MapMatchRegex(m *map[string]string, r *regexp.Regexp) bool {
 	ans := false
 

pkg/helpers/references.go

@@ -0,0 +1,30 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       David Cassany <dcassany@suse.com>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"github.com/asaskevich/govalidator"
+	"strings"
+)
+
+func StripRegistryFromImage(image string) string {
+	img := strings.SplitN(image, "/", 2)
+	if len(img) == 2 && govalidator.IsURL(img[0]) {
+		return img[1]
+	}
+	return image
+}

pkg/helpers/references_test.go

@@ -0,0 +1,44 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       David Cassany <dcassany@suse.com>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Helpers", func() {
+	Context("StripRegistryFromImage", func() {
+		It("Strips the domain name", func() {
+			out := StripRegistryFromImage("valid.domain.org/base/image:tag")
+			Expect(out).To(Equal("base/image:tag"))
+		})
+		It("Strips the domain name when port is included", func() {
+			out := StripRegistryFromImage("valid.domain.org:5000/base/image:tag")
+			Expect(out).To(Equal("base/image:tag"))
+		})
+		It("Does not strip the domain name", func() {
+			out := StripRegistryFromImage("not-a-domain/base/image:tag")
+			Expect(out).To(Equal("not-a-domain/base/image:tag"))
+		})
+		It("Does not strip the domain name on invalid domains", func() {
+			out := StripRegistryFromImage("-invaliddomain.org/base/image:tag")
+			Expect(out).To(Equal("-invaliddomain.org/base/image:tag"))
+		})
+	})
+})

pkg/helpers/slice.go

@@ -0,0 +1,10 @@
+package helpers
+
+func Contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}

pkg/installer/client/docker.go

@@ -16,66 +16,55 @@
 package client
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 
+	"github.com/docker/docker/api/types"
 	"github.com/docker/go-units"
 	"github.com/pkg/errors"
 
-	imgworker "github.com/mudler/luet/pkg/installer/client/imgworker"
-
-	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/helpers/docker"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 )
 
+const (
+	errImageDownloadMsg = "failed downloading image %s: %s"
+)
+
 type DockerClient struct {
 	RepoData RepoData
+	auth     *types.AuthConfig
+	verify   bool
 }
 
 func NewDockerClient(r RepoData) *DockerClient {
-	return &DockerClient{RepoData: r}
+	auth := &types.AuthConfig{}
+
+	dat, _ := json.Marshal(r.Authentication)
+	json.Unmarshal(dat, auth)
+
+	return &DockerClient{RepoData: r, auth: auth}
 }
 
-func downloadAndExtractDockerImage(image, dest string) error {
-	temp, err := config.LuetCfg.GetSystem().TempDir("contentstore")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(temp)
-	Debug("Temporary directory", temp)
-	c, err := imgworker.New(temp)
-	if err != nil {
-		return errors.Wrapf(err, "failed creating client")
-	}
-	defer c.Close()
-
-	// FROM Slightly adapted from genuinetools/img https://github.com/genuinetools/img/blob/54d0ca981c1260546d43961a538550eef55c87cf/pull.go
-	Debug("Pulling image", image)
-	listedImage, err := c.Pull(image)
-	if err != nil {
-		return errors.Wrapf(err, "failed listing images")
-
-	}
-	Debug("Pulled:", listedImage.Target.Digest)
-	Debug("Size:", units.BytesSize(float64(listedImage.ContentSize)))
-	Debug("Unpacking", image, "to", dest)
-	os.RemoveAll(dest)
-	return c.Unpack(image, dest)
-}
-
-func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	//var u *url.URL = nil
 	var err error
 	var temp string
-	var resultingArtifact compiler.Artifact
-	artifactName := path.Base(artifact.GetPath())
+
+	Spinner(22)
+	defer SpinnerStop()
+
+	var resultingArtifact *artifact.PackageArtifact
+	artifactName := path.Base(a.Path)
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 	Debug("Cache file", cacheFile)
-	if err := helpers.EnsureDir(cacheFile); err != nil {
+	if err := fileHelper.EnsureDir(cacheFile); err != nil {
 		return nil, errors.Wrapf(err, "could not create cache folder %s for %s", config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), cacheFile)
 	}
 	ok := false
@@ -88,11 +77,11 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
 	// is done in such cases (see repository.go)
 
 	// Check if file is already in cache
-	if helpers.Exists(cacheFile) {
+	if fileHelper.Exists(cacheFile) {
 		Debug("Cache hit for artifact", artifactName)
-		resultingArtifact = artifact
-		resultingArtifact.SetPath(cacheFile)
-		resultingArtifact.SetChecksums(compiler.Checksums{})
+		resultingArtifact = a
+		resultingArtifact.Path = cacheFile
+		resultingArtifact.Checksums = artifact.Checksums{}
 	} else {
 
 		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
@@ -102,23 +91,32 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
 		defer os.RemoveAll(temp)
 
 		for _, uri := range c.RepoData.Urls {
-			Debug("Downloading artifact", artifactName, "from", uri)
 
-			imageName := fmt.Sprintf("%s:%s", uri, artifact.GetCompileSpec().GetPackage().GetFingerPrint())
+			imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID())
+			Info("Downloading image", imageName)
 
-			// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
-			err = downloadAndExtractDockerImage(imageName, temp)
+			contentstore, err := config.LuetCfg.GetSystem().TempDir("contentstore")
 			if err != nil {
-				Debug("Failed download of image", imageName)
+				Warning("Cannot create contentstore", err.Error())
 				continue
 			}
+
+			// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
+			info, err := docker.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
+			if err != nil {
+				Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
+				continue
+			}
+
+			Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
+			Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
 			Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile)
 
-			newart := artifact
+			newart := a
 			// We discard checksum, that are checked while during pull and unpack
-			newart.SetChecksums(compiler.Checksums{})
-			newart.SetPath(cacheFile)                    // First set to cache file
-			newart.SetPath(newart.GetUncompressedName()) // Calculate the real path from cacheFile
+			newart.Checksums = artifact.Checksums{}
+			newart.Path = cacheFile                    // First set to cache file
+			newart.Path = newart.GetUncompressedName() // Calculate the real path from cacheFile
 			err = newart.Compress(temp, 1)
 			if err != nil {
 				Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))
@@ -141,8 +139,7 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
 func (c *DockerClient) DownloadFile(name string) (string, error) {
 	var file *os.File = nil
 	var err error
-	var temp string
-
+	var temp, contentstore string
 	// Files should be in URI/repository:<file>
 	ok := false
 
@@ -152,25 +149,31 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 	}
 
 	for _, uri := range c.RepoData.Urls {
-
 		file, err = config.LuetCfg.GetSystem().TempFile("DockerClient")
 		if err != nil {
 			continue
 		}
 
-		Debug("Downloading file", name, "from", uri)
-
-		imageName := fmt.Sprintf("%s:%s", uri, name)
-		//imageName := fmt.Sprintf("%s/%s:%s", uri, "repository", name)
-		err = downloadAndExtractDockerImage(imageName, temp)
+		contentstore, err = config.LuetCfg.GetSystem().TempDir("contentstore")
 		if err != nil {
-			Debug("Failed download of image", imageName)
+			Warning("Cannot create contentstore", err.Error())
 			continue
 		}
 
-		Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name())
-		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+		imageName := fmt.Sprintf("%s:%s", uri, docker.StripInvalidStringsFromImage(name))
+		Info("Downloading", imageName)
 
+		info, err := docker.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
+		if err != nil {
+			Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
+			continue
+		}
+
+		Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
+		Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
+
+		Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name())
+		err = fileHelper.CopyFile(filepath.Join(temp, name), file.Name())
 		if err != nil {
 			continue
 		}

pkg/installer/client/docker_test.go

@@ -20,8 +20,10 @@ import (
 	"os"
 	"path/filepath"
 
-	compiler "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	pkg "github.com/mudler/luet/pkg/package"
 
 	. "github.com/mudler/luet/pkg/installer/client"
@@ -30,7 +32,7 @@ import (
 )
 
 // This test expect that the repository defined in UNIT_TEST_DOCKER_IMAGE is in zstd format.
-// the repository is built by the 01_simple_docker.sh integration test file.
+// the repository is built by the 01_simple_docker.sh integration test fileHelper.
 // This test also require root. At the moment, unpacking docker images with 'img' requires root permission to
 // mount/unmount layers.
 var _ = Describe("Docker client", func() {
@@ -49,14 +51,14 @@ var _ = Describe("Docker client", func() {
 		It("Downloads single files", func() {
 			f, err := c.DownloadFile("repository.yaml")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(f)).To(ContainSubstring("Test Repo"))
+			Expect(fileHelper.Read(f)).To(ContainSubstring("Test Repo"))
 			os.RemoveAll(f)
 		})
 
 		It("Downloads artifacts", func() {
-			f, err := c.DownloadArtifact(&compiler.PackageArtifact{
+			f, err := c.DownloadArtifact(&artifact.PackageArtifact{
 				Path: "test.tar",
-				CompileSpec: &compiler.LuetCompilationSpec{
+				CompileSpec: &compilerspec.LuetCompilationSpec{
 					Package: &pkg.DefaultPackage{
 						Name:     "c",
 						Category: "test",
@@ -69,9 +71,9 @@ var _ = Describe("Docker client", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 			Expect(f.Unpack(tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Read(filepath.Join(tmpdir, "c"))).To(Equal("c\n"))
-			Expect(helpers.Read(filepath.Join(tmpdir, "cd"))).To(Equal("c\n"))
-			os.RemoveAll(f.GetPath())
+			Expect(fileHelper.Read(filepath.Join(tmpdir, "c"))).To(Equal("c\n"))
+			Expect(fileHelper.Read(filepath.Join(tmpdir, "cd"))).To(Equal("c\n"))
+			os.RemoveAll(f.Path)
 		})
 	})
 })

pkg/installer/client/http.go

@@ -18,19 +18,20 @@ package client
 import (
 	"fmt"
 	"math"
+	"net/http"
 	"net/url"
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"time"
 
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 
-	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
-
 	"github.com/cavaliercoder/grab"
+	"github.com/mudler/luet/pkg/config"
 
 	"github.com/schollz/progressbar/v3"
 )
@@ -43,6 +44,27 @@ func NewHttpClient(r RepoData) *HttpClient {
 	return &HttpClient{RepoData: r}
 }
 
+func NewGrabClient() *grab.Client {
+	httpTimeout := 30
+	timeout := os.Getenv("HTTP_TIMEOUT")
+	if timeout != "" {
+		timeoutI, err := strconv.Atoi(timeout)
+		if err == nil {
+			httpTimeout = timeoutI
+		}
+	}
+
+	return &grab.Client{
+		UserAgent: "grab",
+		HTTPClient: &http.Client{
+			Timeout: time.Duration(httpTimeout) * time.Second,
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+			},
+		},
+	}
+}
+
 func (c *HttpClient) PrepareReq(dst, url string) (*grab.Request, error) {
 
 	req, err := grab.NewRequest(dst, url)
@@ -66,18 +88,18 @@ func Round(input float64) float64 {
 	return math.Floor(input + 0.5)
 }
 
-func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	var u *url.URL = nil
 	var err error
 	var req *grab.Request
 	var temp string
 
-	artifactName := path.Base(artifact.GetPath())
+	artifactName := path.Base(a.Path)
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 	ok := false
 
 	// Check if file is already in cache
-	if helpers.Exists(cacheFile) {
+	if fileHelper.Exists(cacheFile) {
 		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 
@@ -87,7 +109,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 		}
 		defer os.RemoveAll(temp)
 
-		client := grab.NewClient()
+		client := NewGrabClient()
 
 		for _, uri := range c.RepoData.Urls {
 			Debug("Downloading artifact", artifactName, "from", uri)
@@ -156,7 +178,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 				fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
 
 			Debug("\nCopying file ", filepath.Join(temp, artifactName), "to", cacheFile)
-			err = helpers.CopyFile(filepath.Join(temp, artifactName), cacheFile)
+			err = fileHelper.CopyFile(filepath.Join(temp, artifactName), cacheFile)
 
 			bar.Finish()
 			ok = true
@@ -168,8 +190,8 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 		}
 	}
 
-	newart := artifact
-	newart.SetPath(cacheFile)
+	newart := a
+	newart.Path = cacheFile
 	return newart, nil
 }
 
@@ -187,7 +209,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 		return "", err
 	}
 
-	client := grab.NewClient()
+	client := NewGrabClient()
 
 	for _, uri := range c.RepoData.Urls {
 
@@ -218,7 +240,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 			fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
 			fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
 
-		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+		err = fileHelper.CopyFile(filepath.Join(temp, name), file.Name())
 		if err != nil {
 			continue
 		}

pkg/installer/client/http_test.go

@@ -22,9 +22,8 @@ import (
 	"os"
 	"path/filepath"
 
-	compiler "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
-
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer/client"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -47,7 +46,7 @@ var _ = Describe("Http client", func() {
 			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
 			path, err := c.DownloadFile("test.txt")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path)).To(Equal("test"))
+			Expect(fileHelper.Read(path)).To(Equal("test"))
 			os.RemoveAll(path)
 		})
 
@@ -63,10 +62,10 @@ var _ = Describe("Http client", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
-			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
+			path, err := c.DownloadArtifact(&artifact.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path.GetPath())).To(Equal("test"))
-			os.RemoveAll(path.GetPath())
+			Expect(fileHelper.Read(path.Path)).To(Equal("test"))
+			os.RemoveAll(path.Path)
 		})
 
 	})

pkg/installer/client/interface.go

@@ -18,4 +18,5 @@ package client
 type RepoData struct {
 	Urls           []string
 	Authentication map[string]string
+	Verify         bool
 }

pkg/installer/client/local.go

@@ -20,11 +20,10 @@ import (
 	"path"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
-
-	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/helpers"
 )
 
 type LocalClient struct {
@@ -35,11 +34,11 @@ func NewLocalClient(r RepoData) *LocalClient {
 	return &LocalClient{RepoData: r}
 }
 
-func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+func (c *LocalClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	var err error
 
 	rootfs := ""
-	artifactName := path.Base(artifact.GetPath())
+	artifactName := path.Base(a.Path)
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 
 	if !config.LuetCfg.ConfigFromHost {
@@ -50,7 +49,7 @@ func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Art
 	}
 
 	// Check if file is already in cache
-	if helpers.Exists(cacheFile) {
+	if fileHelper.Exists(cacheFile) {
 		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 		ok := false
@@ -61,7 +60,7 @@ func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Art
 			Info("Downloading artifact", artifactName, "from", uri)
 
 			//defer os.Remove(file.Name())
-			err = helpers.CopyFile(filepath.Join(uri, artifactName), cacheFile)
+			err = fileHelper.CopyFile(filepath.Join(uri, artifactName), cacheFile)
 			if err != nil {
 				continue
 			}
@@ -74,8 +73,8 @@ func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Art
 		}
 	}
 
-	newart := artifact
-	newart.SetPath(cacheFile)
+	newart := a
+	newart.Path = cacheFile
 	return newart, nil
 }
 
@@ -104,7 +103,7 @@ func (c *LocalClient) DownloadFile(name string) (string, error) {
 		}
 		//defer os.Remove(file.Name())
 
-		err = helpers.CopyFile(filepath.Join(uri, name), file.Name())
+		err = fileHelper.CopyFile(filepath.Join(uri, name), file.Name())
 		if err != nil {
 			continue
 		}

pkg/installer/client/local_test.go

@@ -20,9 +20,8 @@ import (
 	"os"
 	"path/filepath"
 
-	compiler "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
-
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer/client"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -42,7 +41,7 @@ var _ = Describe("Local client", func() {
 			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
 			path, err := c.DownloadFile("test.txt")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path)).To(Equal("test"))
+			Expect(fileHelper.Read(path)).To(Equal("test"))
 			os.RemoveAll(path)
 		})
 
@@ -56,10 +55,10 @@ var _ = Describe("Local client", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
-			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
+			path, err := c.DownloadArtifact(&artifact.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path.GetPath())).To(Equal("test"))
-			os.RemoveAll(path.GetPath())
+			Expect(fileHelper.Read(path.Path)).To(Equal("test"))
+			os.RemoveAll(path.Path)
 		})
 
 	})

pkg/installer/finalizer.go

@@ -16,6 +16,7 @@
 package installer
 
 import (
+	"os"
 	"os/exec"
 
 	"github.com/ghodss/yaml"
@@ -48,7 +49,7 @@ func (f *LuetFinalizer) RunInstall(s *System) error {
 	for _, c := range f.Install {
 		toRun := append(args, c)
 		Info(":shell: Executing finalizer on ", s.Target, cmd, toRun)
-		if s.Target == "/" {
+		if s.Target == string(os.PathSeparator) {
 			cmd := exec.Command(cmd, toRun...)
 			stdoutStderr, err := cmd.CombinedOutput()
 			if err != nil {

pkg/installer/installer.go

@@ -24,15 +24,17 @@ import (
 	"strings"
 	"sync"
 
-	. "github.com/logrusorgru/aurora"
 	"github.com/mudler/luet/pkg/bus"
-	compiler "github.com/mudler/luet/pkg/compiler"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	"github.com/mudler/luet/pkg/helpers/match"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 
+	. "github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 )
 
@@ -47,6 +49,7 @@ type LuetInstallerOptions struct {
 	CheckConflicts                                                 bool
 	SolverUpgrade, RemoveUnavailableOnUpgrade, UpgradeNewRevisions bool
 	Ask                                                            bool
+	DownloadOnly                                                   bool
 }
 
 type LuetInstaller struct {
@@ -57,11 +60,11 @@ type LuetInstaller struct {
 
 type ArtifactMatch struct {
 	Package    pkg.Package
-	Artifact   compiler.Artifact
-	Repository Repository
+	Artifact   *artifact.PackageArtifact
+	Repository *LuetSystemRepository
 }
 
-func NewLuetInstaller(opts LuetInstallerOptions) Installer {
+func NewLuetInstaller(opts LuetInstallerOptions) *LuetInstaller {
 	return &LuetInstaller{Options: opts}
 }
 
@@ -105,11 +108,11 @@ func (l *LuetInstaller) computeUpgrade(syncedRepos Repositories, s *System) (pkg
 				continue
 			}
 			for _, artefact := range matches[0].Repo.GetIndex() {
-				if artefact.GetCompileSpec().GetPackage() == nil {
+				if artefact.CompileSpec.GetPackage() == nil {
 					return uninstall, toInstall, errors.New("Package in compilespec empty")
 
 				}
-				if artefact.GetCompileSpec().GetPackage().Matches(p) && artefact.GetCompileSpec().GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
+				if artefact.CompileSpec.GetPackage().Matches(p) && artefact.CompileSpec.GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
 					toInstall = append(toInstall, matches[0].Package).Unique()
 					uninstall = append(uninstall, p).Unique()
 				}
@@ -126,6 +129,8 @@ func packsToList(p pkg.Packages) string {
 	for _, pp := range p {
 		packs = append(packs, pp.HumanReadableString())
 	}
+
+	sort.Strings(packs)
 	return strings.Join(packs, " ")
 }
 
@@ -135,6 +140,7 @@ func matchesToList(artefacts map[string]ArtifactMatch) string {
 	for fingerprint, match := range artefacts {
 		packs = append(packs, fmt.Sprintf("%s (%s)", fingerprint, match.Repository.GetName()))
 	}
+	sort.Strings(packs)
 	return strings.Join(packs, " ")
 }
 
@@ -151,39 +157,7 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 		Info(":memo: note: will consider new build revisions while upgrading")
 	}
 
-	Spinner(32)
-	uninstall, toInstall, err := l.computeUpgrade(syncedRepos, s)
-	if err != nil {
-		return errors.Wrap(err, "failed computing upgrade")
-	}
-	SpinnerStop()
-
-	if len(uninstall) > 0 {
-		Info(":recycle: Packages that are going to be removed from the system:\n ", Yellow(packsToList(uninstall)).BgBlack().String())
-	}
-
-	if len(toInstall) > 0 {
-		Info(":zap:Packages that are going to be installed in the system:\n ", Green(packsToList(toInstall)).BgBlack().String())
-	}
-
-	if len(toInstall) == 0 && len(uninstall) == 0 {
-		Info("Nothing to do")
-		return nil
-	}
-
-	if l.Options.Ask {
-		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
-		if Ask() {
-			l.Options.Ask = false // Don't prompt anymore
-			return l.swap(syncedRepos, uninstall, toInstall, s, true)
-		} else {
-			return errors.New("Aborted by user")
-		}
-	}
-
-	Spinner(32)
-	defer SpinnerStop()
-	return l.swap(syncedRepos, uninstall, toInstall, s, true)
+	return l.checkAndUpgrade(syncedRepos, s)
 }
 
 func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
@@ -224,11 +198,19 @@ func (l *LuetInstaller) Swap(toRemove pkg.Packages, toInstall pkg.Packages, s *S
 			toRemoveFinal = append(toRemoveFinal, pp)
 		}
 	}
+	o := Option{
+		FullUninstall:      false,
+		Force:              true,
+		CheckConflicts:     false,
+		FullCleanUninstall: false,
+		NoDeps:             l.Options.NoDeps,
+		OnlyDeps:           false,
+	}
 
-	return l.swap(syncedRepos, toRemoveFinal, toInstall, s, false)
+	return l.swap(o, syncedRepos, toRemoveFinal, toInstall, s)
 }
 
-func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
+func (l *LuetInstaller) computeSwap(o Option, syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
 
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
@@ -243,8 +225,8 @@ func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packa
 
 	systemAfterChanges := &System{Database: installedtmp}
 
-	packs, err := l.computeUninstall(systemAfterChanges, toRemove...)
-	if err != nil && !l.Options.Force {
+	packs, err := l.computeUninstall(o, systemAfterChanges, toRemove...)
+	if err != nil && !o.Force {
 		Error("Failed computing uninstall for ", packsToList(toRemove))
 		return nil, nil, nil, nil, errors.Wrap(err, "computing uninstall "+packsToList(toRemove))
 	}
@@ -255,30 +237,16 @@ func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packa
 		}
 	}
 
-	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, toInstall, systemAfterChanges)
+	match, packages, assertions, allRepos, err := l.computeInstall(o, syncedRepos, toInstall, systemAfterChanges)
 	for _, p := range toInstall {
 		assertions = append(assertions, solver.PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
 	}
 	return match, packages, assertions, allRepos, err
 }
 
-func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System, forceNodeps bool) error {
-	forced := l.Options.Force
-	nodeps := l.Options.NoDeps
+func (l *LuetInstaller) swap(o Option, syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) error {
 
-	// We don't want any conflict with the installed to raise during the upgrade.
-	// In this way we both force uninstalls and we avoid to check with conflicts
-	// against the current system state which is pending to deletion
-	// E.g. you can't check for conflicts for an upgrade of a new version of A
-	// if the old A results installed in the system. This is due to the fact that
-	// now the solver enforces the constraints and explictly denies two packages
-	// of the same version installed.
-	l.Options.Force = true
-	if forceNodeps {
-		l.Options.NoDeps = true
-	}
-
-	match, packages, assertions, allRepos, err := l.computeSwap(syncedRepos, toRemove, toInstall, s)
+	match, packages, assertions, allRepos, err := l.computeSwap(o, syncedRepos, toRemove, toInstall, s)
 	if err != nil {
 		return errors.Wrap(err, "failed computing package replacement")
 	}
@@ -305,15 +273,235 @@ func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, to
 		return errors.Wrap(err, "Pre-downloading packages")
 	}
 
-	err = l.Uninstall(s, toRemove...)
-	if err != nil && !l.Options.Force {
-		Error("Failed uninstall for ", packsToList(toRemove))
-		return errors.Wrap(err, "uninstalling "+packsToList(toRemove))
+	if err := l.checkFileconflicts(match, false, s); err != nil {
+		if !l.Options.Force {
+			return errors.Wrap(err, "file conflict found")
+		} else {
+			Warning("file conflict found", err.Error())
+		}
 	}
 
-	l.Options.Force = forced
-	l.Options.NoDeps = nodeps
-	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+	if l.Options.DownloadOnly {
+		return nil
+	}
+
+	ops := l.getOpsWithOptions(toRemove, match, Option{
+		Force:         o.Force,
+		NoDeps:        false,
+		OnlyDeps:      o.OnlyDeps,
+		RunFinalizers: false,
+	}, o, syncedRepos, packages, assertions, allRepos)
+
+	err = l.runOps(ops, s)
+	if err != nil {
+		return errors.Wrap(err, "failed running installer options")
+	}
+
+	toFinalize, err := l.getFinalizers(allRepos, assertions, match, o.NoDeps)
+	if err != nil {
+		return errors.Wrap(err, "failed getting package to finalize")
+	}
+
+	return s.ExecuteFinalizers(toFinalize)
+}
+
+type Option struct {
+	Force              bool
+	NoDeps             bool
+	CheckConflicts     bool
+	FullUninstall      bool
+	FullCleanUninstall bool
+	OnlyDeps           bool
+	RunFinalizers      bool
+}
+
+type operation struct {
+	Option  Option
+	Package pkg.Package
+}
+
+type installOperation struct {
+	operation
+	Reposiories Repositories
+	Packages    pkg.Packages
+	Assertions  solver.PackagesAssertions
+	Database    pkg.PackageDatabase
+	Matches     map[string]ArtifactMatch
+}
+
+// installerOp is the operation that is sent to the
+// upgradeWorker's channel (todo)
+type installerOp struct {
+	Uninstall operation
+	Install   installOperation
+}
+
+func (l *LuetInstaller) runOps(ops []installerOp, s *System) error {
+	all := make(chan installerOp)
+
+	wg := new(sync.WaitGroup)
+
+	// Do the real install
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.installerOpWorker(i, wg, all, s)
+	}
+
+	for _, c := range ops {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
+
+	return nil
+}
+
+// TODO: use installerOpWorker in place of all the other workers.
+// This one is general enough to read a list of operations and execute them.
+func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, c <-chan installerOp, s *System) error {
+	defer wg.Done()
+
+	for p := range c {
+		if p.Uninstall.Package != nil {
+			Debug("Replacing package inplace")
+			toUninstall, uninstall, err := l.generateUninstallFn(p.Uninstall.Option, s, p.Uninstall.Package)
+			if err != nil {
+				Error("Failed to generate Uninstall function for" + err.Error())
+				continue
+				//return errors.Wrap(err, "while computing uninstall")
+			}
+
+			err = uninstall()
+			if err != nil {
+				Error("Failed uninstall for ", packsToList(toUninstall))
+				continue
+				//return errors.Wrap(err, "uninstalling "+packsToList(toUninstall))
+			}
+		}
+		if p.Install.Package != nil {
+			artMatch := p.Install.Matches[p.Install.Package.GetFingerPrint()]
+			ass := p.Install.Assertions.Search(p.Install.Package.GetFingerPrint())
+			packageToInstall, _ := p.Install.Packages.Find(p.Install.Package.GetPackageName())
+
+			err := l.install(
+				p.Install.Option,
+				p.Install.Reposiories,
+				map[string]ArtifactMatch{p.Install.Package.GetFingerPrint(): artMatch},
+				pkg.Packages{packageToInstall},
+				solver.PackagesAssertions{*ass},
+				p.Install.Database,
+				s,
+			)
+			if err != nil {
+				Error(err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// checks wheter we can uninstall and install in place and compose installer worker ops
+func (l *LuetInstaller) getOpsWithOptions(
+	toUninstall pkg.Packages, installMatch map[string]ArtifactMatch, installOpt, uninstallOpt Option,
+	syncedRepos Repositories, toInstall pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase) []installerOp {
+	resOps := []installerOp{}
+	for _, match := range installMatch {
+		if pack, err := toUninstall.Find(match.Package.GetPackageName()); err == nil {
+			resOps = append(resOps, installerOp{
+				Uninstall: operation{Package: pack, Option: uninstallOpt},
+				Install: installOperation{
+					operation: operation{
+						Package: match.Package,
+						Option:  installOpt,
+					},
+					Matches:     installMatch,
+					Packages:    toInstall,
+					Reposiories: syncedRepos,
+					Assertions:  solution,
+					Database:    allRepos,
+				},
+			})
+		} else {
+			resOps = append(resOps, installerOp{
+				Install: installOperation{
+					operation:   operation{Package: match.Package, Option: installOpt},
+					Matches:     installMatch,
+					Reposiories: syncedRepos,
+					Packages:    toInstall,
+					Assertions:  solution,
+					Database:    allRepos,
+				},
+			})
+		}
+	}
+
+	for _, p := range toUninstall {
+		found := false
+
+		for _, match := range installMatch {
+			if match.Package.GetPackageName() == p.GetPackageName() {
+				found = true
+			}
+
+		}
+		if !found {
+			resOps = append(resOps, installerOp{
+				Uninstall: operation{Package: p, Option: uninstallOpt},
+			})
+		}
+	}
+	return resOps
+}
+
+func (l *LuetInstaller) checkAndUpgrade(r Repositories, s *System) error {
+	Spinner(32)
+	uninstall, toInstall, err := l.computeUpgrade(r, s)
+	if err != nil {
+		return errors.Wrap(err, "failed computing upgrade")
+	}
+	SpinnerStop()
+
+	if len(uninstall) > 0 {
+		Info(":recycle: Packages that are going to be removed from the system:\n ", Yellow(packsToList(uninstall)).BgBlack().String())
+	}
+
+	if len(toInstall) > 0 {
+		Info(":zap:Packages that are going to be installed in the system:\n ", Green(packsToList(toInstall)).BgBlack().String())
+	}
+
+	if len(toInstall) == 0 && len(uninstall) == 0 {
+		Info("Nothing to do")
+		return nil
+	}
+
+	// We don't want any conflict with the installed to raise during the upgrade.
+	// In this way we both force uninstalls and we avoid to check with conflicts
+	// against the current system state which is pending to deletion
+	// E.g. you can't check for conflicts for an upgrade of a new version of A
+	// if the old A results installed in the system. This is due to the fact that
+	// now the solver enforces the constraints and explictly denies two packages
+	// of the same version installed.
+	o := Option{
+		FullUninstall:      false,
+		Force:              true,
+		CheckConflicts:     false,
+		FullCleanUninstall: false,
+		NoDeps:             true,
+		OnlyDeps:           false,
+	}
+
+	if l.Options.Ask {
+		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
+		if Ask() {
+			l.Options.Ask = false // Don't prompt anymore
+			return l.swap(o, r, uninstall, toInstall, s)
+		} else {
+			return errors.New("Aborted by user")
+		}
+	}
+
+	return l.swap(o, r, uninstall, toInstall, s)
 }
 
 func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
@@ -322,7 +510,20 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 		return err
 	}
 
-	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, cp, s)
+	if len(s.Database.World()) > 0 {
+		Info(":thinking: Checking for available upgrades")
+		if err := l.checkAndUpgrade(syncedRepos, s); err != nil {
+			return errors.Wrap(err, "while checking upgrades before install")
+		}
+	}
+
+	o := Option{
+		NoDeps:        l.Options.NoDeps,
+		Force:         l.Options.Force,
+		OnlyDeps:      l.Options.OnlyDeps,
+		RunFinalizers: true,
+	}
+	match, packages, assertions, allRepos, err := l.computeInstall(o, syncedRepos, cp, s)
 	if err != nil {
 		return err
 	}
@@ -359,12 +560,12 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
 		if Ask() {
 			l.Options.Ask = false // Don't prompt anymore
-			return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+			return l.install(o, syncedRepos, match, packages, assertions, allRepos, s)
 		} else {
 			return errors.New("Aborted by user")
 		}
 	}
-	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+	return l.install(o, syncedRepos, match, packages, assertions, allRepos, s)
 }
 
 func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string]ArtifactMatch) error {
@@ -402,12 +603,12 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 	for _, repo := range syncedRepos {
 		for _, artefact := range repo.GetIndex() {
 			Debug("Checking if",
-				artefact.GetCompileSpec().GetPackage().HumanReadableString(),
+				artefact.CompileSpec.GetPackage().HumanReadableString(),
 				"from", repo.GetName(), "is installed")
 		FILES:
-			for _, f := range artefact.GetFiles() {
-				if helpers.Exists(filepath.Join(s.Target, f)) {
-					p, err := repo.GetTree().GetDatabase().FindPackage(artefact.GetCompileSpec().GetPackage())
+			for _, f := range artefact.Files {
+				if fileHelper.Exists(filepath.Join(s.Target, f)) {
+					p, err := repo.GetTree().GetDatabase().FindPackage(artefact.CompileSpec.GetPackage())
 					if err != nil {
 						return err
 					}
@@ -431,7 +632,7 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Failed creating package")
 		}
-		s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.GetFiles()})
+		s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.Files})
 		Info(":zap:Reclaimed package:", pack.HumanReadableString())
 	}
 	Info("Done!")
@@ -439,7 +640,7 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 	return nil
 }
 
-func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
+func (l *LuetInstaller) computeInstall(o Option, syncedRepos Repositories, cp pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
 	var p pkg.Packages
 	toInstall := map[string]ArtifactMatch{}
 	allRepos := pkg.NewInMemoryDatabase(false)
@@ -472,11 +673,11 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 	var packagesToInstall pkg.Packages
 	var err error
 
-	if !l.Options.NoDeps {
+	if !o.NoDeps {
 		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		solution, err = solv.Install(p)
 		/// TODO: PackageAssertions needs to be a map[fingerprint]pack so lookup is in O(1)
-		if err != nil && !l.Options.Force {
+		if err != nil && !o.Force {
 			return toInstall, p, solution, allRepos, errors.Wrap(err, "Failed solving solution for package")
 		}
 		// Gathers things to install
@@ -489,7 +690,7 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 				packagesToInstall = append(packagesToInstall, assertion.Package)
 			}
 		}
-	} else if !l.Options.OnlyDeps {
+	} else if !o.OnlyDeps {
 		for _, currentPack := range p {
 			if _, err := s.Database.FindPackage(currentPack); err == nil {
 				// skip matching if it is installed already
@@ -508,11 +709,11 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 		}
 	A:
 		for _, artefact := range matches[0].Repo.GetIndex() {
-			if artefact.GetCompileSpec().GetPackage() == nil {
+			if artefact.CompileSpec.GetPackage() == nil {
 				return toInstall, p, solution, allRepos, errors.New("Package in compilespec empty")
 			}
-			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
-				currentPack.SetBuildTimestamp(artefact.GetCompileSpec().GetPackage().GetBuildTimestamp())
+			if matches[0].Package.Matches(artefact.CompileSpec.GetPackage()) {
+				currentPack.SetBuildTimestamp(artefact.CompileSpec.GetPackage().GetBuildTimestamp())
 				// Filter out already installed
 				if _, err := s.Database.FindPackage(currentPack); err != nil {
 					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
@@ -524,12 +725,108 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 	return toInstall, p, solution, allRepos, nil
 }
 
-func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
-	// Install packages into rootfs in parallel.
+func (l *LuetInstaller) getFinalizers(allRepos pkg.PackageDatabase, solution solver.PackagesAssertions, toInstall map[string]ArtifactMatch, nodeps bool) ([]pkg.Package, error) {
+	var toFinalize []pkg.Package
+	if !nodeps {
+		// TODO: Lower those errors as warning
+		for _, w := range toInstall {
+			// Finalizers needs to run in order and in sequence.
+			ordered, err := solution.Order(allRepos, w.Package.GetFingerPrint())
+			if err != nil {
+				return toFinalize, errors.Wrap(err, "While order a solution for "+w.Package.HumanReadableString())
+			}
+		ORDER:
+			for _, ass := range ordered {
+				if ass.Value {
+					installed, ok := toInstall[ass.Package.GetFingerPrint()]
+					if !ok {
+						// It was a dep already installed in the system, so we can skip it safely
+						continue ORDER
+					}
+					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
+					if err != nil {
+						return toFinalize, errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
+					}
+
+					toFinalize = append(toFinalize, treePackage)
+				}
+			}
+
+		}
+	} else {
+		for _, c := range toInstall {
+			treePackage, err := c.Repository.GetTree().GetDatabase().FindPackage(c.Package)
+			if err != nil {
+				return toFinalize, errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
+			}
+			toFinalize = append(toFinalize, treePackage)
+		}
+	}
+	return toFinalize, nil
+}
+
+func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, checkSystem bool, s *System) error {
+	Info("Checking for file conflicts..")
+	defer s.Clean() // Release memory
+
+	filesToInstall := []string{}
+	for _, m := range toInstall {
+		a, err := l.downloadPackage(m)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed downloading package")
+		}
+		files, err := a.FileList()
+		if err != nil && !l.Options.Force {
+			return errors.Wrapf(err, "Could not get filelist for %s", a.CompileSpec.Package.HumanReadableString())
+		}
+
+		for _, f := range files {
+			if helpers.Contains(filesToInstall, f) {
+				return fmt.Errorf(
+					"file conflict between packages to be installed",
+				)
+			}
+			if checkSystem {
+				exists, p, err := s.ExistsPackageFile(f)
+				if err != nil {
+					return errors.Wrap(err, "failed checking into system db")
+				}
+				if exists {
+					return fmt.Errorf(
+						"file conflict between '%s' and '%s' ( file: %s )",
+						p.HumanReadableString(),
+						m.Package.HumanReadableString(),
+						f,
+					)
+				}
+			}
+		}
+		filesToInstall = append(filesToInstall, files...)
+	}
+
+	return nil
+}
+
+func (l *LuetInstaller) install(o Option, syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
+
+	// Download packages in parallel first
 	if err := l.download(syncedRepos, toInstall); err != nil {
 		return errors.Wrap(err, "Downloading packages")
 	}
 
+	// Check file conflicts
+	if err := l.checkFileconflicts(toInstall, true, s); err != nil {
+		if !l.Options.Force {
+			return errors.Wrap(err, "file conflict found")
+		} else {
+			Warning("file conflict found", err.Error())
+		}
+	}
+
+	if l.Options.DownloadOnly {
+		return nil
+	}
+
 	all := make(chan ArtifactMatch)
 
 	wg := new(sync.WaitGroup)
@@ -549,52 +846,25 @@ func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]A
 	for _, c := range toInstall {
 		// Annotate to the system that the package was installed
 		_, err := s.Database.CreatePackage(c.Package)
-		if err != nil && !l.Options.Force {
+		if err != nil && !o.Force {
 			return errors.Wrap(err, "Failed creating package")
 		}
 		bus.Manager.Publish(bus.EventPackageInstall, c)
 	}
-	var toFinalize []pkg.Package
-	if !l.Options.NoDeps {
-		// TODO: Lower those errors as warning
-		for _, w := range p {
-			// Finalizers needs to run in order and in sequence.
-			ordered, err := solution.Order(allRepos, w.GetFingerPrint())
-			if err != nil {
-				return errors.Wrap(err, "While order a solution for "+w.HumanReadableString())
-			}
-		ORDER:
-			for _, ass := range ordered {
-				if ass.Value {
-					installed, ok := toInstall[ass.Package.GetFingerPrint()]
-					if !ok {
-						// It was a dep already installed in the system, so we can skip it safely
-						continue ORDER
-					}
-					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
-					if err != nil {
-						return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
-					}
 
-					toFinalize = append(toFinalize, treePackage)
-				}
-			}
+	if !o.RunFinalizers {
+		return nil
+	}
 
-		}
-	} else {
-		for _, c := range toInstall {
-			treePackage, err := c.Repository.GetTree().GetDatabase().FindPackage(c.Package)
-			if err != nil {
-				return errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
-			}
-			toFinalize = append(toFinalize, treePackage)
-		}
+	toFinalize, err := l.getFinalizers(allRepos, solution, toInstall, o.NoDeps)
+	if err != nil {
+		return errors.Wrap(err, "failed getting package to finalize")
 	}
 
 	return s.ExecuteFinalizers(toFinalize)
 }
 
-func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) {
+func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (*artifact.PackageArtifact, error) {
 
 	artifact, err := a.Repository.Client().DownloadArtifact(a.Artifact)
 	if err != nil {
@@ -608,26 +878,26 @@ func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, err
 	return artifact, nil
 }
 
-func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
+func (l *LuetInstaller) installPackage(m ArtifactMatch, s *System) error {
 
-	artifact, err := l.downloadPackage(a)
+	a, err := l.downloadPackage(m)
 	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Failed downloading package")
 	}
 
-	files, err := artifact.FileList()
+	files, err := a.FileList()
 	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Could not open package archive")
 	}
 
-	err = artifact.Unpack(s.Target, true)
+	err = a.Unpack(s.Target, true)
 	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Error met while unpacking rootfs")
 	}
 
 	// First create client and download
 	// Then unpack to system
-	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.Package.GetFingerPrint(), Files: files})
+	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: m.Package.GetFingerPrint(), Files: files})
 }
 
 func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch) error {
@@ -668,6 +938,51 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 	return nil
 }
 
+func checkAndPrunePath(path string) {
+	// check if now the target path is empty
+	targetPath := filepath.Dir(path)
+
+	fi, err := os.Lstat(targetPath)
+	if err != nil {
+		//	Warning("Dir not found (it was before?) ", err.Error())
+		return
+	}
+
+	switch mode := fi.Mode(); {
+	case mode.IsDir():
+		files, err := ioutil.ReadDir(targetPath)
+		if err != nil {
+			Warning("Failed reading folder", targetPath, err.Error())
+		}
+		if len(files) != 0 {
+			Debug("Preserving not-empty folder", targetPath)
+			return
+		}
+	}
+	if err = os.Remove(targetPath); err != nil {
+		Warning("Failed removing file (maybe not present in the system target anymore ?)", targetPath, err.Error())
+	}
+}
+
+// We will try to cleanup every path from the file, if the folders left behind are empty
+func pruneEmptyFilePath(path string) {
+	checkAndPrunePath(path)
+
+	// A path is for e.g. /usr/bin/bar
+	// we want to create an array as "/usr", "/usr/bin", "/usr/bin/bar"
+	paths := strings.Split(path, string(os.PathSeparator))
+	currentPath := filepath.Join(string(os.PathSeparator), paths[0])
+	allPaths := []string{currentPath}
+	for _, p := range paths[1:] {
+		currentPath = filepath.Join(currentPath, p)
+		allPaths = append(allPaths, currentPath)
+	}
+	match.ReverseAny(allPaths)
+	for _, p := range allPaths {
+		checkAndPrunePath(p)
+	}
+}
+
 func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 	var cp *config.ConfigProtect
 	annotationDir := ""
@@ -690,7 +1005,7 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		cp.Map(files)
 	}
 
-	toRemove, notPresent := helpers.OrderFiles(s.Target, files)
+	toRemove, notPresent := fileHelper.OrderFiles(s.Target, files)
 
 	// Remove from target
 	for _, f := range toRemove {
@@ -729,6 +1044,8 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		if err = os.Remove(target); err != nil {
 			Warning("Failed removing file (maybe not present in the system target anymore ?)", target, err.Error())
 		}
+
+		pruneEmptyFilePath(target)
 	}
 
 	for _, f := range notPresent {
@@ -742,6 +1059,8 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		if err = os.Remove(target); err != nil {
 			Debug("Failed removing file (not present in the system target)", target, err.Error())
 		}
+
+		pruneEmptyFilePath(target)
 	}
 
 	err = s.Database.RemovePackageFiles(p)
@@ -759,17 +1078,17 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 	return nil
 }
 
-func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.Packages, error) {
+func (l *LuetInstaller) computeUninstall(o Option, s *System, packs ...pkg.Package) (pkg.Packages, error) {
 
 	var toUninstall pkg.Packages
 	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) TODO - mark the uninstallation in db
 	// Get installed definition
-	checkConflicts := l.Options.CheckConflicts
-	full := l.Options.FullUninstall
-	if l.Options.Force == true { // IF forced, we want to remove the package and all its requires
-		checkConflicts = false
-		full = false
-	}
+	checkConflicts := o.CheckConflicts
+	full := o.FullUninstall
+	// if o.Force == true { // IF forced, we want to remove the package and all its requires
+	// 	checkConflicts = false
+	// 	full = false
+	// }
 
 	// Create a temporary DB with the installed packages
 	// so the solver is much faster finding the deptree
@@ -779,11 +1098,11 @@ func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.P
 		return toUninstall, errors.Wrap(err, "Failed create temporary in-memory db")
 	}
 
-	if !l.Options.NoDeps {
+	if !o.NoDeps {
 		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		var solution pkg.Packages
 		var err error
-		if l.Options.FullCleanUninstall {
+		if o.FullCleanUninstall {
 			solution, err = solv.UninstallUniverse(packs)
 			if err != nil {
 				return toUninstall, errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
@@ -804,32 +1123,47 @@ func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.P
 
 	return toUninstall, nil
 }
-func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
 
+func (l *LuetInstaller) generateUninstallFn(o Option, s *System, packs ...pkg.Package) (pkg.Packages, func() error, error) {
 	for _, p := range packs {
 		if packs, _ := s.Database.FindPackages(p); len(packs) == 0 {
-			return errors.New("Package not found in the system")
+			return nil, nil, errors.New("Package not found in the system")
 		}
-
 	}
 
-	Spinner(32)
-	toUninstall, err := l.computeUninstall(s, packs...)
+	toUninstall, err := l.computeUninstall(o, s, packs...)
 	if err != nil {
-		return errors.Wrap(err, "while computing uninstall")
+		return nil, nil, errors.Wrap(err, "while computing uninstall")
 	}
-	SpinnerStop()
 
 	uninstall := func() error {
 		for _, p := range toUninstall {
 			err := l.uninstall(p, s)
-			if err != nil && !l.Options.Force {
+			if err != nil && !o.Force {
 				return errors.Wrap(err, "Uninstall failed")
 			}
 		}
 		return nil
 	}
 
+	return toUninstall, uninstall, nil
+}
+
+func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
+
+	Spinner(32)
+	o := Option{
+		FullUninstall:      l.Options.FullUninstall,
+		Force:              l.Options.Force,
+		CheckConflicts:     l.Options.CheckConflicts,
+		FullCleanUninstall: l.Options.FullCleanUninstall,
+	}
+	toUninstall, uninstall, err := l.generateUninstallFn(o, s, packs...)
+	if err != nil {
+		return errors.Wrap(err, "while computing uninstall")
+	}
+	SpinnerStop()
+
 	if len(toUninstall) == 0 {
 		Info("Nothing to do")
 		return nil
@@ -847,4 +1181,4 @@ func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
 	return uninstall()
 }
 
-func (l *LuetInstaller) Repositories(r []Repository) { l.PackageRepositories = r }
+func (l *LuetInstaller) Repositories(r []*LuetSystemRepository) { l.PackageRepositories = r }

pkg/installer/installer_test.go

@@ -23,9 +23,11 @@ import (
 	//	. "github.com/mudler/luet/pkg/installer"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/helpers"
-	"github.com/mudler/luet/pkg/installer"
-	solver "github.com/mudler/luet/pkg/solver"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -34,7 +36,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func stubRepo(tmpdir, tree string) (installer.Repository, error) {
+func stubRepo(tmpdir, tree string) (*LuetSystemRepository, error) {
 	return GenerateRepository(
 		"test",
 		"description",
@@ -43,10 +45,11 @@ func stubRepo(tmpdir, tree string) (installer.Repository, error) {
 		1,
 		tmpdir,
 		[]string{tree},
-		pkg.NewInMemoryDatabase(false), nil, "", false, false)
+		pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 }
 
 var _ = Describe("Installer", func() {
+
 	Context("Writes a repository definition", func() {
 		It("Writes a repo and can install packages from it", func() {
 			//repo:=NewLuetSystemRepository()
@@ -62,7 +65,9 @@ var _ = Describe("Installer", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
+				generalRecipe.GetDatabase(),
+				options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -77,38 +82,37 @@ var _ = Describe("Installer", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
-			artifact, err := c.Compile(false, spec)
+			a, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(a.Path)).To(BeTrue())
+			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -133,8 +137,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -151,8 +155,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
@@ -178,7 +182,7 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
 			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
-				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+				generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -193,43 +197,42 @@ urls:
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 
 			treeFile := NewDefaultTreeRepositoryFile()
-			treeFile.SetCompressionType(compiler.None)
+			treeFile.SetCompressionType(compression.None)
 			repo.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -254,8 +257,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -272,8 +275,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
@@ -298,7 +301,8 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -313,25 +317,24 @@ urls:
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := GenerateRepository(
 				"test",
@@ -340,18 +343,18 @@ urls:
 				[]string{tmpdir}, 1,
 				tmpdir,
 				[]string{"../../tests/fixtures/buildable"},
-				pkg.NewInMemoryDatabase(false), nil, "", false, false)
+				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -381,8 +384,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -399,8 +402,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
@@ -423,7 +426,8 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -438,25 +442,24 @@ urls:
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := GenerateRepository(
 				"test",
@@ -466,18 +469,18 @@ urls:
 				1,
 				tmpdir,
 				[]string{"../../tests/fixtures/buildable"},
-				pkg.NewInMemoryDatabase(false), nil, "", false, false)
+				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -518,7 +521,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), options.Concurrency(2))
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -526,11 +529,10 @@ urls:
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
 
 			spec.SetOutputPath(tmpdir2)
-			c.SetConcurrency(2)
 
 			artifact, err = c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 
 			repo, err = stubRepo(tmpdir2, "../../tests/fixtures/alpine")
 			Expect(err).ToNot(HaveOccurred())
@@ -577,7 +579,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -595,24 +597,23 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -642,8 +643,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -660,11 +661,11 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			// New version - new files
-			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
 			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
 			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -695,8 +696,8 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 			Expect(len(generalRecipeNewRepo.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase())
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -718,21 +719,20 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdirnewrepo)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec3))
 
 			Expect(errs).To(BeEmpty())
 
-			_, errs = c2.CompileParallel(false, compiler.NewLuetCompilationspecs(spec2))
+			_, errs = c2.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeEmpty())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
@@ -775,8 +775,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -794,11 +794,11 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			// New version - new files
-			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
 			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
 			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -827,7 +827,12 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(
+				backend.NewSimpleDockerBackend(),
+				generalRecipe.GetDatabase(),
+				options.Concurrency(2),
+				options.WithCompressionType(compression.GZip),
+			)
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -844,26 +849,25 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
-			c.SetCompressionType(compiler.GZip)
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -893,8 +897,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -911,11 +915,11 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			// New version - new files
-			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
 			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
 			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -985,7 +989,9 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.Concurrency(2),
+				options.WithCompressionType(compression.GZip))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1002,26 +1008,24 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
-			c.SetCompressionType(compiler.GZip)
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -1056,13 +1060,13 @@ urls:
 			Expect(err).To(HaveOccurred())
 
 			Expect(len(system.Database.World())).To(Equal(0))
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeFalse())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeFalse())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "c"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "c"))).To(BeFalse())
 
-			Expect(helpers.Touch(filepath.Join(fakeroot, "test5"))).ToNot(HaveOccurred())
-			Expect(helpers.Touch(filepath.Join(fakeroot, "test6"))).ToNot(HaveOccurred())
-			Expect(helpers.Touch(filepath.Join(fakeroot, "c"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "test5"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "test6"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "c"))).ToNot(HaveOccurred())
 
 			err = inst.Reclaim(system)
 			Expect(err).ToNot(HaveOccurred())
@@ -1090,7 +1094,8 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.WithCompressionType(compression.GZip))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1104,24 +1109,22 @@ urls:
 			defer os.RemoveAll(tmpdir) // clean up
 			spec.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(1)
-			c.SetCompressionType(compiler.GZip)
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec3))
 
 			Expect(errs).To(BeEmpty())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -1156,13 +1159,13 @@ urls:
 			Expect(err).To(HaveOccurred())
 
 			Expect(len(system.Database.World())).To(Equal(0))
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeFalse())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeFalse())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "c"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "c"))).To(BeFalse())
 
-			Expect(helpers.Touch(filepath.Join(fakeroot, "test5"))).ToNot(HaveOccurred())
-			Expect(helpers.Touch(filepath.Join(fakeroot, "test6"))).ToNot(HaveOccurred())
-			Expect(helpers.Touch(filepath.Join(fakeroot, "c"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "test5"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "test6"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "c"))).ToNot(HaveOccurred())
 
 			err = inst.Reclaim(system)
 			Expect(err).ToNot(HaveOccurred())
@@ -1182,7 +1185,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(3))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase())
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1192,7 +1195,7 @@ urls:
 			defer os.RemoveAll(tmpdir2) // clean up
 			spec.SetOutputPath(tmpdir2)
 
-			_, errs = c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec))
+			_, errs = c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 
 			Expect(errs).To(BeEmpty())
 
@@ -1216,11 +1219,11 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			// New version - new files
-			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
 			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
 			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})

pkg/installer/interface.go

@@ -16,62 +16,13 @@
 package installer
 
 import (
-	compiler "github.com/mudler/luet/pkg/compiler"
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/tree"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	//"github.com/mudler/luet/pkg/solver"
 )
 
-type Installer interface {
-	Install(pkg.Packages, *System) error
-	Uninstall(*System, ...pkg.Package) error
-	Upgrade(s *System) error
-	Reclaim(s *System) error
-
-	Repositories([]Repository)
-	SyncRepositories(bool) (Repositories, error)
-	Swap(pkg.Packages, pkg.Packages, *System) error
-}
-
 type Client interface {
-	DownloadArtifact(compiler.Artifact) (compiler.Artifact, error)
+	DownloadArtifact(*artifact.PackageArtifact) (*artifact.PackageArtifact, error)
 	DownloadFile(string) (string, error)
 }
 
-type Repositories []Repository
-
-type Repository interface {
-	GetName() string
-	GetDescription() string
-	GetUrls() []string
-	SetUrls([]string)
-	AddUrl(string)
-	GetPriority() int
-	GetIndex() compiler.ArtifactIndex
-	SetIndex(i compiler.ArtifactIndex)
-	GetTree() tree.Builder
-	SetTree(tree.Builder)
-	Write(path string, resetRevision, force bool) error
-	Sync(bool) (Repository, error)
-	GetTreePath() string
-	SetTreePath(string)
-	GetMetaPath() string
-	SetMetaPath(string)
-	GetType() string
-	SetType(string)
-	SetAuthentication(map[string]string)
-	GetAuthentication() map[string]string
-	GetRevision() int
-	IncrementRevision()
-	GetLastUpdate() string
-	SetLastUpdate(string)
-	Client() Client
-
-	SetPriority(int)
-	GetRepositoryFile(string) (LuetRepositoryFile, error)
-	SetRepositoryFile(string, LuetRepositoryFile)
-	SetName(p string)
-	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
-	GetBackend() compiler.CompilerBackend
-	SetBackend(b compiler.CompilerBackend)
-}
+type Repositories []*LuetSystemRepository

pkg/installer/repository_docker.go

@@ -0,0 +1,274 @@
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mudler/luet/pkg/bus"
+	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/helpers/docker"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/pkg/errors"
+)
+
+type dockerRepositoryGenerator struct {
+	b                compiler.CompilerBackend
+	imagePrefix      string
+	imagePush, force bool
+}
+
+func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
+
+	Info("Generating docker images for packages in", l.imagePrefix)
+	var art []*artifact.PackageArtifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			Debug("Skipping", info.Name(), err.Error())
+			return nil
+		}
+		if info.IsDir() {
+			Debug("Skipping directories")
+			return nil
+		}
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil
+		}
+
+		if err := l.pushImageFromArtifact(artifact.NewPackageArtifact(currentpath), l.b, true); err != nil {
+			return errors.Wrap(err, "while pushing metadata file associated to the artifact")
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		a, err := artifact.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+		// Set the path relative to the file.
+		// The metadata contains the full path where the file was located during buildtime.
+		a.Path = filepath.Join(filepath.Dir(currentpath), filepath.Base(a.Path))
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(a.CompileSpec.Package); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				a.CompileSpec.Package.HumanReadableString()))
+			return nil
+		}
+
+		packageImage := fmt.Sprintf("%s:%s", l.imagePrefix, a.CompileSpec.GetPackage().ImageID())
+
+		if l.imagePush && l.b.ImageAvailable(packageImage) && !l.force {
+			Info("Image", packageImage, "already present, skipping. use --force-push to override")
+		} else {
+			Info("Generating final image", packageImage,
+				"for package ", a.CompileSpec.GetPackage().HumanReadableString())
+			if opts, err := a.GenerateFinalImage(packageImage, l.b, true); err != nil {
+				return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+			}
+		}
+		if l.imagePush {
+			if err := pushImage(l.b, packageImage, l.force); err != nil {
+				return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage)
+			}
+		}
+
+		art = append(art, a)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
+func pushImage(b compiler.CompilerBackend, image string, force bool) error {
+	if b.ImageAvailable(image) && !force {
+		Debug("Image", image, "already present, skipping")
+		return nil
+	}
+	return b.Push(backend.Options{ImageName: image})
+}
+
+func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string) error {
+	Debug("Generating image", imageTree)
+	if opts, err := a.GenerateFinalImage(imageTree, d.b, false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
+	}
+	if d.imagePush {
+		if err := pushImage(d.b, imageTree, true); err != nil {
+			return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree)
+		}
+	}
+	return nil
+}
+
+func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec string, r *LuetSystemRepository) error {
+	// create temp dir for metafile
+	metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaDir) // clean up
+
+	tempRepoFile := filepath.Join(metaDir, REPOSITORY_SPECFILE+".tar")
+	if err := helpers.Tar(repospec, tempRepoFile); err != nil {
+		return errors.Wrap(err, "Error met while archiving repository file")
+	}
+
+	a := artifact.NewPackageArtifact(tempRepoFile)
+	imageRepo := fmt.Sprintf("%s:%s", d.imagePrefix, REPOSITORY_SPECFILE)
+
+	if err := d.pushFileFromArtifact(a, imageRepo); err != nil {
+		return errors.Wrap(err, "while pushing file from artifact")
+	}
+	return nil
+}
+
+func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, b compiler.CompilerBackend, checkIfExists bool) error {
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	treeArchive, err := artifact.CreateArtifactForFile(a.Path)
+	if err != nil {
+		return errors.Wrap(err, "failed generating checksums for tree")
+	}
+	imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, docker.StripInvalidStringsFromImage(a.GetFileName()))
+	if checkIfExists && d.imagePush && d.b.ImageAvailable(imageTree) && !d.force {
+		Info("Image", imageTree, "already present, skipping. use --force-push to override")
+		return nil
+	} else {
+		return d.pushFileFromArtifact(treeArchive, imageTree)
+	}
+}
+
+// Generate creates a Docker luet repository
+func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefix string, resetRevision bool) error {
+	// - Iterate over meta, build final images, push them if necessary
+	//   - while pushing, check if image already exists, and if exist push them only if --force is supplied
+	// - Generate final images for metadata and push
+
+	imageRepository := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
+
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
+
+	repoTemp, err := config.LuetCfg.GetSystem().TempDir("repo")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for repository")
+	}
+	defer os.RemoveAll(repoTemp) // clean up
+
+	if r.GetBackend().ImageAvailable(imageRepository) {
+		if err := r.GetBackend().DownloadImage(backend.Options{ImageName: imageRepository}); err != nil {
+			return errors.Wrapf(err, "while downloading '%s'", imageRepository)
+		}
+
+		if err := r.GetBackend().ExtractRootfs(backend.Options{ImageName: imageRepository, Destination: repoTemp}, false); err != nil {
+			return errors.Wrapf(err, "while extracting '%s'", imageRepository)
+		}
+	}
+
+	repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE)
+
+	// Increment the internal revision version by reading the one which is already available (if any)
+	if err := r.BumpRevision(repospec, resetRevision); err != nil {
+		return err
+	}
+
+	Info(fmt.Sprintf(
+		"For repository %s creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imageRepository,
+	})
+
+	// Create tree and repository file
+	a, err := r.AddTree(r.GetTree(), repoTemp, REPOFILE_TREE_KEY, NewDefaultTreeRepositoryFile())
+	if err != nil {
+		return errors.Wrap(err, "error met while adding runtime tree to repository")
+	}
+
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	if err := d.pushImageFromArtifact(a, d.b, false); err != nil {
+		return errors.Wrap(err, "error met while pushing runtime tree")
+	}
+
+	a, err = r.AddTree(r.BuildTree, repoTemp, REPOFILE_COMPILER_TREE_KEY, NewDefaultCompilerTreeRepositoryFile())
+	if err != nil {
+		return errors.Wrap(err, "error met while adding compiler tree to repository")
+	}
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	if err := d.pushImageFromArtifact(a, d.b, false); err != nil {
+		return errors.Wrap(err, "error met while pushing compiler tree")
+	}
+
+	// create temp dir for metafile
+	metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaDir) // clean up
+
+	a, err = r.AddMetadata(repospec, metaDir)
+	if err != nil {
+		return errors.Wrap(err, "failed adding Metadata file to repository")
+	}
+
+	if err := d.pushImageFromArtifact(a, d.b, false); err != nil {
+		return errors.Wrap(err, "error met while pushing docker image from artifact")
+	}
+
+	if err := d.pushRepoMetadata(repospec, r); err != nil {
+		return errors.Wrap(err, "while pushing repository metadata tree")
+	}
+
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imagePrefix,
+	})
+	return nil
+}

pkg/installer/repository_local.go

@@ -0,0 +1,132 @@
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/mudler/luet/pkg/bus"
+	"github.com/pkg/errors"
+)
+
+type localRepositoryGenerator struct{}
+
+func (l *localRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
+	return buildPackageIndex(path, db)
+}
+
+func buildPackageIndex(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
+
+	var art []*artifact.PackageArtifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			Debug("Failed walking", err.Error())
+			return err
+		}
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil // Skip with no errors
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		a, err := artifact.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(a.CompileSpec.GetPackage()); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				a.CompileSpec.GetPackage().HumanReadableString()))
+			return nil
+		}
+
+		art = append(art, a)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
+// Generate creates a Local luet repository
+func (*localRepositoryGenerator) Generate(r *LuetSystemRepository, dst string, resetRevision bool) error {
+	err := os.MkdirAll(dst, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
+
+	repospec := filepath.Join(dst, REPOSITORY_SPECFILE)
+	// Increment the internal revision version by reading the one which is already available (if any)
+	if err := r.BumpRevision(repospec, resetRevision); err != nil {
+		return err
+	}
+
+	Info(fmt.Sprintf(
+		"Repository %s: creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
+
+	if _, err := r.AddTree(r.GetTree(), dst, REPOFILE_TREE_KEY, NewDefaultTreeRepositoryFile()); err != nil {
+		return errors.Wrap(err, "error met while adding runtime tree to repository")
+	}
+
+	if _, err := r.AddTree(r.BuildTree, dst, REPOFILE_COMPILER_TREE_KEY, NewDefaultCompilerTreeRepositoryFile()); err != nil {
+		return errors.Wrap(err, "error met while adding compiler tree to repository")
+	}
+
+	if _, err := r.AddMetadata(repospec, dst); err != nil {
+		return errors.Wrap(err, "failed adding Metadata file to repository")
+	}
+
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
+	return nil
+}

pkg/installer/repository_test.go

@@ -1,4 +1,4 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+// Copyright © 2019 Ettore Di Giacinto <mudler@sabayon.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -26,18 +26,20 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	config "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
-	"github.com/mudler/luet/pkg/installer"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
-func dockerStubRepo(tmpdir, tree, image string, push, force bool) (installer.Repository, error) {
+func dockerStubRepo(tmpdir, tree, image string, push, force bool) (*LuetSystemRepository, error) {
 	return GenerateRepository(
 		"test",
 		"description",
@@ -46,7 +48,7 @@ func dockerStubRepo(tmpdir, tree, image string, push, force bool) (installer.Rep
 		1,
 		tmpdir,
 		[]string{tree},
-		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force)
+		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force, false, nil)
 }
 
 var _ = Describe("Repository", func() {
@@ -64,7 +66,7 @@ var _ = Describe("Repository", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -79,38 +81,37 @@ var _ = Describe("Repository", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 		})
 
 		It("Generate repository metadata of files ONLY referenced in a tree", func() {
@@ -132,11 +133,11 @@ var _ = Describe("Repository", func() {
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase())
 			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -153,46 +154,44 @@ var _ = Describe("Repository", func() {
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			compiler2.SetConcurrency(1)
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			artifact2, err := compiler2.Compile(false, spec2)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact2.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact2.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact2.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact2.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 
 			// We check now that the artifact not referenced in the tree
 			// (spec2) is not indexed in the repository
@@ -254,7 +253,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -266,22 +265,21 @@ urls:
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			localcompiler.SetConcurrency(1)
 
-			artifact, err := localcompiler.Compile(false, spec)
+			a, err := localcompiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(a.Path)).To(BeTrue())
+			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
 			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/buildable", repoImage, true, true)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(repoImage, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
@@ -298,11 +296,11 @@ urls:
 
 			f, err := c.DownloadFile("repository.yaml")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(f)).To(ContainSubstring("name: test"))
+			Expect(fileHelper.Read(f)).To(ContainSubstring("name: test"))
 
-			a, err := c.DownloadArtifact(&compiler.PackageArtifact{
+			a, err = c.DownloadArtifact(&artifact.PackageArtifact{
 				Path: "test.tar",
-				CompileSpec: &compiler.LuetCompilationSpec{
+				CompileSpec: &compilerspec.LuetCompilationSpec{
 					Package: &pkg.DefaultPackage{
 						Name:     "b",
 						Category: "test",
@@ -313,7 +311,158 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(a.Unpack(extracted, false)).ToNot(HaveOccurred())
-			Expect(helpers.Read(filepath.Join(extracted, "test6"))).To(Equal("artifact6\n"))
+			Expect(fileHelper.Read(filepath.Join(extracted, "test6"))).To(Equal("artifact6\n"))
+		})
+
+		It("generates images of virtual packages", func() {
+			b := backend.NewSimpleDockerBackend()
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/virtuals")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(5))
+
+			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+
+			spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.99"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			a, err := localcompiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(a.Path)).To(BeTrue())
+			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/virtuals", repoImage, true, true)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(repoImage, false, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "tree.tar.gz"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.meta.yaml.tar"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.yaml"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "a-test-1.99"))).To(BeTrue())
+
+			extracted, err := ioutil.TempDir("", "extracted")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(extracted) // clean up
+
+			c := repo.Client()
+
+			f, err := c.DownloadFile("repository.yaml")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Read(f)).To(ContainSubstring("name: test"))
+
+			a, err = c.DownloadArtifact(&artifact.PackageArtifact{
+				Path: "test.tar",
+				CompileSpec: &compilerspec.LuetCompilationSpec{
+					Package: &pkg.DefaultPackage{
+						Name:     "a",
+						Category: "test",
+						Version:  "1.99",
+					},
+				},
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(a.Unpack(extracted, false)).ToNot(HaveOccurred())
+
+			Expect(fileHelper.DirectoryIsEmpty(extracted)).To(BeFalse())
+			content, err := ioutil.ReadFile(filepath.Join(extracted, ".virtual"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(content)).To(Equal(""))
+		})
+
+		It("Searches files", func() {
+			repos := Repositories{
+				&LuetSystemRepository{
+					Index: compiler.ArtifactIndex{
+						&artifact.PackageArtifact{
+							CompileSpec: &compilerspec.LuetCompilationSpec{
+								Package: &pkg.DefaultPackage{},
+							},
+							Path:  "bar",
+							Files: []string{"boo"},
+						},
+						&artifact.PackageArtifact{
+							Path:  "d",
+							Files: []string{"baz"},
+						},
+					},
+				},
+			}
+
+			matches := repos.SearchPackages("bo", FileSearch)
+			Expect(len(matches)).To(Equal(1))
+			Expect(matches[0].Artifact.Path).To(Equal("bar"))
+		})
+
+		It("Searches packages", func() {
+			repo := &LuetSystemRepository{
+				Index: compiler.ArtifactIndex{
+					&artifact.PackageArtifact{
+						Path: "foo",
+						CompileSpec: &compilerspec.LuetCompilationSpec{
+							Package: &pkg.DefaultPackage{
+								Name:     "foo",
+								Category: "bar",
+								Version:  "1.0",
+							},
+						},
+					},
+					&artifact.PackageArtifact{
+						Path: "baz",
+						CompileSpec: &compilerspec.LuetCompilationSpec{
+							Package: &pkg.DefaultPackage{
+								Name:     "foo",
+								Category: "baz",
+								Version:  "1.0",
+							},
+						},
+					},
+				},
+			}
+
+			a, err := repo.SearchArtefact(&pkg.DefaultPackage{
+				Name:     "foo",
+				Category: "baz",
+				Version:  "1.0",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(a.Path).To(Equal("baz"))
+
+			a, err = repo.SearchArtefact(&pkg.DefaultPackage{
+				Name:     "foo",
+				Category: "bar",
+				Version:  "1.0",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(a.Path).To(Equal("foo"))
+
+			// Doesn't exist. so must fail
+			_, err = repo.SearchArtefact(&pkg.DefaultPackage{
+				Name:     "foo",
+				Category: "bar",
+				Version:  "1.1",
+			})
+			Expect(err).To(HaveOccurred())
 		})
 	})
 })

pkg/installer/system.go

@@ -1,16 +1,21 @@
 package installer
 
 import (
+	"sync"
+
 	"github.com/hashicorp/go-multierror"
 	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 )
 
 type System struct {
-	Database pkg.PackageDatabase
-	Target   string
+	Database  pkg.PackageDatabase
+	Target    string
+	fileIndex map[string]pkg.Package
+	sync.Mutex
 }
 
 func (s *System) World() (pkg.Packages, error) {
@@ -23,8 +28,8 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
 	var errs error
 	executedFinalizer := map[string]bool{}
 	for _, p := range packs {
-		if helpers.Exists(p.Rel(tree.FinalizerFile)) {
-			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile), "")
+		if fileHelper.Exists(p.Rel(tree.FinalizerFile)) {
+			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile))
 			if err != nil {
 				Warning("Failed rendering finalizer for ", p.HumanReadableString(), err.Error())
 				errs = multierror.Append(errs, err)
@@ -51,3 +56,38 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
 	}
 	return errs
 }
+
+func (s *System) buildFileIndex() {
+	s.Lock()
+	defer s.Unlock()
+	// Check if cache is empty or if it got modified
+	if s.fileIndex == nil { //|| len(s.Database.GetPackages()) != len(s.fileIndex) {
+		s.fileIndex = make(map[string]pkg.Package)
+		for _, p := range s.Database.World() {
+			files, _ := s.Database.GetPackageFiles(p)
+			for _, f := range files {
+				s.fileIndex[f] = p
+			}
+		}
+	}
+}
+
+func (s *System) Clean() {
+	s.Lock()
+	defer s.Unlock()
+	s.fileIndex = nil
+}
+
+func (s *System) ExistsPackageFile(file string) (bool, pkg.Package, error) {
+	Debug("Checking if file ", file, "belongs to any package")
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	if p, exists := s.fileIndex[file]; exists {
+		Debug(file, "belongs already to", p.HumanReadableString())
+
+		return exists, p, nil
+	}
+	Debug(file, "doesn't belong to any package")
+	return false, nil, nil
+}

pkg/installer/system_test.go

@@ -0,0 +1,70 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer_test
+
+import (
+
+	//	. "github.com/mudler/luet/pkg/installer"
+
+	. "github.com/mudler/luet/pkg/installer"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("System", func() {
+	Context("Files", func() {
+		var s *System
+		var db pkg.PackageDatabase
+		var a, b *pkg.DefaultPackage
+
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			s = &System{Database: db}
+
+			a = &pkg.DefaultPackage{Name: "test", Version: "1", Category: "t"}
+
+			db.CreatePackage(a)
+			db.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.GetFingerPrint(), Files: []string{"foo", "f"}})
+
+			b = &pkg.DefaultPackage{Name: "test2", Version: "1", Category: "t"}
+
+			db.CreatePackage(b)
+			db.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: b.GetFingerPrint(), Files: []string{"barz", "f"}})
+		})
+
+		It("detects when are already shipped by other packages", func() {
+			r, p, err := s.ExistsPackageFile("foo")
+			Expect(r).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(Equal(a))
+			r, p, err = s.ExistsPackageFile("baz")
+			Expect(r).To(BeFalse())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(BeNil())
+
+			r, p, err = s.ExistsPackageFile("f")
+			Expect(r).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(Equal(b))
+			r, p, err = s.ExistsPackageFile("barz")
+			Expect(r).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(Equal(b))
+		})
+	})
+})

pkg/logger/logger.go

@@ -8,6 +8,7 @@ import (
 	"runtime"
 	"strings"
 
+	"sync"
 	. "github.com/mudler/luet/pkg/config"
 
 	"github.com/briandowns/spinner"
@@ -20,7 +21,7 @@ import (
 var s *spinner.Spinner = nil
 var z *zap.Logger = nil
 var aurora Aurora = nil
-
+var spinnerLock = sync.Mutex{}
 func NewSpinner() {
 	if s == nil {
 		s = spinner.New(
@@ -84,6 +85,8 @@ func ZapLogger() error {
 }
 
 func Spinner(i int) {
+	spinnerLock.Lock()
+	defer spinnerLock.Unlock()
 	var confLevel int
 	if LuetCfg.GetGeneral().Debug {
 		confLevel = 3
@@ -120,6 +123,8 @@ func SpinnerText(suffix, prefix string) {
 }
 
 func SpinnerStop() {
+	spinnerLock.Lock()
+	defer spinnerLock.Unlock()
 	var confLevel int
 	if LuetCfg.GetGeneral().Debug {
 		confLevel = 3
@@ -173,7 +178,7 @@ func level2AtomicLevel(level string) zap.AtomicLevel {
 	}
 }
 
-func msg(level string, withoutColor bool, msg ...interface{}) {
+func Msg(level string, withoutColor, ln bool, msg ...interface{}) {
 	var message string
 	var confLevel, msgLevel int
 
@@ -219,11 +224,16 @@ func msg(level string, withoutColor bool, msg ...interface{}) {
 		log2File(level, message)
 	}
 
-	fmt.Println(levelMsg)
+	if ln {
+		fmt.Println(levelMsg)
+	} else {
+		fmt.Print(levelMsg)
+	}
+
 }
 
 func Warning(mess ...interface{}) {
-	msg("warning", false, mess...)
+	Msg("warning", false, true, mess...)
 	if LuetCfg.GetGeneral().FatalWarns {
 		os.Exit(2)
 	}
@@ -235,23 +245,23 @@ func Debug(mess ...interface{}) {
 		mess = append([]interface{}{fmt.Sprintf("DEBUG (%s:#%d:%v)",
 			path.Base(file), line, runtime.FuncForPC(pc).Name())}, mess...)
 	}
-	msg("debug", false, mess...)
+	Msg("debug", false, true, mess...)
 }
 
 func DebugC(mess ...interface{}) {
-	msg("debug", true, mess...)
+	Msg("debug", true, true, mess...)
 }
 
 func Info(mess ...interface{}) {
-	msg("info", false, mess...)
+	Msg("info", false, true, mess...)
 }
 
 func InfoC(mess ...interface{}) {
-	msg("info", true, mess...)
+	Msg("info", true, true, mess...)
 }
 
 func Error(mess ...interface{}) {
-	msg("error", false, mess...)
+	Msg("error", false, true, mess...)
 }
 
 func Fatal(mess ...interface{}) {

pkg/package/database.go

@@ -52,6 +52,7 @@ type PackageSet interface {
 	FindPackageLabel(labelKey string) (Packages, error)
 	FindPackageLabelMatch(pattern string) (Packages, error)
 	FindPackageMatch(pattern string) (Packages, error)
+	FindPackageByFile(pattern string) (Packages, error)
 }
 
 type PackageFile struct {

pkg/package/database_boltdb.go

@@ -436,9 +436,9 @@ func (db *BoltDatabase) FindPackageLabel(labelKey string) (Packages, error) {
 func (db *BoltDatabase) FindPackageLabelMatch(pattern string) (Packages, error) {
 	var ans []Package
 
-	re := regexp.MustCompile(pattern)
-	if re == nil {
-		return nil, errors.New("Invalid regex " + pattern + "!")
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, errors.Wrap(err, "Invalid regex "+pattern+"!")
 	}
 
 	for _, pack := range db.World() {
@@ -450,12 +450,15 @@ func (db *BoltDatabase) FindPackageLabelMatch(pattern string) (Packages, error)
 	return Packages(ans), nil
 }
 
+func (db *BoltDatabase) FindPackageByFile(pattern string) (Packages, error) {
+	return findPackageByFile(db, pattern)
+}
 func (db *BoltDatabase) FindPackageMatch(pattern string) (Packages, error) {
 	var ans []Package
 
-	re := regexp.MustCompile(pattern)
-	if re == nil {
-		return nil, errors.New("Invalid regex " + pattern + "!")
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, errors.Wrap(err, "Invalid regex "+pattern+"!")
 	}
 
 	for _, pack := range db.World() {

pkg/package/database_boltdb_test.go

@@ -57,6 +57,31 @@ var _ = Describe("BoltDB Database", func() {
 
 		})
 
+		It("Find package files", func() {
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = db.SetPackageFiles(&PackageFile{PackageFingerprint: a.GetFingerPrint(), Files: []string{"foo"}})
+			Expect(err).ToNot(HaveOccurred())
+
+			err = db.SetPackageFiles(&PackageFile{PackageFingerprint: a1.GetFingerPrint(), Files: []string{"bar"}})
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.FindPackageByFile("fo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(pack)).To(Equal(1))
+			Expect(pack[0]).To(Equal(a))
+		})
+
 		It("Expands correctly", func() {
 
 			a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})

pkg/package/database_common.go

@@ -15,7 +15,11 @@
 
 package pkg
 
-import "github.com/pkg/errors"
+import (
+	"regexp"
+
+	"github.com/pkg/errors"
+)
 
 func clone(src, dst PackageDatabase) error {
 	for _, i := range src.World() {
@@ -36,3 +40,29 @@ func copy(src PackageDatabase) (PackageDatabase, error) {
 
 	return dst, nil
 }
+
+func findPackageByFile(db PackageDatabase, pattern string) (Packages, error) {
+
+	var ans []Package
+
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, errors.Wrap(err, "Invalid regex "+pattern+"!")
+	}
+
+PACKAGE:
+	for _, pack := range db.World() {
+		files, err := db.GetPackageFiles(pack)
+		if err == nil {
+			for _, f := range files {
+				if re.MatchString(f) {
+					ans = append(ans, pack)
+					continue PACKAGE
+				}
+			}
+		}
+	}
+
+	return Packages(ans), nil
+
+}

pkg/package/database_mem.go

@@ -32,6 +32,7 @@ var DBInMemoryInstance = &InMemoryDatabase{
 	CacheNoVersion:   map[string]map[string]interface{}{},
 	ProvidesDatabase: map[string]map[string]Package{},
 	RevDepsDatabase:  map[string]map[string]Package{},
+	cached:           map[string]interface{}{},
 }
 
 type InMemoryDatabase struct {
@@ -199,6 +200,10 @@ func (db *InMemoryDatabase) populateCaches(p Package) {
 
 	// Create extra cache between package -> []versions
 	db.Lock()
+	if db.cached == nil {
+		db.cached = map[string]interface{}{}
+	}
+
 	if _, ok := db.cached[p.GetFingerPrint()]; ok {
 		db.Unlock()
 		return
@@ -555,3 +560,7 @@ func (db *InMemoryDatabase) FindPackageMatch(pattern string) (Packages, error) {
 
 	return Packages(ans), nil
 }
+
+func (db *InMemoryDatabase) FindPackageByFile(pattern string) (Packages, error) {
+	return findPackageByFile(db, pattern)
+}