Tag 0.14.7

Add unit tests, consume imagehashtree in compiler and cleanup

Fixes: #203

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: mudler
+
+---
+
+<!-- Thanks for helping us to improve Luet! We welcome all bug reports. Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
+
+**Luet version:**
+<!-- Provide the output from "luet --version" -->
+
+**CPU architecture, OS, and Version:**
+<!-- Provide the output from "uname -a"  -->
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+<!-- Steps to reproduce the behavior, including the luet command used -->
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Logs**
+<!-- If applicable, add logs with the "--debug" flag enabled to help explain your problem. -->
+
+**Additional context**
+<!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: mudler
+
+---
+
+<!-- Thanks for helping us to improve Luet! We welcome all feature requests. Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
+
+**Is your feature request related to a problem? Please describe.**
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]  -->
+
+**Describe the solution you'd like**
+<!-- A clear and concise description of what you want to happen.  -->
+
+**Describe alternatives you've considered**
+<!-- A clear and concise description of any alternative solutions or features you've considered.  -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/workflows/release.yml

@@ -11,10 +11,34 @@ jobs:
           go-version: 1.14.x
       - name: Checkout code
         uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+      - name: Build test
+        run: sudo -E env "PATH=$PATH" make multiarch-build-small
+      - name: Login to quay with img
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Tests with Img backend
+        run: |
+            sudo -E env "PATH=$PATH" \
+                    env "LUET_BACKEND=img" \
+                    make test-integration
       - name: Tests
-        run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage
+        run: |
+              sudo -E \
+                    env "PATH=$PATH" \
+                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                    make test-integration test-coverage
       - name: Build
-        run: sudo -E env "PATH=$PATH" make multiarch-build && sudo chmod -R 777 release/
+        run: sudo -E env "PATH=$PATH" make multiarch-build-small && sudo chmod -R 777 release/
       - name: Release
         uses: fnkr/github-action-ghr@v1
         if: startsWith(github.ref, 'refs/tags/')

.github/workflows/test.yml

@@ -17,5 +17,14 @@ jobs:
       uses: actions/checkout@v2
     - name: setup-docker
       uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+    - name: Build
+      run: sudo -E env "PATH=$PATH" make multiarch-build-small
+    - name: Tests with Img backend
+      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
     - name: Tests
-      run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage
+      run: sudo -E env "PATH=$PATH" make test-integration test-coverage

Makefile

@@ -88,6 +88,11 @@ test-docker:
 				--workdir /go/src/github.com/mudler/luet -ti golang:latest \
 				bash -c "make test"
 
-.PHONY: multiarch-build
 multiarch-build:
 	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+
+multiarch-build-small:
+	@$(MAKE) LDFLAGS+="-s -w" multiarch-build
+	for file in $(ROOT_DIR)/release/* ; do \
+		upx --brute -1 $${file} ; \
+	done

README.md

@@ -1,13 +1,16 @@
 # luet - Container-based Package manager
+
 [![Docker Repository on Quay](https://quay.io/repository/luet/base/status "Docker Repository on Quay")](https://quay.io/repository/luet/base)
 [![Go Report Card](https://goreportcard.com/badge/github.com/mudler/luet)](https://goreportcard.com/report/github.com/mudler/luet)
 [![Build Status](https://travis-ci.org/mudler/luet.svg?branch=master)](https://travis-ci.org/mudler/luet)
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
 [![codecov](https://codecov.io/gh/mudler/luet/branch/master/graph/badge.svg)](https://codecov.io/gh/mudler/luet)
 
-Luet is a multi-platform Package Manager based off from containers - it uses Docker (and other tech) to sandbox your builds and generate packages from them. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
+[![asciicast](https://asciinema.org/a/388348.svg)](https://asciinema.org/a/388348)
 
-It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/specfile/) in YAML notation to define both packages and rootfs. As it is based on containers, it can be used to build seed stages for Linux From Scratch installations and it can build and track updates for those systems.
+Luet is a multi-platform Package Manager based off from containers - it uses Docker (and others) to build packages. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
+
+It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/) in YAML notation to define both [packages](https://luet-lab.github.io/docs/docs/concepts/packages/) and [rootfs](https://luet-lab.github.io/docs/docs/concepts/packages/#package-layers). As it is based on containers, it can be also used to build stages for Linux From Scratch installations and it can build and track updates for those systems.
 
 It is written entirely in Golang and where used as package manager, it can run in from scratch environment, with zero dependencies.
 
@@ -16,25 +19,41 @@ It is written entirely in Golang and where used as package manager, it can run i
 - Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.
 - It builds, installs, uninstalls and perform upgrades on machines
 - Installer doesn't depend on anything ( 0 dep installer !), statically built
-- Support for packages as "layers"
-- It uses SAT solving techniques to solve the deptree ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
+- You can install it aside also with your current distro package manager, and start building and distributing your packages
+- [Support for packages as "layers"](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/#building-strategies)
+- [It uses SAT solving techniques to solve the deptree](https://luet-lab.github.io/docs/docs/concepts/overview/constraints/) ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
+- Support for [collections](https://luet-lab.github.io/docs/docs/concepts/packages/collections/) and [templated package definitions](https://luet-lab.github.io/docs/docs/concepts/packages/templates/)
+- [Can be extended with Plugins and Extensions](https://luet-lab.github.io/docs/docs/concepts/plugins-and-extensions/)
+- [Can build packages in Kubernetes (experimental)](https://github.com/mudler/luet-k8s)
 
 ## Install
 
-To install luet, you can grab a release on the [Release page](https://github.com/mudler/luet/releases) or compile it in your machine (requires Golang installed):
+To install luet, you can grab a release on the [Release page](https://github.com/mudler/luet/releases) or to install it in your system:
 
-    $ git clone https://github.com/mudler/luet.git
-    $ cd luet
-    $ make build
+```bash
+$ curl https://get.mocaccino.org/luet/get_luet_root.sh | sudo sh
+$ luet search ...
+$ luet install ..
+$ luet --help
+```
 
-## Status
+## Build from source
 
-Luet is not feature-complete yet, it can build, install/uninstall/upgrade packages - but it doesn't support yet all the features you would normally expect from a Package Manager nowadays.
+```bash
+$ git clone https://github.com/mudler/luet.git
+$ cd luet
+$ make build
+```
+
+## Documentation
+
+[Documentation](https://luet-lab.github.io/docs) is available, or
+run `luet --help`,  any subcommand is documented as well, try e.g.: `luet build --help`.
 
 # Dependency solving
 
 Luet uses SAT and Reinforcement learning engine for dependency solving.
-It encodes the package requirements into a SAT problem, using gophersat to solve the dependency tree and give a concrete model as result.
+It encodes the package requirements into a SAT problem, using [gophersat](https://github.com/crillab/gophersat) to solve the dependency tree and give a concrete model as result.
 
 ## SAT encoding
 
@@ -48,10 +67,6 @@ when they arises while trying to validate your queries against the system model.
 
 To leverage it, simply pass ```--solver-type qlearning``` to the subcommands that supports it ( you can check out by invoking ```--help``` ).
 
-## Documentation
-
-[Documentation](https://luet-lab.github.io/docs) is available, or
-run `luet --help`,  any subcommand is documented as well, try e.g.: `luet build --help`.
 
 ## Authors
 

cmd/build.go

@@ -16,13 +16,18 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
+	"path/filepath"
 
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	"github.com/mudler/luet/pkg/installer"
+
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -36,35 +41,58 @@ import (
 var buildCmd = &cobra.Command{
 	Use:   "build <package name> <package name> <package name> ...",
 	Short: "build a package or a tree",
-	Long:  `build packages or trees from luet tree definitions. Packages are in [category]/[name]-[version] form`,
-	PreRun: func(cmd *cobra.Command, args []string) {
-		viper.BindPFlag("clean", cmd.Flags().Lookup("clean"))
+	Long: `Builds one or more packages from a tree (current directory is implied):
+
+	$ luet build utils/busybox utils/yq ...
+
+Builds all packages
+
+	$ luet build --all
+
+Builds only the leaf packages:
+
+	$ luet build --full
+
+Build package revdeps:
+
+	$ luet build --revdeps utils/yq
+
+Build package without dependencies (needs the images already in the host, or either need to be available online):
+
+	$ luet build --nodeps utils/yq ...
+
+Build packages specifying multiple definition trees:
+
+	$ luet build --tree overlay/path --tree overlay/path2 utils/yq ...
+`, PreRun: func(cmd *cobra.Command, args []string) {
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
 		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
 		viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged"))
-		viper.BindPFlag("database", cmd.Flags().Lookup("database"))
 		viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
 		viper.BindPFlag("all", cmd.Flags().Lookup("all"))
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
 		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("values", cmd.Flags().Lookup("values"))
+		viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
 
 		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
 		viper.BindPFlag("push", cmd.Flags().Lookup("push"))
 		viper.BindPFlag("pull", cmd.Flags().Lookup("pull"))
+		viper.BindPFlag("wait", cmd.Flags().Lookup("wait"))
 		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
 
-		LuetCfg.Viper.BindPFlag("keep-exported-images", cmd.Flags().Lookup("keep-exported-images"))
-
 		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
 		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
 		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("general.show_build_output", cmd.Flags().Lookup("live-output"))
+		LuetCfg.Viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
+
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
-		clean := viper.GetBool("clean")
 		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("destination")
 		concurrency := LuetCfg.GetGeneral().Concurrency
@@ -72,63 +100,50 @@ var buildCmd = &cobra.Command{
 		privileged := viper.GetBool("privileged")
 		revdeps := viper.GetBool("revdeps")
 		all := viper.GetBool("all")
-		databaseType := viper.GetString("database")
 		compressionType := viper.GetString("compression")
 		imageRepository := viper.GetString("image-repository")
+		values := viper.GetStringSlice("values")
+		wait := viper.GetBool("wait")
 		push := viper.GetBool("push")
 		pull := viper.GetBool("pull")
 		keepImages := viper.GetBool("keep-images")
 		nodeps := viper.GetBool("nodeps")
 		onlydeps := viper.GetBool("onlydeps")
-		keepExportedImages := viper.GetBool("keep-exported-images")
 		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
 		full, _ := cmd.Flags().GetBool("full")
-		skip, _ := cmd.Flags().GetBool("skip-if-metadata-exists")
+		rebuild, _ := cmd.Flags().GetBool("rebuild")
+
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		var results Results
+		backendArgs := viper.GetStringSlice("backend-args")
 
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
 			LuetCfg.GetLogging().SetLogLevel("error")
 		}
 		pretend, _ := cmd.Flags().GetBool("pretend")
-		compilerSpecs := compiler.NewLuetCompilationspecs()
-		var compilerBackend compiler.CompilerBackend
+		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
+
+		compilerSpecs := compilerspec.NewLuetCompilationspecs()
 		var db pkg.PackageDatabase
-		switch backendType {
-		case "img":
-			compilerBackend = backend.NewSimpleImgBackend()
-		case "docker":
-			compilerBackend = backend.NewSimpleDockerBackend()
-		}
 
-		switch databaseType {
-		case "memory":
-			db = pkg.NewInMemoryDatabase(false)
+		compilerBackend, err := compiler.NewBackend(backendType)
+		helpers.CheckErr(err)
 
-		case "boltdb":
-			tmpdir, err := ioutil.TempDir("", "package")
-			if err != nil {
-				Fatal(err)
-			}
-			db = pkg.NewBoltDatabase(tmpdir)
-
-		}
+		db = pkg.NewInMemoryDatabase(false)
 		defer db.Clean()
 
 		generalRecipe := tree.NewCompilerRecipe(db)
 
-		if len(treePaths) <= 0 {
-			Fatal("No tree path supplied!")
+		if fromRepo {
+			if err := installer.LoadBuildTree(generalRecipe, db, LuetCfg); err != nil {
+				Warning("errors while loading trees from repositories", err.Error())
+			}
 		}
 
 		for _, src := range treePaths {
 			Info("Loading tree", src)
-
-			err := generalRecipe.Load(src)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
+			helpers.CheckErr(generalRecipe.Load(src))
 		}
 
 		Info("Building in", dst)
@@ -137,37 +152,44 @@ var buildCmd = &cobra.Command{
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
 		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		LuetCfg.GetGeneral().ShowBuildOutput = LuetCfg.Viper.GetBool("general.show_build_output")
 
-		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
-
-		opts := compiler.NewDefaultCompilerOptions()
-		opts.SolverOptions = *LuetCfg.GetSolverOptions()
-		opts.ImageRepository = imageRepository
-		opts.Clean = clean
-		opts.PullFirst = pull
-		opts.KeepImg = keepImages
-		opts.Push = push
-		opts.OnlyDeps = onlydeps
-		opts.NoDeps = nodeps
-		opts.KeepImageExport = keepExportedImages
-		opts.SkipIfMetadataExists = skip
-		opts.PackageTargetOnly = onlyTarget
-
-		var solverOpts solver.Options
-		if concurrent {
-			solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
-		} else {
-			solverOpts = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+		opts := &LuetSolverOptions{
+			Type:        stype,
+			LearnRate:   float32(rate),
+			Discount:    float32(discount),
+			MaxAttempts: attempts,
 		}
 
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts, solverOpts)
-		luetCompiler.SetConcurrency(concurrency)
-		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
+		Debug("Solver", opts.CompactString())
+
+		if concurrent {
+			opts.Options = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
+		} else {
+			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+		}
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
+			options.NoDeps(nodeps),
+			options.WithBackendType(backendType),
+			options.PushImages(push),
+			options.WithBuildValues(values),
+			options.WithPullRepositories(pullRepo),
+			options.WithPushRepository(imageRepository),
+			options.Rebuild(rebuild),
+			options.WithSolverOptions(*opts),
+			options.Wait(wait),
+			options.OnlyTarget(onlyTarget),
+			options.PullFirst(pull),
+			options.KeepImg(keepImages),
+			options.OnlyDeps(onlydeps),
+			options.BackendArgs(backendArgs),
+			options.Concurrency(concurrency),
+			options.WithCompressionType(compression.Implementation(compressionType)),
+		)
+
 		if full {
 			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
 			if err != nil {
@@ -180,7 +202,6 @@ var buildCmd = &cobra.Command{
 			}
 		} else if !all {
 			for _, a := range args {
-
 				pack, err := helpers.ParsePackageStr(a)
 				if err != nil {
 					Fatal("Invalid package string ", a, ": ", err.Error())
@@ -208,13 +229,13 @@ var buildCmd = &cobra.Command{
 			}
 		}
 
-		var artifact []compiler.Artifact
+		var artifact []*artifact.PackageArtifact
 		var errs []error
 		if revdeps {
 			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
 		} else if pretend {
-			toCalculate := []compiler.CompilationSpec{}
+			toCalculate := []*compilerspec.LuetCompilationSpec{}
 			if full {
 				var err error
 				toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...)
@@ -226,11 +247,12 @@ var buildCmd = &cobra.Command{
 			}
 
 			for _, sp := range toCalculate {
-				packs, err := luetCompiler.ComputeDepTree(sp)
+				ht := compiler.NewHashTree(generalRecipe.GetDatabase())
+				hashTree, err := ht.Query(luetCompiler, sp)
 				if err != nil {
 					errs = append(errs, err)
 				}
-				for _, p := range packs {
+				for _, p := range hashTree.Dependencies {
 					results.Packages = append(results.Packages,
 						PackageResult{
 							Name:       p.Package.GetName(),
@@ -264,6 +286,7 @@ var buildCmd = &cobra.Command{
 				}
 			}
 		} else {
+
 			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
 		}
 		if len(errs) != 0 {
@@ -273,7 +296,7 @@ var buildCmd = &cobra.Command{
 			Fatal("Bailing out")
 		}
 		for _, a := range artifact {
-			Info("Artifact generated:", a.GetPath())
+			Info("Artifact generated:", a.Path)
 		}
 	},
 }
@@ -283,33 +306,36 @@ func init() {
 	if err != nil {
 		Fatal(err)
 	}
-	buildCmd.Flags().Bool("clean", true, "Build all packages without considering the packages present in the build directory")
-	buildCmd.Flags().StringSliceP("tree", "t", []string{}, "Path of the tree to use.")
+
+	buildCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the tree to use.")
 	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
-	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
-	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
+	buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
+	buildCmd.Flags().StringSlice("values", []string{}, "Build values file to interpolate with each package")
+	buildCmd.Flags().StringSliceP("backend-args", "a", []string{}, "Backend args")
 
-	buildCmd.Flags().String("destination", path, "Destination folder")
-	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
+	buildCmd.Flags().String("destination", filepath.Join(path, "build"), "Destination folder")
+	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip, zstd")
 	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
 	buildCmd.Flags().Bool("push", false, "Push images to a hub")
 	buildCmd.Flags().Bool("pull", false, "Pull images from a hub")
+	buildCmd.Flags().Bool("wait", false, "Don't build all intermediate images, but wait for them until they are available")
 	buildCmd.Flags().Bool("keep-images", true, "Keep built docker images in the host")
 	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
 	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
-	buildCmd.Flags().Bool("keep-exported-images", false, "Keep exported images used during building")
-	buildCmd.Flags().Bool("skip-if-metadata-exists", false, "Skip package if metadata exists")
 	buildCmd.Flags().Bool("only-target-package", false, "Build packages of only the required target. Otherwise builds all the necessary ones not present in the destination")
 	buildCmd.Flags().String("solver-type", "", "Solver strategy")
 	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
-
+	buildCmd.Flags().Bool("live-output", LuetCfg.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
+	buildCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
+	buildCmd.Flags().Bool("rebuild", false, "To combine with --pull. Allows to rebuild the target package even if an image is available, against a local values file")
 	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
+	buildCmd.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")
 
 	buildCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 

cmd/cleanup.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
+	. "github.com/mudler/luet/pkg/config"
 	config "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -31,13 +32,24 @@ var cleanupCmd = &cobra.Command{
 	Use:   "cleanup",
 	Short: "Clean packages cache.",
 	Long:  `remove downloaded packages tarballs and clean cache directory`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var cleaned int = 0
+		dbpath := LuetCfg.Viper.GetString("system.database_path")
+		rootfs := config.LuetCfg.Viper.GetString("system.rootfs")
+		engine := config.LuetCfg.Viper.GetString("system.database_engine")
 
+		LuetCfg.System.DatabaseEngine = engine
+		LuetCfg.System.DatabasePath = dbpath
+		LuetCfg.System.Rootfs = rootfs
 		// Check if cache dir exists
-		if helpers.Exists(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
+		if helpers.Exists(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
 
-			files, err := ioutil.ReadDir(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
+			files, err := ioutil.ReadDir(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
 			if err != nil {
 				Fatal("Error on read cachedir ", err.Error())
 			}
@@ -47,12 +59,12 @@ var cleanupCmd = &cobra.Command{
 					continue
 				}
 
-				if config.LuetCfg.GetGeneral().Debug {
+				if LuetCfg.GetGeneral().Debug {
 					Info("Removing ", file.Name())
 				}
 
 				err := os.RemoveAll(
-					filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
+					filepath.Join(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
 				if err != nil {
 					Fatal("Error on removing", file.Name())
 				}
@@ -66,5 +78,8 @@ var cleanupCmd = &cobra.Command{
 }
 
 func init() {
+	cleanupCmd.Flags().String("system-dbpath", "", "System db path")
+	cleanupCmd.Flags().String("system-target", "", "System rootpath")
+	cleanupCmd.Flags().String("system-engine", "", "System DB engine")
 	RootCmd.AddCommand(cleanupCmd)
 }

cmd/create-repo.go

@@ -16,11 +16,15 @@ package cmd
 
 import (
 	"os"
+	"path/filepath"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
-	. "github.com/mudler/luet/pkg/logger"
+
+	//	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
@@ -30,11 +34,31 @@ import (
 var createrepoCmd = &cobra.Command{
 	Use:   "create-repo",
 	Short: "Create a luet repository from a build",
-	Long:  `Generate and renew repository metadata`,
+	Long: `Builds tree metadata from a set of packages and a tree definition:
+
+	$ luet create-repo
+
+Provide specific paths for packages, tree, and metadata output which is generated:
+
+	$ luet create-repo --packages my/packages/path --tree my/tree/path --output my/packages/path ...
+
+Provide name and description of the repository:
+
+	$ luet create-repo --name "foo" --description "bar" ...
+
+Change compression method:
+	
+	$ luet create-repo --tree-compression gzip --meta-compression gzip
+
+Create a repository from the metadata description defined in the luet.yaml config file:
+
+	$ luet create-repo --repo repository1
+`,
 	PreRun: func(cmd *cobra.Command, args []string) {
 		viper.BindPFlag("packages", cmd.Flags().Lookup("packages"))
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("output", cmd.Flags().Lookup("output"))
+		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
 		viper.BindPFlag("name", cmd.Flags().Lookup("name"))
 		viper.BindPFlag("descr", cmd.Flags().Lookup("descr"))
 		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
@@ -45,10 +69,14 @@ var createrepoCmd = &cobra.Command{
 		viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename"))
 		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
 		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
+
+		viper.BindPFlag("force-push", cmd.Flags().Lookup("force-push"))
+		viper.BindPFlag("push-images", cmd.Flags().Lookup("push-images"))
+
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var err error
-		var repo installer.Repository
+		var repo *installer.LuetSystemRepository
 
 		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("output")
@@ -63,16 +91,20 @@ var createrepoCmd = &cobra.Command{
 		metatype := viper.GetString("meta-compression")
 		metaName := viper.GetString("meta-filename")
 		source_repo := viper.GetString("repo")
+		backendType := viper.GetString("backend")
+		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
 
 		treeFile := installer.NewDefaultTreeRepositoryFile()
 		metaFile := installer.NewDefaultMetaRepositoryFile()
+		compilerBackend, err := compiler.NewBackend(backendType)
+		helpers.CheckErr(err)
+		force := viper.GetBool("force-push")
+		imagePush := viper.GetBool("push-images")
 
 		if source_repo != "" {
 			// Search for system repository
 			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
+			helpers.CheckErr(err)
 
 			if len(treePaths) <= 0 {
 				treePaths = []string{lrepo.TreePath}
@@ -88,19 +120,23 @@ var createrepoCmd = &cobra.Command{
 				lrepo.Priority,
 				packages,
 				treePaths,
-				pkg.NewInMemoryDatabase(false))
+				pkg.NewInMemoryDatabase(false),
+				compilerBackend,
+				dst,
+				imagePush,
+				force,
+				fromRepo,
+				LuetCfg)
+			helpers.CheckErr(err)
 
 		} else {
 			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
-				treePaths, pkg.NewInMemoryDatabase(false))
-		}
-
-		if err != nil {
-			Fatal("Error: " + err.Error())
+				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force, fromRepo, LuetCfg)
+			helpers.CheckErr(err)
 		}
 
 		if treetype != "" {
-			treeFile.SetCompressionType(compiler.CompressionImplementation(treetype))
+			treeFile.SetCompressionType(compression.Implementation(treetype))
 		}
 
 		if treeName != "" {
@@ -108,7 +144,7 @@ var createrepoCmd = &cobra.Command{
 		}
 
 		if metatype != "" {
-			metaFile.SetCompressionType(compiler.CompressionImplementation(metatype))
+			metaFile.SetCompressionType(compression.Implementation(metatype))
 		}
 
 		if metaName != "" {
@@ -118,32 +154,35 @@ var createrepoCmd = &cobra.Command{
 		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
-		err = repo.Write(dst, reset)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
+		err = repo.Write(dst, reset, true)
+		helpers.CheckErr(err)
+
 	},
 }
 
 func init() {
 	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	createrepoCmd.Flags().String("packages", path, "Packages folder (output from build)")
-	createrepoCmd.Flags().StringSliceP("tree", "t", []string{}, "Path of the source trees to use.")
-	createrepoCmd.Flags().String("output", path, "Destination folder")
+	helpers.CheckErr(err)
+
+	createrepoCmd.Flags().String("packages", filepath.Join(path, "build"), "Packages folder (output from build)")
+	createrepoCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the source trees to use.")
+	createrepoCmd.Flags().String("output", filepath.Join(path, "build"), "Destination for generated archives. With 'docker' repository type, it should be an image reference (e.g 'foo/bar')")
 	createrepoCmd.Flags().String("name", "luet", "Repository name")
 	createrepoCmd.Flags().String("descr", "luet", "Repository description")
 	createrepoCmd.Flags().StringSlice("urls", []string{}, "Repository URLs")
-	createrepoCmd.Flags().String("type", "disk", "Repository type (disk)")
+	createrepoCmd.Flags().String("type", "disk", "Repository type (disk, http, docker)")
 	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
 	createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
+	createrepoCmd.Flags().String("backend", "docker", "backend used (docker,img)")
 
-	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip")
+	createrepoCmd.Flags().Bool("force-push", false, "Force overwrite of docker images if already present online")
+	createrepoCmd.Flags().Bool("push-images", false, "Enable/Disable docker image push for docker repositories")
+
+	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
-	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
+	createrepoCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

cmd/database.go

@@ -25,6 +25,10 @@ import (
 var databaseGroupCmd = &cobra.Command{
 	Use:   "database [command] [OPTIONS]",
 	Short: "Manage system database (dangerous commands ahead!)",
+	Long: `Allows to manipulate Luet internal database of installed packages. Use with caution!
+
+Removing packages by hand from the database can result in a broken system, and thus it's not reccomended.
+`,
 }
 
 func init() {
@@ -32,6 +36,7 @@ func init() {
 
 	databaseGroupCmd.AddCommand(
 		NewDatabaseCreateCommand(),
+		NewDatabaseGetCommand(),
 		NewDatabaseRemoveCommand(),
 	)
 }

cmd/database/create.go

@@ -17,9 +17,9 @@ package cmd_database
 
 import (
 	"io/ioutil"
-	"path/filepath"
 
-	"github.com/mudler/luet/pkg/compiler"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -32,47 +32,64 @@ func NewDatabaseCreateCommand() *cobra.Command {
 	var ans = &cobra.Command{
 		Use:   "create <artifact_metadata1.yaml> <artifact_metadata1.yaml>",
 		Short: "Insert a package in the system DB",
-		Args:  cobra.OnlyValidArgs,
+		Long: `Inserts a package in the system database:
+
+		$ luet database create foo.yaml
+
+"luet database create" injects a package in the system database without actually installing it, use it with caution.
+
+This commands takes multiple yaml input file representing package artifacts, that are usually generated while building packages.
+
+The yaml must contain the package definition, and the file list at least.
+
+For reference, inspect a "metadata.yaml" file generated while running "luet build"`,
+		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
 			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
 			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 
-			var systemDB pkg.PackageDatabase
+			dbpath := LuetCfg.Viper.GetString("system.database_path")
+			rootfs := LuetCfg.Viper.GetString("system.rootfs")
+			engine := LuetCfg.Viper.GetString("system.database_engine")
+
+			LuetCfg.System.DatabaseEngine = engine
+			LuetCfg.System.DatabasePath = dbpath
+			LuetCfg.System.Rootfs = rootfs
+
+			systemDB := LuetCfg.GetSystemDB()
 
 			for _, a := range args {
 				dat, err := ioutil.ReadFile(a)
 				if err != nil {
 					Fatal("Failed reading ", a, ": ", err.Error())
 				}
-				art, err := compiler.NewPackageArtifactFromYaml(dat)
+				art, err := artifact.NewPackageArtifactFromYaml(dat)
 				if err != nil {
 					Fatal("Failed reading yaml ", a, ": ", err.Error())
 				}
 
-				if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-					systemDB = pkg.NewBoltDatabase(
-						filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-				} else {
-					systemDB = pkg.NewInMemoryDatabase(true)
-				}
+				files := art.Files
 
-				files := art.GetFiles()
-
-				if _, err := systemDB.CreatePackage(art.GetCompileSpec().GetPackage()); err != nil {
+				if _, err := systemDB.CreatePackage(art.CompileSpec.GetPackage()); err != nil {
 					Fatal("Failed to create ", a, ": ", err.Error())
 				}
-				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.GetCompileSpec().GetPackage().GetFingerPrint(), Files: files}); err != nil {
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.CompileSpec.GetPackage().GetFingerPrint(), Files: files}); err != nil {
 					Fatal("Failed setting package files for ", a, ": ", err.Error())
 				}
 
-				Info(art.GetCompileSpec().GetPackage().HumanReadableString(), " created")
+				Info(art.CompileSpec.GetPackage().HumanReadableString(), " created")
 			}
 
 		},
 	}
 
+	ans.Flags().String("system-dbpath", "", "System db path")
+	ans.Flags().String("system-target", "", "System rootpath")
+	ans.Flags().String("system-engine", "", "System DB engine")
+
 	return ans
 }

cmd/database/get.go

@@ -0,0 +1,95 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"fmt"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"gopkg.in/yaml.v2"
+
+	. "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseGetCommand() *cobra.Command {
+	var c = &cobra.Command{
+		Use:   "get <package>",
+		Short: "Get a package in the system DB as yaml",
+		Long: `Get a package in the system database in the YAML format:
+
+		$ luet database get system/foo
+
+To return also files:
+		$ luet database get --files system/foo`,
+		Args: cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			showFiles, _ := cmd.Flags().GetBool("files")
+			dbpath := LuetCfg.Viper.GetString("system.database_path")
+			rootfs := LuetCfg.Viper.GetString("system.rootfs")
+			engine := LuetCfg.Viper.GetString("system.database_engine")
+
+			LuetCfg.System.DatabaseEngine = engine
+			LuetCfg.System.DatabasePath = dbpath
+			LuetCfg.System.Rootfs = rootfs
+
+			systemDB := LuetCfg.GetSystemDB()
+
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					continue
+				}
+
+				ps, err := systemDB.FindPackages(pack)
+				if err != nil {
+					continue
+				}
+				for _, p := range ps {
+					y, err := p.Yaml()
+					if err != nil {
+						continue
+					}
+					fmt.Println(string(y))
+					if showFiles {
+						files, err := systemDB.GetPackageFiles(p)
+						if err != nil {
+							continue
+						}
+						b, err := yaml.Marshal(files)
+						if err != nil {
+							continue
+						}
+						fmt.Println("files:\n" + string(b))
+					}
+				}
+			}
+		},
+	}
+	c.Flags().Bool("files", false, "Show package files.")
+	c.Flags().String("system-dbpath", "", "System db path")
+	c.Flags().String("system-target", "", "System rootpath")
+	c.Flags().String("system-engine", "", "System DB engine")
+
+	return c
+}

cmd/database/remove.go

@@ -16,13 +16,10 @@
 package cmd_database
 
 import (
-	"path/filepath"
-
 	. "github.com/mudler/luet/pkg/logger"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
 )
@@ -31,14 +28,29 @@ func NewDatabaseRemoveCommand() *cobra.Command {
 	var ans = &cobra.Command{
 		Use:   "remove [package1] [package2] ...",
 		Short: "Remove a package from the system DB (forcefully - you normally don't want to do that)",
-		Args:  cobra.OnlyValidArgs,
+		Long: `Removes a package in the system database without actually uninstalling it:
+
+		$ luet database remove foo/bar
+
+This commands takes multiple packages as arguments and prunes their entries from the system database.
+`,
+		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
 			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
 			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			var systemDB pkg.PackageDatabase
+			dbpath := LuetCfg.Viper.GetString("system.database_path")
+			rootfs := LuetCfg.Viper.GetString("system.rootfs")
+			engine := LuetCfg.Viper.GetString("system.database_engine")
+
+			LuetCfg.System.DatabaseEngine = engine
+			LuetCfg.System.DatabasePath = dbpath
+			LuetCfg.System.Rootfs = rootfs
+
+			systemDB := LuetCfg.GetSystemDB()
 
 			for _, a := range args {
 				pack, err := helpers.ParsePackageStr(a)
@@ -46,13 +58,6 @@ func NewDatabaseRemoveCommand() *cobra.Command {
 					Fatal("Invalid package string ", a, ": ", err.Error())
 				}
 
-				if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-					systemDB = pkg.NewBoltDatabase(
-						filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-				} else {
-					systemDB = pkg.NewInMemoryDatabase(true)
-				}
-
 				if err := systemDB.RemovePackage(pack); err != nil {
 					Fatal("Failed removing ", a, ": ", err.Error())
 				}
@@ -64,6 +69,9 @@ func NewDatabaseRemoveCommand() *cobra.Command {
 
 		},
 	}
+	ans.Flags().String("system-dbpath", "", "System db path")
+	ans.Flags().String("system-target", "", "System rootpath")
+	ans.Flags().String("system-engine", "", "System DB engine")
 
 	return ans
 }

cmd/helpers/cli.go

@@ -20,6 +20,9 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+	"strings"
+
+	. "github.com/mudler/luet/pkg/logger"
 
 	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -41,7 +44,42 @@ func CreateRegexArray(rgx []string) ([]*regexp.Regexp, error) {
 	return ans, nil
 }
 
+func packageData(p string) (string, string) {
+	cat := ""
+	name := ""
+	if strings.Contains(p, "/") {
+		packagedata := strings.Split(p, "/")
+		cat = packagedata[0]
+		name = packagedata[1]
+	} else {
+		name = p
+	}
+	return cat, name
+}
 func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
+
+	if !(strings.HasPrefix(p, "=") || strings.HasPrefix(p, ">") ||
+		strings.HasPrefix(p, "<")) {
+		ver := ">=0"
+		cat := ""
+		name := ""
+
+		if strings.Contains(p, "@") {
+			packageinfo := strings.Split(p, "@")
+			ver = packageinfo[1]
+			cat, name = packageData(packageinfo[0])
+		} else {
+			cat, name = packageData(p)
+		}
+
+		return &pkg.DefaultPackage{
+			Name:     name,
+			Category: cat,
+			Version:  ver,
+			Uri:      make([]string, 0),
+		}, nil
+	}
+
 	gp, err := _gentoo.ParsePackageStr(p)
 	if err != nil {
 		return nil, err
@@ -76,3 +114,9 @@ func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
 
 	return pack, nil
 }
+
+func CheckErr(err error) {
+	if err != nil {
+		Fatal(err)
+	}
+}

cmd/helpers/cli_suite_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_helpers_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "CLI helpers test Suite")
+}

cmd/helpers/cli_test.go

@@ -0,0 +1,102 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_helpers_test
+
+import (
+	. "github.com/mudler/luet/cmd/helpers"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("CLI Helpers", func() {
+	Context("Can parse package strings correctly", func() {
+		It("accept single package names", func() {
+			pack, err := ParsePackageStr("foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal(""))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept unversioned packages with category", func() {
+			pack, err := ParsePackageStr("cat/foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept versioned packages with category", func() {
+			pack, err := ParsePackageStr("cat/foo@1.1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("1.1"))
+		})
+		It("accept versioned ranges with category", func() {
+			pack, err := ParsePackageStr("cat/foo@>=1.1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=1.1"))
+		})
+		It("accept gentoo regex parsing without versions", func() {
+			pack, err := ParsePackageStr("=cat/foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept gentoo regex parsing with versions", func() {
+			pack, err := ParsePackageStr("=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition", func() {
+			pack, err := ParsePackageStr(">=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition2", func() {
+			pack, err := ParsePackageStr("<cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("<1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition3", func() {
+			pack, err := ParsePackageStr(">cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition4", func() {
+			pack, err := ParsePackageStr("<=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("<=1.2"))
+		})
+	})
+})

cmd/install.go

@@ -15,9 +15,6 @@
 package cmd
 
 import (
-	"os"
-	"path/filepath"
-
 	installer "github.com/mudler/luet/pkg/installer"
 	"github.com/mudler/luet/pkg/solver"
 
@@ -30,12 +27,29 @@ import (
 )
 
 var installCmd = &cobra.Command{
-	Use:     "install <pkg1> <pkg2> ...",
-	Short:   "Install a package",
+	Use:   "install <pkg1> <pkg2> ...",
+	Short: "Install a package",
+	Long: `Installs one or more packages without asking questions:
+
+	$ luet install -y utils/busybox utils/yq ...
+	
+To install only deps of a package:
+	
+	$ luet install --onlydeps utils/busybox ...
+	
+To not install deps of a package:
+	
+	$ luet install --nodeps utils/busybox ...
+
+To force install a package:
+	
+	$ luet install --force utils/busybox ...
+`,
 	Aliases: []string{"i"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
 		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
 		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
 		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
@@ -45,10 +59,8 @@ var installCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
-	Long: `Install packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
 		var toInstall pkg.Packages
-		var systemDB pkg.PackageDatabase
 
 		for _, a := range args {
 			pack, err := helpers.ParsePackageStr(a)
@@ -58,16 +70,6 @@ var installCmd = &cobra.Command{
 			toInstall = append(toInstall, pack)
 		}
 
-		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
-		repos := installer.Repositories{}
-		for _, repo := range LuetCfg.SystemRepositories {
-			if !repo.Enable {
-				continue
-			}
-			r := installer.NewSystemRepository(repo)
-			repos = append(repos, r)
-		}
-
 		stype := LuetCfg.Viper.GetString("solver.type")
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
@@ -77,6 +79,15 @@ var installCmd = &cobra.Command{
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		dbpath := LuetCfg.Viper.GetString("system.database_path")
+		rootfs := LuetCfg.Viper.GetString("system.rootfs")
+		engine := LuetCfg.Viper.GetString("system.database_engine")
+
+		LuetCfg.System.DatabaseEngine = engine
+		LuetCfg.System.DatabasePath = dbpath
+		LuetCfg.System.Rootfs = rootfs
 
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
@@ -90,6 +101,7 @@ var installCmd = &cobra.Command{
 		}
 
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		repos := installer.SystemRepositories(LuetCfg)
 
 		// Load config protect configs
 		installer.LoadConfigProtectConfs(LuetCfg)
@@ -101,17 +113,12 @@ var installCmd = &cobra.Command{
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
 			PreserveSystemEssentialData: true,
+			DownloadOnly:                downloadOnly,
 			Ask:                         !yes,
 		})
 		inst.Repositories(repos)
 
-		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-		} else {
-			systemDB = pkg.NewInMemoryDatabase(true)
-		}
-		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
 		err := inst.Install(toInstall, system)
 		if err != nil {
 			Fatal("Error: " + err.Error())
@@ -120,12 +127,10 @@ var installCmd = &cobra.Command{
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	installCmd.Flags().String("system-dbpath", path, "System db path")
-	installCmd.Flags().String("system-target", path, "System rootpath")
+	installCmd.Flags().String("system-dbpath", "", "System db path")
+	installCmd.Flags().String("system-target", "", "System rootpath")
+	installCmd.Flags().String("system-engine", "", "System DB engine")
+
 	installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
 	installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
@@ -135,6 +140,7 @@ func init() {
 	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
 	installCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	installCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	installCmd.Flags().Bool("download-only", false, "Download only")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/pack.go

@@ -20,7 +20,9 @@ import (
 	"time"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
-	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -31,7 +33,16 @@ import (
 var packCmd = &cobra.Command{
 	Use:   "pack <package name>",
 	Short: "pack a custom package",
-	Long:  `pack and creates metadata directly from a source path`,
+	Long: `Pack creates a package from a directory, generating the metadata required from a tree to generate a repository.
+
+Pack can be used to manually replace what "luet build" does automatically by reading the packages build.yaml files.
+
+	$ mkdir -p output/etc/foo
+	$ echo "my config" > output/etc/foo
+	$ luet pack foo/bar@1.1 --source output
+
+Afterwards, you can use the content generated and associate it with a tree and a corresponding definition.yaml file with "luet create-repo".
+`,
 	PreRun: func(cmd *cobra.Command, args []string) {
 		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
@@ -55,21 +66,21 @@ var packCmd = &cobra.Command{
 			Fatal("Invalid package string ", packageName, ": ", err.Error())
 		}
 
-		spec := &compiler.LuetCompilationSpec{Package: p}
-		artifact := compiler.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
-		artifact.SetCompressionType(compiler.CompressionImplementation(compressionType))
-		err = artifact.Compress(sourcePath, concurrency)
+		spec := &compilerspec.LuetCompilationSpec{Package: p}
+		a := artifact.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
+		a.CompressionType = compression.Implementation(compressionType)
+		err = a.Compress(sourcePath, concurrency)
 		if err != nil {
 			Fatal("failed compressing ", packageName, ": ", err.Error())
 		}
-		artifact.SetCompileSpec(spec)
-		filelist, err := artifact.FileList()
+		a.CompileSpec = spec
+		filelist, err := a.FileList()
 		if err != nil {
 			Fatal("failed generating file list for ", packageName, ": ", err.Error())
 		}
-		artifact.SetFiles(filelist)
-		artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
-		err = artifact.WriteYaml(dst)
+		a.Files = filelist
+		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
+		err = a.WriteYaml(dst)
 		if err != nil {
 			Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
 		}

cmd/reclaim.go

@@ -15,14 +15,10 @@
 package cmd
 
 import (
-	"os"
-	"path/filepath"
-
 	installer "github.com/mudler/luet/pkg/installer"
 
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
-	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
 )
@@ -33,12 +29,23 @@ var reclaimCmd = &cobra.Command{
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
 		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
-	Long: `Add packages to the systemdb if files belonging to packages 
-in available repositories exists in the target root.`,
+	Long: `Reclaim tries to find association between packages in the online repositories and the system one.
+
+	$ luet reclaim
+
+It scans the target file system, and if finds a match with a package available in the repositories, it marks as installed in the system database.
+`,
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
+		dbpath := LuetCfg.Viper.GetString("system.database_path")
+		rootfs := LuetCfg.Viper.GetString("system.rootfs")
+		engine := LuetCfg.Viper.GetString("system.database_engine")
+
+		LuetCfg.System.DatabaseEngine = engine
+		LuetCfg.System.DatabasePath = dbpath
+		LuetCfg.System.Rootfs = rootfs
 
 		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
 		repos := installer.Repositories{}
@@ -61,13 +68,7 @@ in available repositories exists in the target root.`,
 		})
 		inst.Repositories(repos)
 
-		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-		} else {
-			systemDB = pkg.NewInMemoryDatabase(true)
-		}
-		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
 		err := inst.Reclaim(system)
 		if err != nil {
 			Fatal("Error: " + err.Error())
@@ -76,12 +77,11 @@ in available repositories exists in the target root.`,
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	reclaimCmd.Flags().String("system-dbpath", path, "System db path")
-	reclaimCmd.Flags().String("system-target", path, "System rootpath")
+
+	reclaimCmd.Flags().String("system-dbpath", "", "System db path")
+	reclaimCmd.Flags().String("system-target", "", "System rootpath")
+	reclaimCmd.Flags().String("system-engine", "", "System DB engine")
+
 	reclaimCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
 
 	RootCmd.AddCommand(reclaimCmd)

cmd/replace.go

@@ -0,0 +1,156 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+var replaceCmd = &cobra.Command{
+	Use:     "replace <pkg1> <pkg2> --for <pkg3> --for <pkg4> ...",
+	Short:   "replace a set of packages",
+	Aliases: []string{"r"},
+	Long: `Replaces one or a group of packages without asking questions:
+
+	$ luet replace -y system/busybox ... --for shells/bash --for system/coreutils ...
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("for", cmd.Flags().Lookup("for"))
+
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var toUninstall pkg.Packages
+		var toAdd pkg.Packages
+
+		f := LuetCfg.Viper.GetStringSlice("for")
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps := LuetCfg.Viper.GetBool("nodeps")
+		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
+		dbpath := LuetCfg.Viper.GetString("system.database_path")
+		rootfs := LuetCfg.Viper.GetString("system.rootfs")
+		engine := LuetCfg.Viper.GetString("system.database_engine")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		LuetCfg.System.DatabaseEngine = engine
+		LuetCfg.System.DatabasePath = dbpath
+		LuetCfg.System.Rootfs = rootfs
+
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toUninstall = append(toUninstall, pack)
+		}
+
+		for _, a := range f {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toAdd = append(toAdd, pack)
+		}
+
+		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
+		})
+		inst.Repositories(repos)
+
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Swap(toUninstall, toAdd, system)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+
+	replaceCmd.Flags().String("system-dbpath", "", "System db path")
+	replaceCmd.Flags().String("system-target", "", "System rootpath")
+	replaceCmd.Flags().String("system-engine", "", "System DB engine")
+
+	replaceCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	replaceCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	replaceCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	replaceCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	replaceCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	replaceCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	replaceCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	replaceCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	replaceCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	replaceCmd.Flags().StringSlice("for", []string{}, "Packages that has to be installed in place of others")
+	replaceCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(replaceCmd)
+}

cmd/repo/list.go

@@ -72,8 +72,8 @@ func NewRepoListCommand() *cobra.Command {
 					if repo.Cached {
 
 						r := installer.NewSystemRepository(repo)
-						localRepo, _ := r.(*installer.LuetSystemRepository).ReadSpecFile(filepath.Join(repobasedir,
-							installer.REPOSITORY_SPECFILE), false)
+						localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir,
+							installer.REPOSITORY_SPECFILE))
 						if localRepo != nil {
 							tsec, _ := strconv.ParseInt(localRepo.GetLastUpdate(), 10, 64)
 							repoRevision = Bold(Red(localRepo.GetRevision())).String() +

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.9.7"
+	LuetCLIVersion = "0.14.7"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -55,12 +55,12 @@ var (
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
 	Use:   "luet",
-	Short: "Package manager for the XXth century!",
+	Short: "Container based package manager",
 	Long: `Luet is a single-binary package manager based on containers to build packages.
 	
 To install a package:
 
-$ luet install package
+	$ luet install package
 
 To search for a package in the repositories:
 
@@ -68,15 +68,15 @@ $ luet search package
 
 To list all packages installed in the system:
 
-$ luet search --installed .
+	$ luet search --installed .
 
 To show hidden packages:
 
-$ luet search --hidden package
+	$ luet search --hidden package
 
 To build a package, from a tree definition:
 
-$ luet build --tree tree/path package
+	$ luet build --tree tree/path package
 	
 `,
 	Version: fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime),
@@ -97,7 +97,7 @@ $ luet build --tree tree/path package
 
 		plugin := viper.GetStringSlice("plugin")
 
-		bus.Manager.Load(plugin...).Register()
+		bus.Manager.Initialize(plugin...)
 		if len(bus.Manager.Plugins) != 0 {
 			Info(":lollipop:Enabled plugins:")
 			for _, p := range bus.Manager.Plugins {

cmd/search.go

@@ -16,8 +16,6 @@ package cmd
 
 import (
 	"fmt"
-	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/ghodss/yaml"
@@ -31,12 +29,13 @@ import (
 )
 
 type PackageResult struct {
-	Name       string `json:"name"`
-	Category   string `json:"category"`
-	Version    string `json:"version"`
-	Repository string `json:"repository"`
-	Target     string `json:"target"`
-	Hidden     bool   `json:"hidden"`
+	Name       string   `json:"name"`
+	Category   string   `json:"category"`
+	Version    string   `json:"version"`
+	Repository string   `json:"repository"`
+	Target     string   `json:"target"`
+	Hidden     bool     `json:"hidden"`
+	Files      []string `json:"files"`
 }
 
 type Results struct {
@@ -65,10 +64,246 @@ func packageToList(l list.Writer, repo string, p pkg.Package) {
 	l.UnIndent()
 }
 
+func searchLocally(term string, l list.Writer, t table.Writer, label, labelMatch, revdeps, hidden bool) Results {
+	var results Results
+
+	system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+
+	var err error
+	iMatches := pkg.Packages{}
+	if label {
+		iMatches, err = system.Database.FindPackageLabel(term)
+	} else if labelMatch {
+		iMatches, err = system.Database.FindPackageLabelMatch(term)
+	} else {
+		iMatches, err = system.Database.FindPackageMatch(term)
+	}
+
+	if err != nil {
+		Fatal("Error: " + err.Error())
+	}
+
+	for _, pack := range iMatches {
+		if !revdeps {
+			if !pack.IsHidden() || pack.IsHidden() && hidden {
+
+				t.AppendRow(packageToRow("system", pack))
+				packageToList(l, "system", pack)
+				f, _ := system.Database.GetPackageFiles(pack)
+				results.Packages = append(results.Packages,
+					PackageResult{
+						Name:       pack.GetName(),
+						Version:    pack.GetVersion(),
+						Category:   pack.GetCategory(),
+						Repository: "system",
+						Hidden:     pack.IsHidden(),
+						Files:      f,
+					})
+			}
+		} else {
+
+			packs, _ := system.Database.GetRevdeps(pack)
+			for _, revdep := range packs {
+				if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+					t.AppendRow(packageToRow("system", pack))
+					packageToList(l, "system", pack)
+					f, _ := system.Database.GetPackageFiles(revdep)
+					results.Packages = append(results.Packages,
+						PackageResult{
+							Name:       revdep.GetName(),
+							Version:    revdep.GetVersion(),
+							Category:   revdep.GetCategory(),
+							Repository: "system",
+							Hidden:     revdep.IsHidden(),
+							Files:      f,
+						})
+				}
+			}
+		}
+	}
+	return results
+
+}
+func searchOnline(term string, l list.Writer, t table.Writer, label, labelMatch, revdeps, hidden bool) Results {
+	var results Results
+
+	repos := installer.Repositories{}
+	for _, repo := range LuetCfg.SystemRepositories {
+		if !repo.Enable {
+			continue
+		}
+		r := installer.NewSystemRepository(repo)
+		repos = append(repos, r)
+	}
+
+	inst := installer.NewLuetInstaller(
+		installer.LuetInstallerOptions{
+			Concurrency:   LuetCfg.GetGeneral().Concurrency,
+			SolverOptions: *LuetCfg.GetSolverOptions(),
+		},
+	)
+	inst.Repositories(repos)
+	synced, err := inst.SyncRepositories(false)
+	if err != nil {
+		Fatal("Error: " + err.Error())
+	}
+
+	Info("--- Search results (" + term + "): ---")
+
+	matches := []installer.PackageMatch{}
+	if label {
+		matches = synced.SearchLabel(term)
+	} else if labelMatch {
+		matches = synced.SearchLabelMatch(term)
+	} else {
+		matches = synced.Search(term)
+	}
+
+	for _, m := range matches {
+		if !revdeps {
+			if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
+				t.AppendRow(packageToRow(m.Repo.GetName(), m.Package))
+				packageToList(l, m.Repo.GetName(), m.Package)
+				r := &PackageResult{
+					Name:       m.Package.GetName(),
+					Version:    m.Package.GetVersion(),
+					Category:   m.Package.GetCategory(),
+					Repository: m.Repo.GetName(),
+					Hidden:     m.Package.IsHidden(),
+				}
+				if m.Artifact != nil {
+					r.Files = m.Artifact.Files
+				}
+				results.Packages = append(results.Packages, *r)
+			}
+		} else {
+			packs, _ := m.Repo.GetTree().GetDatabase().GetRevdeps(m.Package)
+			for _, revdep := range packs {
+				if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+					t.AppendRow(packageToRow(m.Repo.GetName(), revdep))
+					packageToList(l, m.Repo.GetName(), revdep)
+					r := &PackageResult{
+						Name:       revdep.GetName(),
+						Version:    revdep.GetVersion(),
+						Category:   revdep.GetCategory(),
+						Repository: m.Repo.GetName(),
+						Hidden:     revdep.IsHidden(),
+					}
+					if m.Artifact != nil {
+						r.Files = m.Artifact.Files
+					}
+					results.Packages = append(results.Packages, *r)
+				}
+			}
+		}
+	}
+	return results
+}
+func searchLocalFiles(term string, l list.Writer, t table.Writer) Results {
+	var results Results
+	Info("--- Search results (" + term + "): ---")
+
+	matches, _ := LuetCfg.GetSystemDB().FindPackageByFile(term)
+	for _, pack := range matches {
+		t.AppendRow(packageToRow("system", pack))
+		packageToList(l, "system", pack)
+		f, _ := LuetCfg.GetSystemDB().GetPackageFiles(pack)
+		results.Packages = append(results.Packages,
+			PackageResult{
+				Name:       pack.GetName(),
+				Version:    pack.GetVersion(),
+				Category:   pack.GetCategory(),
+				Repository: "system",
+				Hidden:     pack.IsHidden(),
+				Files:      f,
+			})
+	}
+
+	return results
+}
+
+func searchFiles(term string, l list.Writer, t table.Writer) Results {
+	var results Results
+
+	repos := installer.Repositories{}
+	for _, repo := range LuetCfg.SystemRepositories {
+		if !repo.Enable {
+			continue
+		}
+		r := installer.NewSystemRepository(repo)
+		repos = append(repos, r)
+	}
+
+	inst := installer.NewLuetInstaller(
+		installer.LuetInstallerOptions{
+			Concurrency:   LuetCfg.GetGeneral().Concurrency,
+			SolverOptions: *LuetCfg.GetSolverOptions(),
+		},
+	)
+	inst.Repositories(repos)
+	synced, err := inst.SyncRepositories(false)
+	if err != nil {
+		Fatal("Error: " + err.Error())
+	}
+
+	Info("--- Search results (" + term + "): ---")
+
+	matches := []installer.PackageMatch{}
+
+	matches = synced.SearchPackages(term, installer.FileSearch)
+
+	for _, m := range matches {
+		t.AppendRow(packageToRow(m.Repo.GetName(), m.Package))
+		packageToList(l, m.Repo.GetName(), m.Package)
+		results.Packages = append(results.Packages,
+			PackageResult{
+				Name:       m.Package.GetName(),
+				Version:    m.Package.GetVersion(),
+				Category:   m.Package.GetCategory(),
+				Repository: m.Repo.GetName(),
+				Hidden:     m.Package.IsHidden(),
+				Files:      m.Artifact.Files,
+			})
+	}
+	return results
+}
+
 var searchCmd = &cobra.Command{
-	Use:     "search <term>",
-	Short:   "Search packages",
-	Long:    `Search for installed and available packages`,
+	Use:   "search <term>",
+	Short: "Search packages",
+	Long: `Search for installed and available packages
+	
+To search a package in the repositories:
+
+	$ luet search <regex>
+
+To search a package and display results in a table (wide screens):
+
+	$ luet search --table <regex>
+
+To look into the installed packages:
+
+	$ luet search --installed <regex>
+
+Note: the regex argument is optional, if omitted implies "all"
+
+To search a package by label:
+
+	$ luet search --by-label <label>
+
+or by regex against the label:
+
+	$ luet search --by-label-regex <label>
+
+It can also show a package revdeps by:
+
+	$ luet search --revdeps <regex>
+
+Search can also return results in the terminal in different ways: as terminal output, as json or as yaml.
+
+	$ luet search --json <regex> # JSON output
+	$ luet search --yaml <regex> # YAML output
+`,
 	Aliases: []string{"s"},
 	PreRun: func(cmd *cobra.Command, args []string) {
 		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
@@ -76,11 +311,11 @@ var searchCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
 		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
 		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
 		var results Results
 		if len(args) > 1 {
 			Fatal("Wrong number of arguments (expected 1)")
@@ -98,7 +333,14 @@ var searchCmd = &cobra.Command{
 		searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex")
 		revdeps, _ := cmd.Flags().GetBool("revdeps")
 		tableMode, _ := cmd.Flags().GetBool("table")
+		files, _ := cmd.Flags().GetBool("files")
+		dbpath := LuetCfg.Viper.GetString("system.database_path")
+		rootfs := LuetCfg.Viper.GetString("system.rootfs")
+		engine := LuetCfg.Viper.GetString("system.database_engine")
 
+		LuetCfg.System.DatabaseEngine = engine
+		LuetCfg.System.DatabasePath = dbpath
+		LuetCfg.System.Rootfs = rootfs
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
 			LuetCfg.GetLogging().SetLogLevel("error")
@@ -114,127 +356,15 @@ var searchCmd = &cobra.Command{
 		t.AppendHeader(rows)
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
-		if !installed {
-
-			repos := installer.Repositories{}
-			for _, repo := range LuetCfg.SystemRepositories {
-				if !repo.Enable {
-					continue
-				}
-				r := installer.NewSystemRepository(repo)
-				repos = append(repos, r)
-			}
-
-			inst := installer.NewLuetInstaller(
-				installer.LuetInstallerOptions{
-					Concurrency:   LuetCfg.GetGeneral().Concurrency,
-					SolverOptions: *LuetCfg.GetSolverOptions(),
-				},
-			)
-			inst.Repositories(repos)
-			synced, err := inst.SyncRepositories(false)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-
-			Info("--- Search results (" + args[0] + "): ---")
-
-			matches := []installer.PackageMatch{}
-			if searchWithLabel {
-				matches = synced.SearchLabel(args[0])
-			} else if searchWithLabelMatch {
-				matches = synced.SearchLabelMatch(args[0])
-			} else {
-				matches = synced.Search(args[0])
-			}
-			for _, m := range matches {
-				if !revdeps {
-					if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
-						t.AppendRow(packageToRow(m.Repo.GetName(), m.Package))
-						packageToList(l, m.Repo.GetName(), m.Package)
-						results.Packages = append(results.Packages,
-							PackageResult{
-								Name:       m.Package.GetName(),
-								Version:    m.Package.GetVersion(),
-								Category:   m.Package.GetCategory(),
-								Repository: m.Repo.GetName(),
-								Hidden:     m.Package.IsHidden(),
-							})
-					}
-				} else {
-					packs, _ := m.Repo.GetTree().GetDatabase().GetRevdeps(m.Package)
-					for _, revdep := range packs {
-						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
-							t.AppendRow(packageToRow(m.Repo.GetName(), revdep))
-							packageToList(l, m.Repo.GetName(), revdep)
-							results.Packages = append(results.Packages,
-								PackageResult{
-									Name:       revdep.GetName(),
-									Version:    revdep.GetVersion(),
-									Category:   revdep.GetCategory(),
-									Repository: m.Repo.GetName(),
-									Hidden:     revdep.IsHidden(),
-								})
-						}
-					}
-				}
-			}
-		} else {
-
-			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-				systemDB = pkg.NewBoltDatabase(
-					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-			} else {
-				systemDB = pkg.NewInMemoryDatabase(true)
-			}
-			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
-
-			var err error
-			iMatches := pkg.Packages{}
-			if searchWithLabel {
-				iMatches, err = system.Database.FindPackageLabel(args[0])
-			} else if searchWithLabelMatch {
-				iMatches, err = system.Database.FindPackageLabelMatch(args[0])
-			} else {
-				iMatches, err = system.Database.FindPackageMatch(args[0])
-			}
-
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-
-			for _, pack := range iMatches {
-				if !revdeps {
-					if !pack.IsHidden() || pack.IsHidden() && hidden {
-						t.AppendRow(packageToRow("system", pack))
-						packageToList(l, "system", pack)
-						results.Packages = append(results.Packages,
-							PackageResult{
-								Name:       pack.GetName(),
-								Version:    pack.GetVersion(),
-								Category:   pack.GetCategory(),
-								Repository: "system",
-								Hidden:     pack.IsHidden(),
-							})
-					}
-				} else {
-					packs, _ := system.Database.GetRevdeps(pack)
-					for _, revdep := range packs {
-						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
-							t.AppendRow(packageToRow("system", pack))
-							packageToList(l, "system", pack)
-							results.Packages = append(results.Packages,
-								PackageResult{
-									Name:       revdep.GetName(),
-									Version:    revdep.GetVersion(),
-									Category:   revdep.GetCategory(),
-									Repository: "system",
-									Hidden:     revdep.IsHidden(),
-								})
-						}
-					}
-				}
-			}
+		switch {
+		case files && installed:
+			results = searchLocalFiles(args[0], l, t)
+		case files && !installed:
+			results = searchFiles(args[0], l, t)
+		case !installed:
+			results = searchOnline(args[0], l, t, searchWithLabel, searchWithLabelMatch, revdeps, hidden)
+		default:
+			results = searchLocally(args[0], l, t, searchWithLabel, searchWithLabelMatch, revdeps, hidden)
 		}
 
 		t.AppendFooter(rows)
@@ -268,12 +398,10 @@ var searchCmd = &cobra.Command{
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	searchCmd.Flags().String("system-dbpath", path, "System db path")
-	searchCmd.Flags().String("system-target", path, "System rootpath")
+	searchCmd.Flags().String("system-dbpath", "", "System db path")
+	searchCmd.Flags().String("system-target", "", "System rootpath")
+	searchCmd.Flags().String("system-engine", "", "System DB engine")
+
 	searchCmd.Flags().Bool("installed", false, "Search between system packages")
 	searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
 	searchCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
@@ -285,6 +413,7 @@ func init() {
 	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
 	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
 	searchCmd.Flags().Bool("table", false, "show output in a table (wider screens)")
+	searchCmd.Flags().Bool("files", false, "Search between packages files")
 
 	RootCmd.AddCommand(searchCmd)
 }

cmd/tree/images.go

@@ -18,12 +18,14 @@ package cmd_tree
 
 import (
 	"fmt"
+	"os"
 
 	//. "github.com/mudler/luet/pkg/config"
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/options"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -56,6 +58,7 @@ func NewTreeImageCommand() *cobra.Command {
 
 			treePath, _ := cmd.Flags().GetStringArray("tree")
 			imageRepository := viper.GetString("image-repository")
+			pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
 			out, _ := cmd.Flags().GetString("output")
 			if out != "terminal" {
@@ -72,12 +75,15 @@ func NewTreeImageCommand() *cobra.Command {
 			}
 			compilerBackend := backend.NewSimpleDockerBackend()
 
-			opts := compiler.NewDefaultCompilerOptions()
-			opts.SolverOptions = *LuetCfg.GetSolverOptions()
-			opts.ImageRepository = imageRepository
-
-			solverOpts := solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
-			luetCompiler := compiler.NewLuetCompiler(compilerBackend, reciper.GetDatabase(), opts, solverOpts)
+			opts := *LuetCfg.GetSolverOptions()
+			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
+			luetCompiler := compiler.NewLuetCompiler(
+				compilerBackend,
+				reciper.GetDatabase(),
+				options.WithPushRepository(imageRepository),
+				options.WithPullRepositories(pullRepo),
+				options.WithSolverOptions(opts),
+			)
 
 			a := args[0]
 
@@ -127,10 +133,14 @@ func NewTreeImageCommand() *cobra.Command {
 			}
 		},
 	}
-
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
 	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
-	ans.Flags().StringArrayP("tree", "t", []string{}, "Path of the tree to use.")
+	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
 	ans.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+	ans.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")
 
 	return ans
 }

cmd/tree/pkglist.go

@@ -18,6 +18,7 @@ package cmd_tree
 
 import (
 	"fmt"
+	"os"
 	"sort"
 
 	//. "github.com/mudler/luet/pkg/config"
@@ -266,7 +267,10 @@ func NewTreePkglistCommand() *cobra.Command {
 
 		},
 	}
-
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
 	ans.Flags().BoolP("buildtime", "b", false, "Build time match")
 	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 	ans.Flags().Bool("revdeps", false, "Search package reverse dependencies")
@@ -274,7 +278,7 @@ func NewTreePkglistCommand() *cobra.Command {
 
 	ans.Flags().BoolP("verbose", "v", false, "Add package version")
 	ans.Flags().BoolP("full", "f", false, "Show package detail")
-	ans.Flags().StringArrayP("tree", "t", []string{}, "Path of the tree to use.")
+	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
 	ans.Flags().StringSliceVarP(&matches, "matches", "m", []string{},
 		"Include only matched packages from list. (Use string as regex).")
 	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},

cmd/tree/validate.go

@@ -244,10 +244,33 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 				Spinner(32)
 				solution, err := depSolver.Install(pkg.Packages{r})
 				ass := solution.SearchByName(r.GetPackageName())
-				if err == nil {
-					_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
-				}
 				SpinnerStop()
+				if err == nil {
+					if ass == nil {
+
+						ans = errors.New(
+							fmt.Sprintf("[%9s] %s/%s-%s: solution doesn't retrieve package %s/%s-%s.",
+								checkType,
+								p.GetCategory(), p.GetName(), p.GetVersion(),
+								r.GetCategory(), r.GetName(), r.GetVersion(),
+							))
+
+						if LuetCfg.GetGeneral().Debug {
+							for idx, pa := range solution {
+								fmt.Println(fmt.Sprintf("[%9s] %s/%s-%s: solution %d: %s",
+									checkType,
+									p.GetCategory(), p.GetName(), p.GetVersion(), idx,
+									pa.Package.GetPackageName()))
+							}
+						}
+
+						Error(ans.Error())
+						opts.IncrBrokenDeps()
+						validpkg = false
+					} else {
+						_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+					}
+				}
 
 				if err != nil {
 
@@ -458,12 +481,15 @@ func NewTreeValidateCommand() *cobra.Command {
 			}
 		},
 	}
-
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
 	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
 	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")
 	ans.Flags().BoolP("with-solver", "s", false,
 		"Enable check of requires also with solver.")
-	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{},
+	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{path},
 		"Path of the tree to use.")
 	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},
 		"Exclude matched packages from analysis. (Use string as regex).")

cmd/uninstall.go

@@ -15,9 +15,6 @@
 package cmd
 
 import (
-	"os"
-	"path/filepath"
-
 	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
@@ -43,76 +40,78 @@ var uninstallCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
-
+		toRemove := []pkg.Package{}
 		for _, a := range args {
 
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
 				Fatal("Invalid package string ", a, ": ", err.Error())
 			}
+			toRemove = append(toRemove, pack)
+		}
 
-			stype := LuetCfg.Viper.GetString("solver.type")
-			discount := LuetCfg.Viper.GetFloat64("solver.discount")
-			rate := LuetCfg.Viper.GetFloat64("solver.rate")
-			attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
-			force := LuetCfg.Viper.GetBool("force")
-			nodeps, _ := cmd.Flags().GetBool("nodeps")
-			full, _ := cmd.Flags().GetBool("full")
-			checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
-			fullClean, _ := cmd.Flags().GetBool("full-clean")
-			concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
-			yes := LuetCfg.Viper.GetBool("yes")
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps, _ := cmd.Flags().GetBool("nodeps")
+		full, _ := cmd.Flags().GetBool("full")
+		checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
+		fullClean, _ := cmd.Flags().GetBool("full-clean")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
+		dbpath := LuetCfg.Viper.GetString("system.database_path")
+		rootfs := LuetCfg.Viper.GetString("system.rootfs")
+		engine := LuetCfg.Viper.GetString("system.database_engine")
 
-			LuetCfg.GetSolverOptions().Type = stype
-			LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-			LuetCfg.GetSolverOptions().Discount = float32(discount)
-			LuetCfg.GetSolverOptions().MaxAttempts = attempts
-			if concurrent {
-				LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
-			} else {
-				LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
-			}
-			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		LuetCfg.System.DatabaseEngine = engine
+		LuetCfg.System.DatabasePath = dbpath
+		LuetCfg.System.Rootfs = rootfs
 
-			// Load config protect configs
-			installer.LoadConfigProtectConfs(LuetCfg)
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
-			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-				Concurrency:        LuetCfg.GetGeneral().Concurrency,
-				SolverOptions:      *LuetCfg.GetSolverOptions(),
-				NoDeps:             nodeps,
-				Force:              force,
-				FullUninstall:      full,
-				FullCleanUninstall: fullClean,
-				CheckConflicts:     checkconflicts,
-				Ask:                !yes,
-			})
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
 
-			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-				systemDB = pkg.NewBoltDatabase(
-					filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-			} else {
-				systemDB = pkg.NewInMemoryDatabase(true)
-			}
-			system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
-			err = inst.Uninstall(pack, system)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			FullUninstall:               full,
+			FullCleanUninstall:          fullClean,
+			CheckConflicts:              checkconflicts,
+			Ask:                         !yes,
+			PreserveSystemEssentialData: true,
+		})
+
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+
+		if err := inst.Uninstall(system, toRemove...); err != nil {
+			Fatal("Error: " + err.Error())
 		}
 	},
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	uninstallCmd.Flags().String("system-dbpath", path, "System db path")
-	uninstallCmd.Flags().String("system-target", path, "System rootpath")
+
+	uninstallCmd.Flags().String("system-dbpath", "", "System db path")
+	uninstallCmd.Flags().String("system-target", "", "System rootpath")
+	uninstallCmd.Flags().String("system-engine", "", "System DB engine")
+
 	uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
 	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")

cmd/upgrade.go

@@ -15,13 +15,9 @@
 package cmd
 
 import (
-	"os"
-	"path/filepath"
-
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
-	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/spf13/cobra"
@@ -40,10 +36,10 @@ var upgradeCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 	},
 	Long: `Upgrades packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
-		var systemDB pkg.PackageDatabase
 
 		repos := installer.Repositories{}
 		for _, repo := range LuetCfg.SystemRepositories {
@@ -67,7 +63,14 @@ var upgradeCmd = &cobra.Command{
 		sync, _ := cmd.Flags().GetBool("sync")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
+		dbpath := LuetCfg.Viper.GetString("system.database_path")
+		rootfs := LuetCfg.Viper.GetString("system.rootfs")
+		engine := LuetCfg.Viper.GetString("system.database_engine")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
+		LuetCfg.System.DatabaseEngine = engine
+		LuetCfg.System.DatabasePath = dbpath
+		LuetCfg.System.Rootfs = rootfs
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
@@ -84,26 +87,21 @@ var upgradeCmd = &cobra.Command{
 		installer.LoadConfigProtectConfs(LuetCfg)
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                LuetCfg.GetGeneral().Concurrency,
-			SolverOptions:              *LuetCfg.GetSolverOptions(),
-			Force:                      force,
-			FullUninstall:              full,
-			NoDeps:                     nodeps,
-			SolverUpgrade:              universe,
-			RemoveUnavailableOnUpgrade: clean,
-			UpgradeNewRevisions:        sync,
-			Ask:                        !yes,
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			Force:                       force,
+			FullUninstall:               full,
+			NoDeps:                      nodeps,
+			SolverUpgrade:               universe,
+			RemoveUnavailableOnUpgrade:  clean,
+			UpgradeNewRevisions:         sync,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
 		})
 		inst.Repositories(repos)
 
-		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
-			systemDB = pkg.NewBoltDatabase(
-				filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-		} else {
-			systemDB = pkg.NewInMemoryDatabase(true)
-		}
-		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
-
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
 		if err := inst.Upgrade(system); err != nil {
 			Fatal("Error: " + err.Error())
 		}
@@ -111,24 +109,23 @@ var upgradeCmd = &cobra.Command{
 }
 
 func init() {
-	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
-	upgradeCmd.Flags().String("system-dbpath", path, "System db path")
-	upgradeCmd.Flags().String("system-target", path, "System rootpath")
+	upgradeCmd.Flags().String("system-dbpath", "", "System db path")
+	upgradeCmd.Flags().String("system-target", "", "System rootpath")
+	upgradeCmd.Flags().String("system-engine", "", "System DB engine")
+
 	upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
 	upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
 	upgradeCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
-	upgradeCmd.Flags().Bool("full", true, "Attempts to remove as much packages as possible which aren't required (slow)")
+	upgradeCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
 	upgradeCmd.Flags().Bool("universe", false, "Use ONLY the SAT solver to compute upgrades (experimental)")
 	upgradeCmd.Flags().Bool("clean", false, "Try to drop removed packages (experimental, only when --universe is enabled)")
 	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
 	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	upgradeCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	upgradeCmd.Flags().Bool("download-only", false, "Download only")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

cmd/util.go

@@ -0,0 +1,36 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/util"
+
+	"github.com/spf13/cobra"
+)
+
+var utilGroup = &cobra.Command{
+	Use:   "util [command] [OPTIONS]",
+	Short: "General luet internal utilities exposed",
+}
+
+func init() {
+	RootCmd.AddCommand(utilGroup)
+
+	utilGroup.AddCommand(
+		NewUnpackCommand(),
+	)
+}

cmd/util/unpack.go

@@ -0,0 +1,98 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/go-units"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+func NewUnpackCommand() *cobra.Command {
+
+	c := &cobra.Command{
+		Use:   "unpack image path",
+		Short: "Unpack a docker image natively",
+		Long: `unpack doesn't need the docker daemon to run, and unpacks a docker image in the specified directory:
+		
+	luet util unpack golang:alpine /alpine
+`,
+		PreRun: func(cmd *cobra.Command, args []string) {
+
+			if len(args) != 2 {
+				Fatal("Expects an image and a path")
+			}
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			image := args[0]
+			destination, err := filepath.Abs(args[1])
+			if err != nil {
+				Error("Invalid path %s", destination)
+				os.Exit(1)
+			}
+
+			verify, _ := cmd.Flags().GetBool("verify")
+			user, _ := cmd.Flags().GetString("auth-username")
+			pass, _ := cmd.Flags().GetString("auth-password")
+			authType, _ := cmd.Flags().GetString("auth-type")
+			server, _ := cmd.Flags().GetString("auth-server-address")
+			identity, _ := cmd.Flags().GetString("auth-identity-token")
+			registryToken, _ := cmd.Flags().GetString("auth-registry-token")
+
+			temp, err := config.LuetCfg.GetSystem().TempDir("contentstore")
+			if err != nil {
+				Fatal("Cannot create a tempdir", err.Error())
+			}
+
+			Info("Downloading", image, "to", destination)
+			auth := &types.AuthConfig{
+				Username:      user,
+				Password:      pass,
+				ServerAddress: server,
+				Auth:          authType,
+				IdentityToken: identity,
+				RegistryToken: registryToken,
+			}
+
+			info, err := helpers.DownloadAndExtractDockerImage(temp, image, destination, auth, verify)
+			if err != nil {
+				Error(err.Error())
+				os.Exit(1)
+			}
+			Info(fmt.Sprintf("Pulled: %s %s", info.Target.Digest, info.Name))
+			Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
+		},
+	}
+
+	c.Flags().String("auth-username", "", "Username to authenticate to registry/notary")
+	c.Flags().String("auth-password", "", "Password to authenticate to registry")
+	c.Flags().String("auth-type", "", "Auth type")
+	c.Flags().String("auth-server-address", "", "Authentication server address")
+	c.Flags().String("auth-identity-token", "", "Authentication identity token")
+	c.Flags().String("auth-registry-token", "", "Authentication registry token")
+	c.Flags().Bool("verify", false, "Verify signed images to notary before to pull")
+	return c
+}

contrib/config/get_luet_root.sh

@@ -1,14 +1,19 @@
 #!/bin/bash
+if [ $(id -u) -ne 0 ]
+  then echo "Please run the installer with sudo/as root"
+  exit
+fi
+
 set -ex
 export LUET_NOLOCK=true
 
-LUET_VERSION=0.8.6
+LUET_VERSION=$(curl -s https://api.github.com/repos/mudler/luet/releases/latest | ( grep -oP '"tag_name": "\K(.*)(?=")' || echo "0.9.24" ))
 LUET_ROOTFS=${LUET_ROOTFS:-/}
 LUET_DATABASE_PATH=${LUET_DATABASE_PATH:-/var/luet/db}
 LUET_DATABASE_ENGINE=${LUET_DATABASE_ENGINE:-boltdb}
 LUET_CONFIG_PROTECT=${LUET_CONFIG_PROTECT:-1}
 
-wget -q https://github.com/mudler/luet/releases/download/0.8.6/luet-0.8.6-linux-amd64 -O luet
+curl -L https://github.com/mudler/luet/releases/download/${LUET_VERSION}/luet-${LUET_VERSION}-linux-amd64 --output luet
 chmod +x luet
 
 mkdir -p /etc/luet/repos.conf.d || true
@@ -17,9 +22,9 @@ mkdir -p /var/tmp/luet || true
 
 if [ "${LUET_CONFIG_PROTECT}" = "1" ] ; then
   mkdir -p /etc/luet/config.protect.d || true
-  wget -q  https://raw.githubusercontent.com/mudler/luet/master/contrib/config/config.protect.d/01_etc.yml.example -O /etc/luet/config.protect.d/01_etc.yml
+  curl -L https://raw.githubusercontent.com/mudler/luet/master/contrib/config/config.protect.d/01_etc.yml.example --output /etc/luet/config.protect.d/01_etc.yml
 fi
-wget -q https://raw.githubusercontent.com/mocaccinoOS/repository-index/master/packages/mocaccino-repository-index/mocaccino-repository-index.yml -O /etc/luet/repos.conf.d/mocaccino-repository-index.yml
+curl -L https://raw.githubusercontent.com/mocaccinoOS/repository-index/master/packages/mocaccino-repository-index.yml --output /etc/luet/repos.conf.d/mocaccino-repository-index.yml
 
 cat > /etc/luet/luet.yaml <<EOF
 general:
@@ -31,7 +36,8 @@ system:
   tmpdir_base: "/var/tmp/luet"
 EOF
 
-./luet install repository/luet repository/mocaccino-repository-index
-./luet install system/luet system/luet-extensions
+./luet install -y repository/luet repository/mocaccino-repository-index
+./luet install -y system/luet system/luet-extensions
 
 rm -rf luet
+

go.mod

@@ -1,51 +1,71 @@
 module github.com/mudler/luet
 
-go 1.12
+go 1.14
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/Sabayon/pkgs-checker v0.7.2
+	github.com/Sabayon/pkgs-checker v0.8.1
+	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
-	github.com/briandowns/spinner v1.7.0
+	github.com/briandowns/spinner v1.12.1-0.20201220203425-e201aaea0a31
 	github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec
+	github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc
 	github.com/crillab/gophersat v1.3.2-0.20201023142334-3fc2ac466765
-	github.com/docker/docker v17.12.0-ce-rc1.0.20200417035958-130b0bc6032c+incompatible
-	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
+	github.com/docker/cli v0.0.0-20200227165822-2298e6a3fe24
+	github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
+	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
-	github.com/fsouza/go-dockerclient v1.6.4
+	github.com/genuinetools/img v0.5.11
 	github.com/ghodss/yaml v1.0.0
-	github.com/hashicorp/go-version v1.2.0
+	github.com/google/go-containerregistry v0.2.1
+	github.com/google/renameio v1.0.0
+	github.com/hashicorp/go-multierror v1.0.0
+	github.com/hashicorp/go-version v1.2.1
+	github.com/imdario/mergo v0.3.8
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/jedib0t/go-pretty/v6 v6.0.5
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
+	github.com/klauspost/compress v1.8.3
 	github.com/klauspost/pgzip v1.2.1
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
-	github.com/kr/pretty v0.2.0 // indirect
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
-	github.com/moby/sys/mount v0.1.1-0.20200320164225-6154f11e6840 // indirect
+	github.com/moby/buildkit v0.7.2
+	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+	github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3
+	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.0.1
 	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
-	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
 	github.com/pkg/errors v0.9.1
 	github.com/schollz/progressbar/v3 v3.7.1
-	github.com/spf13/cobra v1.0.0
-	github.com/spf13/viper v1.6.3
-	go.etcd.io/bbolt v1.3.4
+	github.com/sirupsen/logrus v1.7.0
+	github.com/spf13/cobra v1.1.1
+	github.com/spf13/viper v1.7.1
+	github.com/theupdateframework/notary v0.7.0
+	go.etcd.io/bbolt v1.3.5
 	go.uber.org/atomic v1.5.1 // indirect
-	go.uber.org/multierr v1.4.0 // indirect
+	go.uber.org/multierr v1.4.0
 	go.uber.org/zap v1.13.0
-	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
+	google.golang.org/grpc v1.29.1
 	gopkg.in/yaml.v2 v2.3.0
-	gotest.tools/v3 v3.0.2 // indirect
 	helm.sh/helm/v3 v3.3.4
+
 )
 
 replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200605210607-749178b8f80d+incompatible
+
+replace github.com/containerd/containerd => github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55
+
+replace github.com/hashicorp/go-immutable-radix => github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe
+
+replace github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305
+
+replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc9.0.20200221051241-688cf6d43cc4

pkg/bus/events.go

@@ -1,6 +1,8 @@
 package bus
 
 import (
+	. "github.com/mudler/luet/pkg/logger"
+
 	"github.com/mudler/go-pluggable"
 )
 
@@ -16,13 +18,28 @@ var (
 
 	// EventPackagePreBuild is the event fired before a package is being built
 	EventPackagePreBuild pluggable.EventType = "package.pre.build"
-	// EventPackagePreBuildArtifact is the event fired before a package artifact is being built
+	// EventPackagePreBuildArtifact is the event fired before a package tarball is being generated
 	EventPackagePreBuildArtifact pluggable.EventType = "package.pre.build_artifact"
-	// EventPackagePostBuildArtifact is the event fired after a package artifact was built
+	// EventPackagePostBuildArtifact is the event fired after a package tarball is generated
 	EventPackagePostBuildArtifact pluggable.EventType = "package.post.build_artifact"
 	// EventPackagePostBuild is the event fired after a package was built
 	EventPackagePostBuild pluggable.EventType = "package.post.build"
 
+	// Image build
+
+	// EventImagePreBuild is the event fired before a image is being built
+	EventImagePreBuild pluggable.EventType = "image.pre.build"
+	// EventImagePrePull is the event fired before a image is being pulled
+	EventImagePrePull pluggable.EventType = "image.pre.pull"
+	// EventImagePrePush is the event fired before a image is being pushed
+	EventImagePrePush pluggable.EventType = "image.pre.push"
+	// EventImagePostBuild is the event fired after an image is being built
+	EventImagePostBuild pluggable.EventType = "image.post.build"
+	// EventImagePostPull is the event fired after an image is being pulled
+	EventImagePostPull pluggable.EventType = "image.post.pull"
+	// EventImagePostPush is the event fired after an image is being pushed
+	EventImagePostPush pluggable.EventType = "image.post.push"
+
 	// Repository events
 
 	// EventRepositoryPreBuild is the event fired before a repository is being built
@@ -32,15 +49,47 @@ var (
 )
 
 // Manager is the bus instance manager, which subscribes plugins to events emitted by Luet
-var Manager *pluggable.Manager = pluggable.NewManager(
-	[]pluggable.EventType{
-		EventPackageInstall,
-		EventPackageUnInstall,
-		EventPackagePreBuild,
-		EventPackagePreBuildArtifact,
-		EventPackagePostBuildArtifact,
-		EventPackagePostBuild,
-		EventRepositoryPreBuild,
-		EventRepositoryPostBuild,
-	},
-)
+var Manager *Bus = &Bus{
+	Manager: pluggable.NewManager(
+		[]pluggable.EventType{
+			EventPackageInstall,
+			EventPackageUnInstall,
+			EventPackagePreBuild,
+			EventPackagePreBuildArtifact,
+			EventPackagePostBuildArtifact,
+			EventPackagePostBuild,
+			EventRepositoryPreBuild,
+			EventRepositoryPostBuild,
+			EventImagePreBuild,
+			EventImagePrePull,
+			EventImagePrePush,
+			EventImagePostBuild,
+			EventImagePostPull,
+			EventImagePostPush,
+		},
+	),
+}
+
+type Bus struct {
+	*pluggable.Manager
+}
+
+func (b *Bus) Initialize(plugin ...string) {
+	b.Manager.Load(plugin...).Register()
+
+	for _, e := range b.Manager.Events {
+		b.Manager.Response(e, func(p *pluggable.Plugin, r *pluggable.EventResponse) {
+			if r.Errored() {
+				Fatal("Plugin", p.Name, "at", p.Executable, "Error", r.Error)
+			}
+			Debug(
+				"plugin_event",
+				"received from",
+				p.Name,
+				"at",
+				p.Executable,
+				r,
+			)
+		})
+	}
+}

pkg/compiler/artifact_test.go

@@ -1,158 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler_test
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	. "github.com/mudler/luet/pkg/compiler/backend"
-	"github.com/mudler/luet/pkg/solver"
-
-	. "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/tree"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Artifact", func() {
-	Context("Simple package build definition", func() {
-		It("Generates a verified delta", func() {
-
-			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
-
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
-
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
-			Expect(err).ToNot(HaveOccurred())
-
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
-			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
-			Expect(lspec.Image).To(Equal("luet/base"))
-			Expect(lspec.Seed).To(Equal("alpine"))
-			tmpdir, err := ioutil.TempDir(os.TempDir(), "tree")
-			Expect(err).ToNot(HaveOccurred())
-			defer os.RemoveAll(tmpdir) // clean up
-
-			tmpdir2, err := ioutil.TempDir(os.TempDir(), "tree2")
-			Expect(err).ToNot(HaveOccurred())
-			defer os.RemoveAll(tmpdir2) // clean up
-
-			unpacked, err := ioutil.TempDir(os.TempDir(), "unpacked")
-			Expect(err).ToNot(HaveOccurred())
-			defer os.RemoveAll(unpacked) // clean up
-
-			rootfs, err := ioutil.TempDir(os.TempDir(), "rootfs")
-			Expect(err).ToNot(HaveOccurred())
-			defer os.RemoveAll(rootfs) // clean up
-
-			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(dockerfile).To(Equal(`
-FROM alpine
-COPY . /luetbuild
-WORKDIR /luetbuild
-ENV PACKAGE_NAME=enman
-ENV PACKAGE_VERSION=1.4.0
-ENV PACKAGE_CATEGORY=app-admin`))
-			b := NewSimpleDockerBackend()
-			opts := CompilerBackendOptions{
-				ImageName:      "luet/base",
-				SourcePath:     tmpdir,
-				DockerFileName: "Dockerfile",
-				Destination:    filepath.Join(tmpdir2, "output1.tar"),
-			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
-			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
-
-			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(dockerfile).To(Equal(`
-FROM luet/base
-ENV PACKAGE_NAME=enman
-ENV PACKAGE_VERSION=1.4.0
-ENV PACKAGE_CATEGORY=app-admin
-RUN echo foo > /test
-RUN echo bar > /test2`))
-			opts = CompilerBackendOptions{
-				ImageName:      "test",
-				SourcePath:     tmpdir,
-				DockerFileName: "LuetDockerfile",
-				Destination:    filepath.Join(tmpdir, "output2.tar"),
-			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
-			diffs, err := b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))
-			Expect(err).ToNot(HaveOccurred())
-
-			artifacts := []ArtifactNode{}
-			if os.Getenv("DOCKER_BUILDKIT") == "1" {
-				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
-			}
-			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
-			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
-
-			Expect(diffs).To(Equal(
-				[]ArtifactLayer{{
-					FromImage: filepath.Join(tmpdir2, "output1.tar"),
-					ToImage:   filepath.Join(tmpdir, "output2.tar"),
-					Diffs: ArtifactDiffs{
-						Additions: artifacts,
-					},
-				}}))
-			err = b.ExtractRootfs(CompilerBackendOptions{SourcePath: filepath.Join(tmpdir, "output2.tar"), Destination: rootfs}, false)
-			Expect(err).ToNot(HaveOccurred())
-
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, None)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
-			err = helpers.Untar(artifact.GetPath(), unpacked, false)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
-			content1, err := helpers.Read(filepath.Join(unpacked, "test"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(content1).To(Equal("foo\n"))
-			content2, err := helpers.Read(filepath.Join(unpacked, "test2"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(content2).To(Equal("bar\n"))
-
-			err = artifact.Hash()
-			Expect(err).ToNot(HaveOccurred())
-			err = artifact.Verify()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
-
-			err = artifact.Verify()
-			Expect(err).To(HaveOccurred())
-		})
-
-	})
-})

pkg/compiler/backend.go

@@ -0,0 +1,247 @@
+package compiler
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+
+	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/pkg/errors"
+
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+func NewBackend(s string) (CompilerBackend, error) {
+	var compilerBackend CompilerBackend
+
+	switch s {
+	case backend.ImgBackend:
+		compilerBackend = backend.NewSimpleImgBackend()
+	case backend.DockerBackend:
+		compilerBackend = backend.NewSimpleDockerBackend()
+	default:
+		return nil, errors.New("invalid backend. Unsupported")
+	}
+
+	return compilerBackend, nil
+}
+
+type CompilerBackend interface {
+	BuildImage(backend.Options) error
+	ExportImage(backend.Options) error
+	RemoveImage(backend.Options) error
+	ImageDefinitionToTar(backend.Options) error
+	ExtractRootfs(opts backend.Options, keepPerms bool) error
+
+	CopyImage(string, string) error
+	DownloadImage(opts backend.Options) error
+
+	Push(opts backend.Options) error
+	ImageAvailable(string) bool
+
+	ImageExists(string) bool
+}
+
+// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
+// example of json return: [
+//   {
+//     "Image1": "luet/base",
+//     "Image2": "alpine",
+//     "DiffType": "File",
+//     "Diff": {
+//       "Adds": null,
+//       "Dels": [
+//         {
+//           "Name": "/luetbuild",
+//           "Size": 5830706
+//         },
+//         {
+//           "Name": "/luetbuild/Dockerfile",
+//           "Size": 50
+//         },
+//         {
+//           "Name": "/luetbuild/output1",
+//           "Size": 5830656
+//         }
+//       ],
+//       "Mods": null
+//     }
+//   }
+// ]
+func GenerateChanges(b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) {
+
+	res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
+
+	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(srcRootFS) // clean up
+
+	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(dstRootFS) // clean up
+
+	srcImageExtract := backend.Options{
+		ImageName:   fromImage.ImageName,
+		Destination: srcRootFS,
+	}
+	Debug("Extracting source image", fromImage.ImageName)
+	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
+	}
+
+	dstImageExtract := backend.Options{
+		ImageName:   toImage.ImageName,
+		Destination: dstRootFS,
+	}
+	Debug("Extracting destination image", toImage.ImageName)
+	err = b.ExtractRootfs(dstImageExtract, false)
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
+	}
+
+	// Get Additions/Changes. dst -> src
+	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, dstRootFS, "", -1)
+		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
+		if err == nil {
+			var sizeA, sizeB int64
+			sizeA = fileInfo.Size()
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+
+			if sizeA != sizeB {
+				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
+				res.Diffs.Changes = append(res.Diffs.Changes, artifact.ArtifactNode{
+					Name: filepath.Join("/", realpath),
+					Size: int(sizeB),
+				})
+			} else {
+				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
+			}
+		} else {
+			var sizeB int64
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+			res.Diffs.Additions = append(res.Diffs.Additions, artifact.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+				Size: int(sizeB),
+			})
+
+			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+	}
+
+	// Get deletions. src -> dst
+	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, srcRootFS, "", -1)
+		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
+			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
+			res.Diffs.Deletions = append(res.Diffs.Deletions, artifact.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+			})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+	}
+
+	diffs := []artifact.ArtifactLayer{res}
+
+	if config.LuetCfg.GetGeneral().Debug {
+		summary := ComputeArtifactLayerSummary(diffs)
+		for _, l := range summary.Layers {
+			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
+				l.FromImage, l.ToImage,
+				l.AddFiles, l.AddSizes,
+				l.DelFiles, l.DelSizes,
+				l.ChangeFiles, l.ChangeSizes))
+		}
+	}
+
+	return diffs, nil
+}
+
+type ArtifactLayerSummary struct {
+	FromImage   string `json:"image1"`
+	ToImage     string `json:"image2"`
+	AddFiles    int    `json:"add_files"`
+	AddSizes    int64  `json:"add_sizes"`
+	DelFiles    int    `json:"del_files"`
+	DelSizes    int64  `json:"del_sizes"`
+	ChangeFiles int    `json:"change_files"`
+	ChangeSizes int64  `json:"change_sizes"`
+}
+
+type ArtifactLayersSummary struct {
+	Layers []ArtifactLayerSummary `json:"summary"`
+}
+
+func ComputeArtifactLayerSummary(diffs []artifact.ArtifactLayer) ArtifactLayersSummary {
+
+	ans := ArtifactLayersSummary{
+		Layers: make([]ArtifactLayerSummary, 0),
+	}
+
+	for _, layer := range diffs {
+		sum := ArtifactLayerSummary{
+			FromImage:   layer.FromImage,
+			ToImage:     layer.ToImage,
+			AddFiles:    0,
+			AddSizes:    0,
+			DelFiles:    0,
+			DelSizes:    0,
+			ChangeFiles: 0,
+			ChangeSizes: 0,
+		}
+		for _, a := range layer.Diffs.Additions {
+			sum.AddFiles++
+			sum.AddSizes += int64(a.Size)
+		}
+		for _, d := range layer.Diffs.Deletions {
+			sum.DelFiles++
+			sum.DelSizes += int64(d.Size)
+		}
+		for _, c := range layer.Diffs.Changes {
+			sum.ChangeFiles++
+			sum.ChangeSizes += int64(c.Size)
+		}
+		ans.Layers = append(ans.Layers, sum)
+	}
+
+	return ans
+}

pkg/compiler/backend/common.go

@@ -0,0 +1,82 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"os/exec"
+
+	"github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/pkg/errors"
+)
+
+const (
+	ImgBackend    = "img"
+	DockerBackend = "docker"
+)
+
+func imageAvailable(image string) bool {
+	_, err := crane.Digest(image)
+	return err == nil
+}
+
+type Options struct {
+	ImageName      string
+	SourcePath     string
+	DockerFileName string
+	Destination    string
+	Context        string
+	BackendArgs    []string
+}
+
+func runCommand(cmd *exec.Cmd) error {
+	output := ""
+	buffered := !config.LuetCfg.GetGeneral().ShowBuildOutput
+	writer := NewBackendWriter(buffered)
+
+	cmd.Stdout = writer
+	cmd.Stderr = writer
+
+	if buffered {
+		Spinner(22)
+		defer SpinnerStop()
+	}
+
+	err := cmd.Start()
+	if err != nil {
+		return errors.Wrap(err, "Failed starting command")
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		output = writer.GetCombinedOutput()
+		return errors.Wrapf(err, "Failed running command: %s", output)
+	}
+
+	return nil
+}
+
+func genBuildCommand(opts Options) []string {
+	context := opts.Context
+
+	if context == "" {
+		context = "."
+	}
+	buildarg := append(opts.BackendArgs, "-f", opts.DockerFileName, "-t", opts.ImageName, context)
+	return append([]string{"build"}, buildarg...)
+}

pkg/compiler/backend/diff.go

@@ -1,199 +0,0 @@
-// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package backend
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/config"
-	"github.com/pkg/errors"
-)
-
-// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
-// example of json return: [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]compiler.ArtifactLayer, error) {
-
-	res := compiler.ArtifactLayer{FromImage: srcImage, ToImage: dstImage}
-
-	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-
-	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(srcRootFS) // clean up
-
-	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(dstRootFS) // clean up
-
-	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
-	if !strings.HasPrefix(srcImage, "/") {
-		srcImageTar, err := ioutil.TempFile(tmpdiffs, "srctar")
-		if err != nil {
-			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-		}
-
-		defer os.Remove(srcImageTar.Name()) // clean up
-		srcImageExport := compiler.CompilerBackendOptions{
-			ImageName:   srcImage,
-			Destination: srcImageTar.Name(),
-		}
-		err = b.ExportImage(srcImageExport)
-		if err != nil {
-			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting src image "+srcImage)
-		}
-		srcImage = srcImageTar.Name()
-	}
-
-	srcImageExtract := compiler.CompilerBackendOptions{
-		SourcePath:  srcImage,
-		Destination: srcRootFS,
-	}
-	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+srcImage)
-	}
-
-	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
-	if !strings.HasPrefix(dstImage, "/") {
-		dstImageTar, err := ioutil.TempFile(tmpdiffs, "dsttar")
-		if err != nil {
-			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-		}
-
-		defer os.Remove(dstImageTar.Name()) // clean up
-		dstImageExport := compiler.CompilerBackendOptions{
-			ImageName:   dstImage,
-			Destination: dstImageTar.Name(),
-		}
-		err = b.ExportImage(dstImageExport)
-		if err != nil {
-			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting dst image "+dstImage)
-		}
-		dstImage = dstImageTar.Name()
-	}
-
-	dstImageExtract := compiler.CompilerBackendOptions{
-		SourcePath:  dstImage,
-		Destination: dstRootFS,
-	}
-	err = b.ExtractRootfs(dstImageExtract, false)
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+dstImage)
-	}
-
-	// Get Additions/Changes. dst -> src
-	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
-		if info.IsDir() {
-			return nil
-		}
-
-		realpath := strings.Replace(path, dstRootFS, "", -1)
-		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
-		if err == nil {
-			var sizeA, sizeB int64
-			sizeA = fileInfo.Size()
-
-			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
-				sizeB = s.Size()
-			}
-
-			if sizeA != sizeB {
-				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
-				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
-					Name: filepath.Join("/", realpath),
-					Size: int(sizeB),
-				})
-			} else {
-				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
-			}
-		} else {
-			var sizeB int64
-
-			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
-				sizeB = s.Size()
-			}
-			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
-				Name: filepath.Join("/", realpath),
-				Size: int(sizeB),
-			})
-
-			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
-		}
-
-		return nil
-	})
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
-	}
-
-	// Get deletions. src -> dst
-	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
-		if info.IsDir() {
-			return nil
-		}
-
-		realpath := strings.Replace(path, srcRootFS, "", -1)
-		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
-			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
-			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
-				Name: filepath.Join("/", realpath),
-			})
-		}
-
-		return nil
-	})
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
-	}
-
-	return []compiler.ArtifactLayer{res}, nil
-}

pkg/compiler/backend/simpledocker.go

@@ -17,18 +17,16 @@ package backend
 
 import (
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 
-	docker "github.com/fsouza/go-dockerclient"
+	bus "github.com/mudler/luet/pkg/bus"
+
 	capi "github.com/mudler/docker-companion/api"
 
-	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -37,47 +35,27 @@ import (
 
 type SimpleDocker struct{}
 
-func NewSimpleDockerBackend() compiler.CompilerBackend {
+func NewSimpleDockerBackend() *SimpleDocker {
 	return &SimpleDocker{}
 }
 
 // TODO: Missing still: labels, and build args expansion
-func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) BuildImage(opts Options) error {
 	name := opts.ImageName
-	path := opts.SourcePath
-	dockerfileName := opts.DockerFileName
+	bus.Manager.Publish(bus.EventImagePreBuild, opts)
 
-	buildarg := []string{"build", "-f", dockerfileName, "-t", name, "."}
-
-	Debug(":whale2: Building image " + name)
+	buildarg := genBuildCommand(opts)
+	Info(":whale2: Building image " + name)
 	cmd := exec.Command("docker", buildarg...)
-	cmd.Dir = path
-	out, err := cmd.CombinedOutput()
+	cmd.Dir = opts.SourcePath
+	err := runCommand(cmd)
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return err
 	}
+
 	Info(":whale: Building image " + name + " done")
 
-	if os.Getenv("DOCKER_SQUASH") == "true" {
-		Info(":whale: Squashing image " + name)
-		var client *docker.Client
-
-		client, err = docker.NewClientFromEnv()
-		if err != nil {
-			return errors.Wrap(err, "could not connect to the Docker daemon")
-		}
-		err = capi.Squash(client, name, name)
-		if err != nil {
-			return errors.Wrap(err, "Failed squashing image")
-		}
-		Info(":whale: Squashing image " + name + " done")
-	}
-
-	if config.LuetCfg.GetGeneral().ShowBuildOutput {
-		Info(string(out))
-	} else {
-		Debug(string(out))
-	}
+	bus.Manager.Publish(bus.EventImagePostBuild, opts)
 
 	return nil
 }
@@ -93,16 +71,25 @@ func (*SimpleDocker) CopyImage(src, dst string) error {
 	return nil
 }
 
-func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) DownloadImage(opts Options) error {
 	name := opts.ImageName
+	bus.Manager.Publish(bus.EventImagePrePull, opts)
+
 	buildarg := []string{"pull", name}
 	Debug(":whale: Downloading image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	cmd := exec.Command("docker", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed pulling image: "+string(out))
 	}
+
 	Info(":whale: Downloaded image:", name)
+	bus.Manager.Publish(bus.EventImagePostPull, opts)
+
 	return nil
 }
 
@@ -119,7 +106,11 @@ func (*SimpleDocker) ImageExists(imagename string) bool {
 	return true
 }
 
-func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) ImageAvailable(imagename string) bool {
+	return imageAvailable(imagename)
+}
+
+func (*SimpleDocker) RemoveImage(opts Options) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
 	out, err := exec.Command("docker", buildarg...).CombinedOutput()
@@ -131,19 +122,26 @@ func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
-func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) Push(opts Options) error {
 	name := opts.ImageName
 	pusharg := []string{"push", name}
+	bus.Manager.Publish(bus.EventImagePrePush, opts)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("docker", pusharg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed pushing image: "+string(out))
 	}
 	Info(":whale: Pushed image:", name)
+	bus.Manager.Publish(bus.EventImagePostPush, opts)
+
 	//Info(string(out))
 	return nil
 }
 
-func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
 	}
@@ -156,18 +154,22 @@ func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions
 	return nil
 }
 
-func (*SimpleDocker) ExportImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleDocker) ExportImage(opts Options) error {
 	name := opts.ImageName
 	path := opts.Destination
 
 	buildarg := []string{"save", name, "-o", path}
 	Debug(":whale: Saving image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("docker", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed exporting image: "+string(out))
 	}
 
-	Info(":whale: Exported image:", name)
+	Debug(":whale: Exported image:", name)
 	return nil
 }
 
@@ -175,10 +177,42 @@ type ManifestEntry struct {
 	Layers []string `json:"Layers"`
 }
 
-func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
-	src := opts.SourcePath
+func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
+	name := opts.ImageName
 	dst := opts.Destination
 
+	tempexport, err := ioutil.TempDir(dst, "tmprootfs")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tempexport) // clean up
+
+	imageExport := filepath.Join(tempexport, "image.tar")
+
+	Spinner(22)
+	defer SpinnerStop()
+
+	if err := b.ExportImage(Options{ImageName: name, Destination: imageExport}); err != nil {
+		return errors.Wrap(err, "failed while extracting rootfs for "+name)
+	}
+
+	SpinnerStop()
+
+	src := imageExport
+
+	if src == "" && opts.ImageName != "" {
+		tempUnpack, err := ioutil.TempDir(dst, "tempUnpack")
+		if err != nil {
+			return errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+		defer os.RemoveAll(tempUnpack) // clean up
+		imageExport := filepath.Join(tempUnpack, "image.tar")
+		if err := b.ExportImage(Options{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
+			return errors.Wrap(err, "while exporting image before extraction")
+		}
+		src = imageExport
+	}
+
 	rootfs, err := ioutil.TempDir(dst, "tmprootfs")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
@@ -219,13 +253,13 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 			}
 		}
 	}
-
+	// TODO: Drop capi in favor of the img approach already used in pkg/installer/repository
 	export, err := capi.CreateExport(rootfs)
 	if err != nil {
 		return err
 	}
 
-	err = export.UnPackLayers(layers_sha, dst, "")
+	err = export.UnPackLayers(layers_sha, dst, "containerd")
 	if err != nil {
 		return err
 	}
@@ -237,21 +271,3 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 
 	return nil
 }
-
-// Changes retrieves changes between image layers
-func (d *SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	diffs, err := GenerateChanges(d, fromImage, toImage)
-
-	if config.LuetCfg.GetGeneral().Debug {
-		summary := compiler.ComputeArtifactLayerSummary(diffs)
-		for _, l := range summary.Layers {
-			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
-				l.FromImage, l.ToImage,
-				l.AddFiles, l.AddSizes,
-				l.DelFiles, l.DelSizes,
-				l.ChangeFiles, l.ChangeSizes))
-		}
-	}
-
-	return diffs, err
-}

pkg/compiler/backend/simpledocker_test.go

@@ -16,9 +16,11 @@
 package backend_test
 
 import (
+	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
-	"github.com/mudler/luet/pkg/solver"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 
 	"io/ioutil"
 	"os"
@@ -41,13 +43,10 @@ var _ = Describe("Docker backend", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
 			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 			Expect(lspec.Image).To(Equal("luet/base"))
 			Expect(lspec.Seed).To(Equal("alpine"))
@@ -71,15 +70,16 @@ ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin`))
 			b := NewSimpleDockerBackend()
-			opts := CompilerBackendOptions{
+			opts := backend.Options{
 				ImageName:      "luet/base",
 				SourcePath:     tmpdir,
 				DockerFileName: "Dockerfile",
 				Destination:    filepath.Join(tmpdir2, "output1.tar"),
 			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
@@ -87,37 +87,60 @@ ENV PACKAGE_CATEGORY=app-admin`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
-			opts = CompilerBackendOptions{
+			opts2 := backend.Options{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
 				Destination:    filepath.Join(tmpdir, "output2.tar"),
 			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
+
+			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
-			artifacts := []ArtifactNode{}
+			artifacts := []artifact.ArtifactNode{{
+				Name: "/luetbuild/LuetDockerfile",
+				Size: 175,
+			}}
 			if os.Getenv("DOCKER_BUILDKIT") == "1" {
-				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+				artifacts = append(artifacts, artifact.ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
 			}
-			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
-			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4})
 
-			Expect(b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))).To(Equal(
-				[]ArtifactLayer{{
-					FromImage: filepath.Join(tmpdir2, "output1.tar"),
-					ToImage:   filepath.Join(tmpdir, "output2.tar"),
-					Diffs: ArtifactDiffs{
+			Expect(compiler.GenerateChanges(b, opts, opts2)).To(Equal(
+				[]artifact.ArtifactLayer{{
+					FromImage: "luet/base",
+					ToImage:   "test",
+					Diffs: artifact.ArtifactDiffs{
 						Additions: artifacts,
 					},
 				}}))
 
+			opts2 = backend.Options{
+				ImageName:      "test",
+				SourcePath:     tmpdir,
+				DockerFileName: "LuetDockerfile",
+				Destination:    filepath.Join(tmpdir, "output3.tar"),
+			}
+
+			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
+			Expect(b.ImageExists(opts2.ImageName)).To(BeFalse())
 		})
 
+		It("Detects available images", func() {
+			b := NewSimpleDockerBackend()
+			Expect(b.ImageAvailable("quay.io/mocaccino/extra")).To(BeTrue())
+			Expect(b.ImageAvailable("ubuntu:20.10")).To(BeTrue())
+			Expect(b.ImageAvailable("igjo5ijgo25nho52")).To(BeFalse())
+		})
 	})
 })

pkg/compiler/backend/simpleimg.go

@@ -18,8 +18,10 @@ package backend
 import (
 	"os"
 	"os/exec"
+	"strings"
+
+	bus "github.com/mudler/luet/pkg/bus"
 
-	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/pkg/errors"
@@ -27,58 +29,64 @@ import (
 
 type SimpleImg struct{}
 
-func NewSimpleImgBackend() compiler.CompilerBackend {
+func NewSimpleImgBackend() *SimpleImg {
 	return &SimpleImg{}
 }
 
 // TODO: Missing still: labels, and build args expansion
-func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) BuildImage(opts Options) error {
 	name := opts.ImageName
-	path := opts.SourcePath
-	dockerfileName := opts.DockerFileName
+	bus.Manager.Publish(bus.EventImagePreBuild, opts)
+
+	buildarg := genBuildCommand(opts)
+
+	Info(":tea: Building image " + name)
 
-	buildarg := []string{"build", "-f", dockerfileName, "-t", name, "."}
-	Spinner(22)
-	defer SpinnerStop()
-	Debug(":tea: Building image " + name)
 	cmd := exec.Command("img", buildarg...)
-	cmd.Dir = path
-	out, err := cmd.CombinedOutput()
-
+	cmd.Dir = opts.SourcePath
+	err := runCommand(cmd)
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return err
 	}
+	bus.Manager.Publish(bus.EventImagePostBuild, opts)
+
 	Info(":tea: Building image " + name + " done")
+
 	return nil
 }
 
-func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) RemoveImage(opts Options) error {
 	name := opts.ImageName
 	buildarg := []string{"rm", name}
 	Spinner(22)
 	defer SpinnerStop()
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed removing image: "+string(out))
 	}
 
 	Info(":tea: Image " + name + " removed")
 	return nil
 }
 
-func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
-
+func (*SimpleImg) DownloadImage(opts Options) error {
 	name := opts.ImageName
-	buildarg := []string{"pull", name}
+	bus.Manager.Publish(bus.EventImagePrePull, opts)
 
+	buildarg := []string{"pull", name}
 	Debug(":tea: Downloading image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	cmd := exec.Command("img", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed downloading image: "+string(out))
 	}
 
 	Info(":tea: Image " + name + " downloaded")
+	bus.Manager.Publish(bus.EventImagePostPull, opts)
 
 	return nil
 }
@@ -97,14 +105,24 @@ func (*SimpleImg) CopyImage(src, dst string) error {
 	return nil
 }
 
+func (*SimpleImg) ImageAvailable(imagename string) bool {
+	return imageAvailable(imagename)
+}
+
+// ImageExists check if the given image is available locally
 func (*SimpleImg) ImageExists(imagename string) bool {
-	// NOOP: not implemented
-	// TODO: Since img doesn't have an inspect command,
-	// we need to parse the ls output manually
+	cmd := exec.Command("img", "ls")
+	out, err := cmd.Output()
+	if err != nil {
+		return false
+	}
+	if strings.Contains(string(out), imagename) {
+		return true
+	}
 	return false
 }
 
-func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleImg) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
 	}
@@ -117,50 +135,62 @@ func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) e
 	return nil
 }
 
-func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) ExportImage(opts Options) error {
 	name := opts.ImageName
 	path := opts.Destination
 	buildarg := []string{"save", "-o", path, name}
 	Debug(":tea: Saving image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed exporting image: "+string(out))
 	}
 	Info(":tea: Image " + name + " saved")
 	return nil
 }
 
-// TODO: Dup in docker, refactor common code in helpers for shared parts
-func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
+// ExtractRootfs extracts the docker image content inside the destination
+func (s *SimpleImg) ExtractRootfs(opts Options, keepPerms bool) error {
 	name := opts.ImageName
 	path := opts.Destination
 
+	if !s.ImageExists(name) {
+		if err := s.DownloadImage(opts); err != nil {
+			return errors.Wrap(err, "failed pulling image "+name+" during extraction")
+		}
+	}
+
 	os.RemoveAll(path)
+
 	buildarg := []string{"unpack", "-o", path, name}
 	Debug(":tea: Extracting image " + name)
+
+	Spinner(22)
+	defer SpinnerStop()
+
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed extracting image: "+string(out))
 	}
-	Info(":tea: Image " + name + " extracted")
+	Debug(":tea: Image " + name + " extracted")
 	return nil
-	//return NewSimpleDockerBackend().ExtractRootfs(opts, keepPerms)
 }
 
-// TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (i *SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	return GenerateChanges(i, fromImage, toImage)
-}
-
-func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {
+func (*SimpleImg) Push(opts Options) error {
 	name := opts.ImageName
+	bus.Manager.Publish(bus.EventImagePrePush, opts)
+
 	pusharg := []string{"push", name}
 	out, err := exec.Command("img", pusharg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed pushing image: "+string(out))
 	}
 	Info(":tea: Pushed image:", name)
+	bus.Manager.Publish(bus.EventImagePostPush, opts)
+
 	//Info(string(out))
 	return nil
 }

pkg/compiler/backend/writer.go

@@ -0,0 +1,48 @@
+// Copyright © 2021 Daniele Rondina <geaaru@sabayonlinux.org>
+//                  Ettore Di Giacinto <mudler@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"bytes"
+
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+type BackendWriter struct {
+	BufferedOutput bool
+	Buffer         *bytes.Buffer
+}
+
+func NewBackendWriter(buffered bool) *BackendWriter {
+	return &BackendWriter{
+		BufferedOutput: buffered,
+		Buffer:         &bytes.Buffer{},
+	}
+}
+
+func (b *BackendWriter) Write(p []byte) (int, error) {
+	if b.BufferedOutput {
+		return b.Buffer.Write(p)
+	}
+
+	Msg("info", false, false, (string(p)))
+
+	return len(p), nil
+}
+
+func (b *BackendWriter) Close() error              { return nil }
+func (b *BackendWriter) GetCombinedOutput() string { return b.Buffer.String() }

pkg/compiler/backend_test.go

@@ -13,10 +13,10 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package backend_test
+package compiler_test
 
 import (
-	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
 
 	. "github.com/onsi/ginkgo"
@@ -24,7 +24,7 @@ import (
 )
 
 var _ = Describe("Docker image diffs", func() {
-	var b compiler.CompilerBackend
+	var b CompilerBackend
 
 	BeforeEach(func() {
 		b = NewSimpleDockerBackend()
@@ -32,12 +32,13 @@ var _ = Describe("Docker image diffs", func() {
 
 	Context("Generate diffs from docker images", func() {
 		It("Detect no changes", func() {
-			err := b.DownloadImage(compiler.CompilerBackendOptions{
+			opts := Options{
 				ImageName: "alpine:latest",
-			})
+			}
+			err := b.DownloadImage(opts)
 			Expect(err).ToNot(HaveOccurred())
 
-			layers, err := GenerateChanges(b, "alpine:latest", "alpine:latest")
+			layers, err := GenerateChanges(b, opts, opts)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(layers)).To(Equal(1))
 			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
@@ -46,16 +47,20 @@ var _ = Describe("Docker image diffs", func() {
 		})
 
 		It("Detects additions and changed files", func() {
-			err := b.DownloadImage(compiler.CompilerBackendOptions{
+			err := b.DownloadImage(Options{
 				ImageName: "quay.io/mocaccino/micro",
 			})
 			Expect(err).ToNot(HaveOccurred())
-			err = b.DownloadImage(compiler.CompilerBackendOptions{
+			err = b.DownloadImage(Options{
 				ImageName: "quay.io/mocaccino/extra",
 			})
 			Expect(err).ToNot(HaveOccurred())
 
-			layers, err := GenerateChanges(b, "quay.io/mocaccino/micro", "quay.io/mocaccino/extra")
+			layers, err := GenerateChanges(b, Options{
+				ImageName: "quay.io/mocaccino/micro",
+			}, Options{
+				ImageName: "quay.io/mocaccino/extra",
+			})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(layers)).To(Equal(1))
 

pkg/compiler/compiler_test.go

@@ -21,9 +21,11 @@ import (
 
 	. "github.com/mudler/luet/pkg/compiler"
 	sd "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -39,7 +41,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -50,16 +52,15 @@ var _ = Describe("Compiler", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -83,7 +84,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -98,12 +99,11 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 		})
@@ -118,7 +118,7 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/templates")
 			Expect(err).ToNot(HaveOccurred())
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 			pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -142,7 +142,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -159,15 +159,14 @@ var _ = Describe("Compiler", func() {
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2, spec3))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(3))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
@@ -199,7 +198,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -207,23 +206,22 @@ var _ = Describe("Compiler", func() {
 			Expect(err).ToNot(HaveOccurred())
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
-			artifacts2, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec2))
+			artifacts2, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts2)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			for _, artifact := range artifacts2 {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			Expect(helpers.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
@@ -241,7 +239,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -250,15 +248,14 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -276,7 +273,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -285,15 +282,14 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -312,7 +308,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -321,15 +317,14 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -348,7 +343,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -357,14 +352,13 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
@@ -382,7 +376,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -391,14 +385,13 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
@@ -416,7 +409,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -425,14 +418,13 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -454,7 +446,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -463,15 +455,14 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
@@ -495,7 +486,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -504,16 +495,15 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
@@ -539,7 +529,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -548,16 +538,15 @@ var _ = Describe("Compiler", func() {
 			//		Expect(err).ToNot(HaveOccurred())
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
@@ -581,7 +570,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -591,13 +580,13 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(2))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
@@ -619,7 +608,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
 			Expect(err).ToNot(HaveOccurred())
@@ -629,13 +618,13 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(6))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(6))
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
@@ -658,19 +647,19 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(4))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			// A deps on B, so A artifacts are here:
@@ -710,7 +699,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -722,17 +711,16 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-				for _, d := range artifact.GetDependencies() {
-					Expect(helpers.Exists(d.GetPath())).To(BeTrue())
-					Expect(helpers.Untar(d.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+				for _, d := range artifact.Dependencies {
+					Expect(helpers.Exists(d.Path)).To(BeTrue())
+					Expect(helpers.Untar(d.Path, tmpdir, false)).ToNot(HaveOccurred())
 				}
 			}
 
@@ -753,7 +741,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -765,12 +753,11 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 			Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
@@ -786,7 +773,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{
 				Name:     "dironly",
@@ -814,12 +801,10 @@ var _ = Describe("Compiler", func() {
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir2)
 
-			compiler.SetConcurrency(1)
-
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(2))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(0))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(0))
 
 			Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
@@ -841,11 +826,11 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
-			compiler.SetCompressionType(GZip)
+			compiler.Options.CompressionType = compression.GZip
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
 
 			tmpdir, err := ioutil.TempDir("", "tree")
@@ -853,12 +838,11 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
 			Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred())
@@ -877,7 +861,7 @@ var _ = Describe("Compiler", func() {
 			err := generalRecipe.Load("../../tests/fixtures/includeimage")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
 			Expect(err).ToNot(HaveOccurred())
@@ -896,11 +880,11 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
-			compiler.SetCompressionType(GZip)
+			compiler.Options.CompressionType = compression.GZip
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
 
 			tmpdir, err := ioutil.TempDir("", "tree")
@@ -908,20 +892,19 @@ var _ = Describe("Compiler", func() {
 			defer os.RemoveAll(tmpdir) // clean up
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
-			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-			Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
-			Expect(artifacts[0].GetFiles()).To(ContainElement("bin/busybox"))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
+			Expect(artifacts[0].Files).To(ContainElement("bin/busybox"))
 
 			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
 
 			art, err := LoadArtifactFromYaml(spec)
 			Expect(err).ToNot(HaveOccurred())
 
-			files := art.GetFiles()
+			files := art.Files
 			Expect(files).To(ContainElement("bin/busybox"))
 		})
 	})

pkg/compiler/imagehashtree.go

@@ -0,0 +1,126 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"fmt"
+
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	"github.com/mudler/luet/pkg/config"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	"github.com/pkg/errors"
+)
+
+type ImageHashTree struct {
+	Database      pkg.PackageDatabase
+	SolverOptions config.LuetSolverOptions
+}
+
+type PackageImageHashTree struct {
+	Target                       *solver.PackageAssert
+	Dependencies                 solver.PackagesAssertions
+	Solution                     solver.PackagesAssertions
+	dependencyBuilderImageHashes map[string]string
+	SourceHash                   string
+	BuilderImageHash             string
+}
+
+func NewHashTree(db pkg.PackageDatabase) *ImageHashTree {
+	return &ImageHashTree{
+		Database: db,
+	}
+}
+
+func (ht *PackageImageHashTree) DependencyBuildImage(p pkg.Package) (string, error) {
+	found, ok := ht.dependencyBuilderImageHashes[p.GetFingerPrint()]
+	if !ok {
+		return "", errors.New("package hash not found")
+	}
+	return found, nil
+}
+
+// TODO: ___ When computing the hash per package (and evaluating the sat solver solution tree part)
+// we should use the hash of each package  + its fingerprint  instead as a salt.
+// That's because the hash will be salted with its `build.yaml`.
+// In this way, we trigger recompilations if some dep of a target changes
+// a build.yaml, without touching the version
+func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) {
+	assertions, err := ht.resolve(cs, p)
+	if err != nil {
+		return nil, err
+	}
+	targetAssertion := assertions.Search(p.GetPackage().GetFingerPrint())
+
+	dependencies := assertions.Drop(p.GetPackage())
+	var sourceHash string
+	imageHashes := map[string]string{}
+	for _, assertion := range dependencies {
+		var depbuildImageTag string
+		compileSpec, err := cs.FromPackage(assertion.Package)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+		}
+		if compileSpec.GetImage() != "" {
+			depbuildImageTag = assertion.Hash.BuildHash
+		} else {
+			depbuildImageTag = ht.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash)
+		}
+		imageHashes[assertion.Package.GetFingerPrint()] = depbuildImageTag
+		sourceHash = assertion.Hash.PackageHash
+	}
+
+	return &PackageImageHashTree{
+		Dependencies:                 dependencies,
+		Target:                       targetAssertion,
+		SourceHash:                   sourceHash,
+		BuilderImageHash:             ht.genBuilderImageTag(p, targetAssertion.Hash.PackageHash),
+		dependencyBuilderImageHashes: imageHashes,
+		Solution:                     assertions,
+	}, nil
+}
+
+func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(packageImage))
+}
+
+// resolve computes the dependency tree of a compilation spec and returns solver assertions
+// in order to be able to compile the spec.
+func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
+	dependencies, err := cs.ComputeDepTree(p)
+	if err != nil {
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
+	}
+
+	assertions := solver.PackagesAssertions{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			nthsolution := dependencies.Cut(assertion.Package)
+			assertion.Hash = solver.PackageHash{
+				BuildHash:   nthsolution.HashFrom(assertion.Package),
+				PackageHash: nthsolution.AssertionHash(),
+			}
+			assertion.Package.SetTreeDir(p.Package.GetTreeDir())
+			assertions = append(assertions, assertion)
+		}
+	}
+
+	return assertions, nil
+}

pkg/compiler/imagehashtree_test.go

@@ -0,0 +1,94 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler_test
+
+import (
+	. "github.com/mudler/luet/pkg/compiler"
+	sd "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ImageHashTree", func() {
+	generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+	compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+	hashtree := NewHashTree(generalRecipe.GetDatabase())
+	Context("Simple package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			err := generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+
+		It("Calculates the hash correctly", func() {
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(packageHash.Target.Hash.BuildHash).To(Equal("6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-79462b60bf899ad79db63f194a3c9c2a"))
+		})
+	})
+
+	Context("complex package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
+			Expect(packageHash.SourceHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-37f4d05ba8a39525742ca364f69b4090"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("bb1d9a99c0c309a297c75b436504e664a42121fadbb4e035bda403cd418117aa"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+			Expect(assertionA.Hash.PackageHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+			Expect(assertionB.Hash.PackageHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hashB).To(Equal("6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"))
+		})
+	})
+
+})

pkg/compiler/interface.go

@@ -1,189 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler
-
-import (
-	"runtime"
-
-	"github.com/mudler/luet/pkg/config"
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
-)
-
-type Compiler interface {
-	Compile(bool, CompilationSpec) (Artifact, error)
-	CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
-	ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error)
-	SetConcurrency(i int)
-	FromPackage(pkg.Package) (CompilationSpec, error)
-	FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error)
-	SetBackend(CompilerBackend)
-	GetBackend() CompilerBackend
-	SetCompressionType(t CompressionImplementation)
-}
-
-type CompilerBackendOptions struct {
-	ImageName      string
-	SourcePath     string
-	DockerFileName string
-	Destination    string
-}
-
-type CompilerOptions struct {
-	ImageRepository          string
-	PullFirst, KeepImg, Push bool
-	Concurrency              int
-	CompressionType          CompressionImplementation
-	Clean                    bool
-	KeepImageExport          bool
-
-	OnlyDeps             bool
-	NoDeps               bool
-	SolverOptions        config.LuetSolverOptions
-	SkipIfMetadataExists bool
-
-	PackageTargetOnly bool
-}
-
-func NewDefaultCompilerOptions() *CompilerOptions {
-	return &CompilerOptions{
-		ImageRepository: "luet/cache",
-		PullFirst:       false,
-		Push:            false,
-		CompressionType: None,
-		KeepImg:         true,
-		Concurrency:     runtime.NumCPU(),
-		Clean:           true,
-		OnlyDeps:        false,
-		NoDeps:          false,
-	}
-}
-
-type CompilerBackend interface {
-	BuildImage(CompilerBackendOptions) error
-	ExportImage(CompilerBackendOptions) error
-	RemoveImage(CompilerBackendOptions) error
-	Changes(fromImage, toImage string) ([]ArtifactLayer, error)
-	ImageDefinitionToTar(CompilerBackendOptions) error
-	ExtractRootfs(opts CompilerBackendOptions, keepPerms bool) error
-
-	CopyImage(string, string) error
-	DownloadImage(opts CompilerBackendOptions) error
-
-	Push(opts CompilerBackendOptions) error
-
-	ImageExists(string) bool
-}
-
-type Artifact interface {
-	GetPath() string
-	SetPath(string)
-	GetDependencies() []Artifact
-	SetDependencies(d []Artifact)
-	GetSourceAssertion() solver.PackagesAssertions
-	SetSourceAssertion(as solver.PackagesAssertions)
-
-	SetCompileSpec(as CompilationSpec)
-	GetCompileSpec() CompilationSpec
-	WriteYaml(dst string) error
-	Unpack(dst string, keepPerms bool) error
-	Compress(src string, concurrency int) error
-	SetCompressionType(t CompressionImplementation)
-	FileList() ([]string, error)
-	Hash() error
-	Verify() error
-
-	SetFiles(f []string)
-	GetFiles() []string
-
-	GetChecksums() Checksums
-	SetChecksums(c Checksums)
-}
-
-type ArtifactNode struct {
-	Name string `json:"Name"`
-	Size int    `json:"Size"`
-}
-type ArtifactDiffs struct {
-	Additions []ArtifactNode `json:"Adds"`
-	Deletions []ArtifactNode `json:"Dels"`
-	Changes   []ArtifactNode `json:"Mods"`
-}
-type ArtifactLayer struct {
-	FromImage string        `json:"Image1"`
-	ToImage   string        `json:"Image2"`
-	Diffs     ArtifactDiffs `json:"Diff"`
-}
-type ArtifactLayerSummary struct {
-	FromImage   string `json:"image1"`
-	ToImage     string `json:"image2"`
-	AddFiles    int    `json:"add_files"`
-	AddSizes    int64  `json:"add_sizes"`
-	DelFiles    int    `json:"del_files"`
-	DelSizes    int64  `json:"del_sizes"`
-	ChangeFiles int    `json:"change_files"`
-	ChangeSizes int64  `json:"change_sizes"`
-}
-type ArtifactLayersSummary struct {
-	Layers []ArtifactLayerSummary `json:"summary"`
-}
-
-// CompilationSpec represent a compilation specification derived from a package
-type CompilationSpec interface {
-	ImageUnpack() bool // tells if the definition is just an image
-	GetIncludes() []string
-	GetExcludes() []string
-
-	RenderBuildImage() (string, error)
-	WriteBuildImageDefinition(string) error
-
-	RenderStepImage(image string) (string, error)
-	WriteStepImageDefinition(fromimage, path string) error
-
-	GetPackage() pkg.Package
-	BuildSteps() []string
-
-	GetSeedImage() string
-	SetSeedImage(string)
-
-	GetImage() string
-	SetImage(string)
-
-	SetOutputPath(string)
-	GetOutputPath() string
-	Rel(string) string
-
-	GetPreBuildSteps() []string
-
-	GetSourceAssertion() solver.PackagesAssertions
-	SetSourceAssertion(as solver.PackagesAssertions)
-
-	GetRetrieve() []string
-	CopyRetrieves(dest string) error
-
-	SetPackageDir(string)
-	GetPackageDir() string
-}
-
-type CompilationSpecs interface {
-	Unique() CompilationSpecs
-	Len() int
-	All() []CompilationSpec
-	Add(CompilationSpec)
-	Remove(s CompilationSpecs) CompilationSpecs
-}

pkg/compiler/types/artifact/artifact.go

@@ -13,12 +13,14 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler
+package artifact
 
 import (
 	"archive/tar"
 	"bufio"
 	"bytes"
+	"crypto/sha1"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -27,7 +29,7 @@ import (
 	"path/filepath"
 	"regexp"
 
-	system "github.com/docker/docker/pkg/system"
+	zstd "github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
@@ -35,6 +37,9 @@ import (
 	"sync"
 
 	bus "github.com/mudler/luet/pkg/bus"
+	backend "github.com/mudler/luet/pkg/compiler/backend"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -44,53 +49,30 @@ import (
 	yaml "gopkg.in/yaml.v2"
 )
 
-type CompressionImplementation string
-
-const (
-	None CompressionImplementation = "none" // e.g. tar for standard packages
-	GZip CompressionImplementation = "gzip"
-)
-
-type ArtifactIndex []Artifact
-
-func (i ArtifactIndex) CleanPath() ArtifactIndex {
-	newIndex := ArtifactIndex{}
-	for _, n := range i {
-		art := n.(*PackageArtifact)
-		// FIXME: This is a dup and makes difficult to add attributes to artifacts
-		newIndex = append(newIndex, &PackageArtifact{
-			Path:            path.Base(n.GetPath()),
-			SourceAssertion: art.SourceAssertion,
-			CompileSpec:     art.CompileSpec,
-			Dependencies:    art.Dependencies,
-			CompressionType: art.CompressionType,
-			Checksums:       art.Checksums,
-			Files:           art.Files,
-		})
-	}
-	return newIndex
-	//Update if exists, otherwise just create
-}
-
 //  When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository
 // which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact.
 // In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages.
 type PackageArtifact struct {
 	Path string `json:"path"`
 
-	Dependencies    []*PackageArtifact        `json:"dependencies"`
-	CompileSpec     *LuetCompilationSpec      `json:"compilationspec"`
-	Checksums       Checksums                 `json:"checksums"`
-	SourceAssertion solver.PackagesAssertions `json:"-"`
-	CompressionType CompressionImplementation `json:"compressiontype"`
-	Files           []string                  `json:"files"`
+	Dependencies    []*PackageArtifact                `json:"dependencies"`
+	CompileSpec     *compilerspec.LuetCompilationSpec `json:"compilationspec"`
+	Checksums       Checksums                         `json:"checksums"`
+	SourceAssertion solver.PackagesAssertions         `json:"-"`
+	CompressionType compression.Implementation        `json:"compressiontype"`
+	Files           []string                          `json:"files"`
 }
 
-func NewPackageArtifact(path string) Artifact {
-	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: None}
+func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
+	copy := *p
+	return &copy
 }
 
-func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
+func NewPackageArtifact(path string) *PackageArtifact {
+	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: compression.None}
+}
+
+func NewPackageArtifactFromYaml(data []byte) (*PackageArtifact, error) {
 	p := &PackageArtifact{Checksums: Checksums{}}
 	err := yaml.Unmarshal(data, &p)
 	if err != nil {
@@ -100,56 +82,20 @@ func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
 	return p, err
 }
 
-func LoadArtifactFromYaml(spec CompilationSpec) (Artifact, error) {
-
-	metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml"
-	dat, err := ioutil.ReadFile(spec.Rel(metaFile))
-	if err != nil {
-		return nil, errors.Wrap(err, "Error reading file "+metaFile)
-	}
-	art, err := NewPackageArtifactFromYaml(dat)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error writing file "+metaFile)
-	}
-	// It is relative, set it back to abs
-	art.SetPath(spec.Rel(art.GetPath()))
-	return art, nil
-}
-
-func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) {
-	a.CompressionType = t
-}
-
-func (a *PackageArtifact) GetChecksums() Checksums {
-	return a.Checksums
-}
-
-func (a *PackageArtifact) SetChecksums(c Checksums) {
-	a.Checksums = c
-}
-
-func (a *PackageArtifact) SetFiles(f []string) {
-	a.Files = f
-}
-
-func (a *PackageArtifact) GetFiles() []string {
-	return a.Files
-}
-
 func (a *PackageArtifact) Hash() error {
 	return a.Checksums.Generate(a)
 }
 
 func (a *PackageArtifact) Verify() error {
 	sum := Checksums{}
-	err := sum.Generate(a)
-	if err != nil {
+	if err := sum.Generate(a); err != nil {
 		return err
 	}
-	err = sum.Compare(a.Checksums)
-	if err != nil {
+
+	if err := sum.Compare(a.Checksums); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -179,8 +125,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 	}
 	//p := a.CompileSpec.GetPackage().GetPath()
 
-	mangle.GetCompileSpec().GetPackage().SetPath("")
-	for _, ass := range mangle.GetCompileSpec().GetSourceAssertion() {
+	mangle.CompileSpec.GetPackage().SetPath("")
+	for _, ass := range mangle.CompileSpec.GetSourceAssertion() {
 		ass.Package.SetPath("")
 	}
 
@@ -189,7 +135,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
 	}
 
-	err = ioutil.WriteFile(filepath.Join(dst, a.GetCompileSpec().GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm)
+	err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetMetadataFilePath()), data, os.ModePerm)
 	if err != nil {
 		return errors.Wrap(err, "While writing PackageArtifact YAML")
 	}
@@ -199,50 +145,102 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
 	return nil
 }
 
-func (a *PackageArtifact) GetSourceAssertion() solver.PackagesAssertions {
-	return a.SourceAssertion
+func (a *PackageArtifact) GetFileName() string {
+	return path.Base(a.Path)
 }
 
-func (a *PackageArtifact) SetCompileSpec(as CompilationSpec) {
-	a.CompileSpec = as.(*LuetCompilationSpec)
+func (a *PackageArtifact) genDockerfile() string {
+	return `
+FROM scratch
+COPY . /`
 }
 
-func (a *PackageArtifact) GetCompileSpec() CompilationSpec {
-	return a.CompileSpec
-}
-
-func (a *PackageArtifact) SetSourceAssertion(as solver.PackagesAssertions) {
-	a.SourceAssertion = as
-}
-
-func (a *PackageArtifact) GetDependencies() []Artifact {
-	ret := []Artifact{}
-	for _, d := range a.Dependencies {
-		ret = append(ret, d)
+// CreateArtifactForFile creates a new artifact from the given file
+func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
+	if _, err := os.Stat(s); os.IsNotExist(err) {
+		return nil, errors.Wrap(err, "artifact path doesn't exist")
 	}
-	return ret
-}
-
-func (a *PackageArtifact) SetDependencies(d []Artifact) {
-	ret := []*PackageArtifact{}
-	for _, dd := range d {
-		ret = append(ret, dd.(*PackageArtifact))
+	fileName := path.Base(s)
+	archive, err := LuetCfg.GetSystem().TempDir("archive")
+	if err != nil {
+		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
 	}
-	a.Dependencies = ret
+	defer os.RemoveAll(archive) // clean up
+	dst := filepath.Join(archive, fileName)
+	if err := helpers.CopyFile(s, dst); err != nil {
+		return nil, errors.Wrapf(err, "error while copying %s to %s", s, dst)
+	}
+
+	artifact, err := LuetCfg.GetSystem().TempDir("artifact")
+	if err != nil {
+		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
+	}
+	a := &PackageArtifact{Path: filepath.Join(artifact, fileName)}
+
+	for _, o := range opts {
+		o(a)
+	}
+
+	return a, a.Compress(archive, 1)
 }
 
-func (a *PackageArtifact) GetPath() string {
-	return a.Path
+type ImageBuilder interface {
+	BuildImage(backend.Options) error
 }
 
-func (a *PackageArtifact) SetPath(p string) {
-	a.Path = p
+// GenerateFinalImage takes an artifact and builds a Docker image with its content
+func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) {
+	builderOpts := backend.Options{}
+	archive, err := LuetCfg.GetSystem().TempDir("archive")
+	if err != nil {
+		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	}
+	defer os.RemoveAll(archive) // clean up
+
+	uncompressedFiles := filepath.Join(archive, "files")
+	dockerFile := filepath.Join(archive, "Dockerfile")
+
+	if err := os.MkdirAll(uncompressedFiles, os.ModePerm); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	}
+
+	if err := a.Unpack(uncompressedFiles, keepPerms); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
+	}
+
+	empty, err := helpers.DirectoryIsEmpty(uncompressedFiles)
+	if err != nil {
+		return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles)
+	}
+
+	// See https://github.com/moby/moby/issues/38039.
+	// We can't generate FROM scratch empty images. Docker will refuse to export them
+	// workaround: Inject a .virtual empty file
+	if empty {
+		helpers.Touch(filepath.Join(uncompressedFiles, ".virtual"))
+	}
+
+	data := a.genDockerfile()
+	if err := ioutil.WriteFile(dockerFile, []byte(data), 0644); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path)
+	}
+
+	builderOpts = backend.Options{
+		ImageName:      imageName,
+		SourcePath:     archive,
+		DockerFileName: dockerFile,
+		Context:        uncompressedFiles,
+	}
+	return builderOpts, b.BuildImage(builderOpts)
 }
 
-// Compress Archives and compress (TODO) to the artifact path
+// Compress is responsible to archive and compress to the artifact Path.
+// It accepts a source path, which is the content to be archived/compressed
+// and a concurrency parameter.
 func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	switch a.CompressionType {
-	case GZip:
+
+	case compression.Zstandard:
 		err := helpers.Tar(src, a.Path)
 		if err != nil {
 			return err
@@ -253,7 +251,45 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 		}
 		defer original.Close()
 
-		gzipfile := a.Path + ".gz"
+		zstdFile := a.getCompressedName()
+		bufferedReader := bufio.NewReader(original)
+
+		// Open a file for writing.
+		dst, err := os.Create(zstdFile)
+		if err != nil {
+			return err
+		}
+
+		enc, err := zstd.NewWriter(dst)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(enc, bufferedReader)
+		if err != nil {
+			enc.Close()
+			return err
+		}
+		if err := enc.Close(); err != nil {
+			return err
+		}
+
+		os.RemoveAll(a.Path) // Remove original
+		Debug("Removed artifact", a.Path)
+
+		a.Path = zstdFile
+		return nil
+	case compression.GZip:
+		err := helpers.Tar(src, a.Path)
+		if err != nil {
+			return err
+		}
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return err
+		}
+		defer original.Close()
+
+		gzipfile := a.getCompressedName()
 		bufferedReader := bufio.NewReader(original)
 
 		// Open a file for writing.
@@ -263,7 +299,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 		}
 		// Create gzip writer.
 		w := gzip.NewWriter(dst)
-		w.SetConcurrency(concurrency, 10)
+		w.SetConcurrency(1<<20, concurrency)
 		defer w.Close()
 		defer dst.Close()
 		_, err = io.Copy(w, bufferedReader)
@@ -272,6 +308,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 		}
 		w.Close()
 		os.RemoveAll(a.Path) // Remove original
+		Debug("Removed artifact", a.Path)
 		//	a.CompressedPath = gzipfile
 		a.Path = gzipfile
 		return nil
@@ -279,10 +316,50 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.Tar(src, a.Path)
-
+		return helpers.Tar(src, a.getCompressedName())
 	}
-	return errors.New("Compression type must be supplied")
+}
+
+func (a *PackageArtifact) getCompressedName() string {
+	switch a.CompressionType {
+	case compression.Zstandard:
+		return a.Path + ".zst"
+
+	case compression.GZip:
+		return a.Path + ".gz"
+	}
+	return a.Path
+}
+
+// GetUncompressedName returns the artifact path without the extension suffix
+func (a *PackageArtifact) GetUncompressedName() string {
+	switch a.CompressionType {
+	case compression.Zstandard, compression.GZip:
+		return strings.TrimSuffix(a.Path, filepath.Ext(a.Path))
+	}
+	return a.Path
+}
+
+func hashContent(bv []byte) string {
+	hasher := sha1.New()
+	hasher.Write(bv)
+	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	return sha
+}
+
+func hashFileContent(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	h := sha1.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+
+	return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil
 }
 
 func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
@@ -296,6 +373,7 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return nil, nil, err
 		}
 	}
+	tarHash := hashContent(buffer.Bytes())
 
 	// If file is not present on archive but is defined on mods
 	// I receive the callback. Prevent nil exception.
@@ -308,8 +386,21 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return header, buffer.Bytes(), nil
 		}
 
+		existingHash := ""
+		f, err := os.Lstat(destPath)
+		if err == nil {
+			Debug("File exists already, computing hash for", destPath)
+			hash, herr := hashFileContent(destPath)
+			if herr == nil {
+				existingHash = hash
+			}
+		}
+
+		Debug("Existing file hash: ", existingHash, "Tar file hashsum: ", tarHash)
+		// We want to protect file only if the hash of the files are differing OR the file size are
+		differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
 		// Check if exists
-		if helpers.Exists(destPath) {
+		if helpers.Exists(destPath) && differs {
 			for i := 1; i < 1000; i++ {
 				name := filepath.Join(filepath.Join(filepath.Dir(path),
 					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
@@ -367,13 +458,47 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
 
 	switch a.CompressionType {
-	case GZip:
+	case compression.Zstandard:
 		// Create the uncompressed archive
-		archive, err := os.Create(a.GetPath() + ".uncompressed")
+		archive, err := os.Create(a.Path + ".uncompressed")
 		if err != nil {
 			return err
 		}
-		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer os.RemoveAll(a.Path + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+
+		d, err := zstd.NewReader(bufferedReader)
+		if err != nil {
+			return err
+		}
+		defer d.Close()
+
+		_, err = io.Copy(archive, d)
+		if err != nil {
+			return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
+		}
+
+		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
+			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
+		if err != nil {
+			return err
+		}
+		return nil
+	case compression.GZip:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.Path + ".uncompressed")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(a.Path + ".uncompressed")
 		defer archive.Close()
 
 		original, err := os.Open(a.Path)
@@ -391,10 +516,10 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 
 		_, err = io.Copy(archive, r)
 		if err != nil {
-			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
+			return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
 		}
 
-		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
 			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
@@ -402,7 +527,7 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 		return nil
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.UntarProtect(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner,
+		return helpers.UntarProtect(a.Path, dst, LuetCfg.GetGeneral().SameOwner,
 			protectedFiles, tarModifier)
 	}
 	return errors.New("Compression type must be supplied")
@@ -412,13 +537,34 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 func (a *PackageArtifact) FileList() ([]string, error) {
 	var tr *tar.Reader
 	switch a.CompressionType {
-	case GZip:
-		// Create the uncompressed archive
-		archive, err := os.Create(a.GetPath() + ".uncompressed")
+	case compression.Zstandard:
+		archive, err := os.Create(a.Path + ".uncompressed")
 		if err != nil {
 			return []string{}, err
 		}
-		defer os.RemoveAll(a.GetPath() + ".uncompressed")
+		defer os.RemoveAll(a.Path + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := zstd.NewReader(bufferedReader)
+		if err != nil {
+			return []string{}, err
+		}
+		defer r.Close()
+		tr = tar.NewReader(r)
+	case compression.GZip:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.Path + ".uncompressed")
+		if err != nil {
+			return []string{}, err
+		}
+		defer os.RemoveAll(a.Path + ".uncompressed")
 		defer archive.Close()
 
 		original, err := os.Open(a.Path)
@@ -437,7 +583,7 @@ func (a *PackageArtifact) FileList() ([]string, error) {
 
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		tarFile, err := os.Open(a.GetPath())
+		tarFile, err := os.Open(a.Path)
 		if err != nil {
 			return []string{}, errors.Wrap(err, "Could not open package archive")
 		}
@@ -474,53 +620,51 @@ type CopyJob struct {
 	Artifact string
 }
 
-func copyXattr(srcPath, dstPath, attr string) error {
-	data, err := system.Lgetxattr(srcPath, attr)
-	if err != nil {
-		return err
-	}
-	if data != nil {
-		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func doCopyXattrs(srcPath, dstPath string) error {
-	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
-		return err
-	}
-
-	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
-}
-
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
 	for job := range s {
-		//Info("#"+strconv.Itoa(i), "copying", job.Src, "to", job.Dst)
-		// if dir, err := helpers.IsDirectory(job.Src); err == nil && dir {
-		// 	err = helpers.CopyDir(job.Src, job.Dst)
-		// 	if err != nil {
-		// 		Warning("Error copying dir", job, err)
-		// 	}
-		// 	continue
-		// }
-
 		_, err := os.Lstat(job.Dst)
 		if err != nil {
 			Debug("Copying ", job.Src)
-			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
+			if err := helpers.DeepCopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
-			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
 
+func compileRegexes(regexes []string) []*regexp.Regexp {
+	var result []*regexp.Regexp
+	for _, i := range regexes {
+		r, e := regexp.Compile(i)
+		if e != nil {
+			Warning("Failed compiling regex:", e)
+			continue
+		}
+		result = append(result, r)
+	}
+	return result
+}
+
+type ArtifactNode struct {
+	Name string `json:"Name"`
+	Size int    `json:"Size"`
+}
+type ArtifactDiffs struct {
+	Additions []ArtifactNode `json:"Adds"`
+	Deletions []ArtifactNode `json:"Dels"`
+	Changes   []ArtifactNode `json:"Mods"`
+}
+
+type ArtifactLayer struct {
+	FromImage string        `json:"Image1"`
+	ToImage   string        `json:"Image2"`
+	Diffs     ArtifactDiffs `json:"Diff"`
+}
+
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t CompressionImplementation) (Artifact, error) {
+func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) {
 
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
@@ -552,15 +696,7 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 	// Handle includes in spec. If specified they filter what gets in the package
 
 	if len(includes) > 0 && len(excludes) == 0 {
-		var includeRegexp []*regexp.Regexp
-		for _, i := range includes {
-			r, e := regexp.Compile(i)
-			if e != nil {
-				Warning("Failed compiling regex:", e)
-				continue
-			}
-			includeRegexp = append(includeRegexp, r)
-		}
+		includeRegexp := compileRegexes(includes)
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 		ADDS:
@@ -581,15 +717,7 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 		}
 
 	} else if len(includes) == 0 && len(excludes) != 0 {
-		var excludeRegexp []*regexp.Regexp
-		for _, i := range excludes {
-			r, e := regexp.Compile(i)
-			if e != nil {
-				Warning("Failed compiling regex:", e)
-				continue
-			}
-			excludeRegexp = append(excludeRegexp, r)
-		}
+		excludeRegexp := compileRegexes(excludes)
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 		ADD:
@@ -610,25 +738,8 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 		}
 
 	} else if len(includes) != 0 && len(excludes) != 0 {
-
-		var includeRegexp []*regexp.Regexp
-		for _, i := range includes {
-			r, e := regexp.Compile(i)
-			if e != nil {
-				Warning("Failed compiling regex:", e)
-				continue
-			}
-			includeRegexp = append(includeRegexp, r)
-		}
-		var excludeRegexp []*regexp.Regexp
-		for _, i := range excludes {
-			r, e := regexp.Compile(i)
-			if e != nil {
-				Warning("Failed compiling regex:", e)
-				continue
-			}
-			excludeRegexp = append(excludeRegexp, r)
-		}
+		includeRegexp := compileRegexes(includes)
+		excludeRegexp := compileRegexes(excludes)
 
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
@@ -675,45 +786,10 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 	wg.Wait()
 
 	a := NewPackageArtifact(dst)
-	a.SetCompressionType(t)
+	a.CompressionType = t
 	err = a.Compress(archive, concurrency)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}
 	return a, nil
 }
-
-func ComputeArtifactLayerSummary(diffs []ArtifactLayer) ArtifactLayersSummary {
-
-	ans := ArtifactLayersSummary{
-		Layers: make([]ArtifactLayerSummary, 0),
-	}
-
-	for _, layer := range diffs {
-		sum := ArtifactLayerSummary{
-			FromImage:   layer.FromImage,
-			ToImage:     layer.ToImage,
-			AddFiles:    0,
-			AddSizes:    0,
-			DelFiles:    0,
-			DelSizes:    0,
-			ChangeFiles: 0,
-			ChangeSizes: 0,
-		}
-		for _, a := range layer.Diffs.Additions {
-			sum.AddFiles++
-			sum.AddSizes += int64(a.Size)
-		}
-		for _, d := range layer.Diffs.Deletions {
-			sum.DelFiles++
-			sum.DelSizes += int64(d.Size)
-		}
-		for _, c := range layer.Diffs.Changes {
-			sum.ChangeFiles++
-			sum.ChangeSizes += int64(c.Size)
-		}
-		ans.Layers = append(ans.Layers, sum)
-	}
-
-	return ans
-}

pkg/compiler/types/artifact/artifact_test.go

@@ -0,0 +1,268 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+	backend "github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/compiler/types/artifact"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+
+	. "github.com/mudler/luet/pkg/compiler"
+	helpers "github.com/mudler/luet/pkg/helpers"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Artifact", func() {
+	Context("Simple package build definition", func() {
+		It("Generates a verified delta", func() {
+
+			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/buildtree")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
+			Expect(lspec.Image).To(Equal("luet/base"))
+			Expect(lspec.Seed).To(Equal("alpine"))
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpdir2, err := ioutil.TempDir(os.TempDir(), "tree2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir2) // clean up
+
+			unpacked, err := ioutil.TempDir(os.TempDir(), "unpacked")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(unpacked) // clean up
+
+			rootfs, err := ioutil.TempDir(os.TempDir(), "rootfs")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(rootfs) // clean up
+
+			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin`))
+			b := NewSimpleDockerBackend()
+			opts := backend.Options{
+				ImageName:      "luet/base",
+				SourcePath:     tmpdir,
+				DockerFileName: "Dockerfile",
+				Destination:    filepath.Join(tmpdir2, "output1.tar"),
+			}
+			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
+
+			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(dockerfile).To(Equal(`
+FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin
+RUN echo foo > /test
+RUN echo bar > /test2`))
+			opts2 := backend.Options{
+				ImageName:      "test",
+				SourcePath:     tmpdir,
+				DockerFileName: "LuetDockerfile",
+				Destination:    filepath.Join(tmpdir, "output2.tar"),
+			}
+			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
+			diffs, err := compiler.GenerateChanges(b, opts, opts2)
+			Expect(err).ToNot(HaveOccurred())
+
+			artifacts := []ArtifactNode{{
+				Name: "/luetbuild/LuetDockerfile",
+				Size: 175,
+			}}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
+			Expect(diffs).To(Equal(
+				[]ArtifactLayer{{
+					FromImage: "luet/base",
+					ToImage:   "test",
+					Diffs: ArtifactDiffs{
+						Additions: artifacts,
+					},
+				}}))
+			err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			a, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
+			err = helpers.Untar(a.Path, unpacked, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
+			Expect(helpers.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
+			content1, err := helpers.Read(filepath.Join(unpacked, "test"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("foo\n"))
+			content2, err := helpers.Read(filepath.Join(unpacked, "test2"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content2).To(Equal("bar\n"))
+
+			err = a.Hash()
+			Expect(err).ToNot(HaveOccurred())
+			err = a.Verify()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
+
+			err = a.Verify()
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("Generates packages images", func() {
+			b := NewSimpleDockerBackend()
+			imageprefix := "foo/"
+			testString := []byte(`funky test data`)
+
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "artifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpWork, err := ioutil.TempDir(os.TempDir(), "artifact2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpWork) // clean up
+
+			Expect(os.MkdirAll(filepath.Join(tmpdir, "foo", "bar"), os.ModePerm)).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "test"), testString, 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "foo", "bar", "test"), testString, 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}
+
+			err = a.Compress(tmpdir, 1)
+			Expect(err).ToNot(HaveOccurred())
+			resultingImage := imageprefix + "foo--1.0"
+			opts, err := a.GenerateFinalImage(resultingImage, b, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(opts.ImageName).To(Equal(resultingImage))
+
+			Expect(b.ImageExists(resultingImage)).To(BeTrue())
+
+			result, err := ioutil.TempDir(os.TempDir(), "result")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(result) // clean up
+
+			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			content, err := ioutil.ReadFile(filepath.Join(result, "test"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(content).To(Equal(testString))
+
+			content, err = ioutil.ReadFile(filepath.Join(result, "foo", "bar", "test"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(content).To(Equal(testString))
+		})
+
+		It("Generates empty packages images", func() {
+			b := NewSimpleDockerBackend()
+			imageprefix := "foo/"
+
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "artifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpWork, err := ioutil.TempDir(os.TempDir(), "artifact2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpWork) // clean up
+
+			a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}
+
+			err = a.Compress(tmpdir, 1)
+			Expect(err).ToNot(HaveOccurred())
+			resultingImage := imageprefix + "foo--1.0"
+			opts, err := a.GenerateFinalImage(resultingImage, b, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(opts.ImageName).To(Equal(resultingImage))
+
+			Expect(b.ImageExists(resultingImage)).To(BeTrue())
+
+			result, err := ioutil.TempDir(os.TempDir(), "result")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(result) // clean up
+
+			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.DirectoryIsEmpty(result)).To(BeFalse())
+			content, err := ioutil.ReadFile(filepath.Join(result, ".virtual"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(content)).To(Equal(""))
+		})
+
+		It("Retrieves uncompressed name", func() {
+			a := NewPackageArtifact("foo.tar.gz")
+			a.CompressionType = (compression.GZip)
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+
+			a = NewPackageArtifact("foo.tar.zst")
+			a.CompressionType = compression.Zstandard
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+
+			a = NewPackageArtifact("foo.tar")
+			a.CompressionType = compression.None
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+		})
+	})
+})

pkg/compiler/types/artifact/checksum.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler
+package artifact
 
 import (
 
@@ -43,7 +43,7 @@ type HashOptions struct {
 }
 
 // Generate generates all Checksums supported for the artifact
-func (c *Checksums) Generate(a Artifact) error {
+func (c *Checksums) Generate(a *PackageArtifact) error {
 	return c.generateSHA256(a)
 }
 
@@ -56,13 +56,13 @@ func (c Checksums) Compare(d Checksums) error {
 	return nil
 }
 
-func (c *Checksums) generateSHA256(a Artifact) error {
+func (c *Checksums) generateSHA256(a *PackageArtifact) error {
 	return c.generateSum(a, HashOptions{Hasher: sha256.New(), Type: SHA256})
 }
 
-func (c *Checksums) generateSum(a Artifact, opts HashOptions) error {
+func (c *Checksums) generateSum(a *PackageArtifact, opts HashOptions) error {
 
-	f, err := os.Open(a.GetPath())
+	f, err := os.Open(a.Path)
 	if err != nil {
 		return err
 	}

pkg/compiler/types/artifact/checksum_test.go

@@ -13,13 +13,13 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler_test
+package artifact_test
 
 import (
 	"io/ioutil"
 	"os"
 
-	. "github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/types/artifact"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

pkg/compiler/types/compression/compression.go

@@ -0,0 +1,9 @@
+package compression
+
+type Implementation string
+
+const (
+	None      Implementation = "none" // e.g. tar for standard packages
+	GZip      Implementation = "gzip"
+	Zstandard Implementation = "zstd"
+)

pkg/compiler/types/options/compiler_options.go

@@ -0,0 +1,199 @@
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package options
+
+import (
+	"runtime"
+
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/solver"
+)
+
+type Compiler struct {
+	PushImageRepository      string
+	PullImageRepository      []string
+	PullFirst, KeepImg, Push bool
+	Concurrency              int
+	CompressionType          compression.Implementation
+
+	Wait            bool
+	OnlyDeps        bool
+	NoDeps          bool
+	SolverOptions   config.LuetSolverOptions
+	BuildValuesFile []string
+	BuildValues     []map[string]interface{}
+
+	PackageTargetOnly bool
+	Rebuild           bool
+
+	BackendArgs []string
+
+	BackendType string
+}
+
+func NewDefaultCompiler() *Compiler {
+	return &Compiler{
+		PushImageRepository: "luet/cache",
+		PullFirst:           false,
+		Push:                false,
+		CompressionType:     compression.None,
+		KeepImg:             true,
+		Concurrency:         runtime.NumCPU(),
+		OnlyDeps:            false,
+		NoDeps:              false,
+		SolverOptions:       config.LuetSolverOptions{Options: solver.Options{Concurrency: 1, Type: solver.SingleCoreSimple}},
+	}
+}
+
+type Option func(cfg *Compiler) error
+
+func (cfg *Compiler) Apply(opts ...Option) error {
+	for _, opt := range opts {
+		if opt == nil {
+			continue
+		}
+		if err := opt(cfg); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func WithOptions(opt *Compiler) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg = opt
+		return nil
+	}
+}
+
+func WithBackendType(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BackendType = r
+		return nil
+	}
+}
+
+func WithBuildValues(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BuildValuesFile = r
+		return nil
+	}
+}
+
+func WithPullRepositories(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PullImageRepository = r
+		return nil
+	}
+}
+
+func WithPushRepository(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		if len(cfg.PullImageRepository) == 0 {
+			cfg.PullImageRepository = []string{cfg.PushImageRepository}
+		}
+		cfg.PushImageRepository = r
+		return nil
+	}
+}
+
+func BackendArgs(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BackendArgs = r
+		return nil
+	}
+}
+
+func PullFirst(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PullFirst = b
+		return nil
+	}
+}
+
+func KeepImg(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.KeepImg = b
+		return nil
+	}
+}
+
+func Rebuild(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Rebuild = b
+		return nil
+	}
+}
+
+func PushImages(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Push = b
+		return nil
+	}
+}
+
+func Wait(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Wait = b
+		return nil
+	}
+}
+
+func OnlyDeps(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.OnlyDeps = b
+		return nil
+	}
+}
+
+func OnlyTarget(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PackageTargetOnly = b
+		return nil
+	}
+}
+
+func NoDeps(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.NoDeps = b
+		return nil
+	}
+}
+
+func Concurrency(i int) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		if i == 0 {
+			i = runtime.NumCPU()
+		}
+		cfg.Concurrency = i
+		return nil
+	}
+}
+
+func WithCompressionType(t compression.Implementation) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.CompressionType = t
+		return nil
+	}
+}
+
+func WithSolverOptions(c config.LuetSolverOptions) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.SolverOptions = c
+		return nil
+	}
+}

pkg/compiler/types/spec/spec.go

@@ -13,12 +13,15 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler
+package compilerspec
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
 
+	options "github.com/mudler/luet/pkg/compiler/types/options"
+
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/otiai10/copy"
@@ -27,7 +30,7 @@ import (
 
 type LuetCompilationspecs []LuetCompilationSpec
 
-func NewLuetCompilationspecs(s ...CompilationSpec) CompilationSpecs {
+func NewLuetCompilationspecs(s ...*LuetCompilationSpec) *LuetCompilationspecs {
 	all := LuetCompilationspecs{}
 
 	for _, spec := range s {
@@ -40,7 +43,7 @@ func (specs LuetCompilationspecs) Len() int {
 	return len(specs)
 }
 
-func (specs *LuetCompilationspecs) Remove(s CompilationSpecs) CompilationSpecs {
+func (specs *LuetCompilationspecs) Remove(s *LuetCompilationspecs) *LuetCompilationspecs {
 	newSpecs := LuetCompilationspecs{}
 SPECS:
 	for _, spec := range specs.All() {
@@ -54,16 +57,12 @@ SPECS:
 	return &newSpecs
 }
 
-func (specs *LuetCompilationspecs) Add(s CompilationSpec) {
-	c, ok := s.(*LuetCompilationSpec)
-	if !ok {
-		panic("LuetCompilationspecs supports only []LuetCompilationSpec")
-	}
-	*specs = append(*specs, *c)
+func (specs *LuetCompilationspecs) Add(s *LuetCompilationSpec) {
+	*specs = append(*specs, *s)
 }
 
-func (specs *LuetCompilationspecs) All() []CompilationSpec {
-	var cspecs []CompilationSpec
+func (specs *LuetCompilationspecs) All() []*LuetCompilationSpec {
+	var cspecs []*LuetCompilationSpec
 	for i, _ := range *specs {
 		f := (*specs)[i]
 		cspecs = append(cspecs, &f)
@@ -72,7 +71,7 @@ func (specs *LuetCompilationspecs) All() []CompilationSpec {
 	return cspecs
 }
 
-func (specs *LuetCompilationspecs) Unique() CompilationSpecs {
+func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
 	newSpecs := LuetCompilationspecs{}
 	seen := map[string]bool{}
 
@@ -87,6 +86,13 @@ func (specs *LuetCompilationspecs) Unique() CompilationSpecs {
 	return &newSpecs
 }
 
+type CopyField struct {
+	Package     *pkg.DefaultPackage `json:"package"`
+	Image       string              `json:"image"`
+	Source      string              `json:"src"`
+	Destination string              `json:"dst"`
+}
+
 type LuetCompilationSpec struct {
 	Steps           []string                  `json:"steps"` // Are run inside a container and the result layer diff is saved
 	Env             []string                  `json:"env"`
@@ -103,9 +109,13 @@ type LuetCompilationSpec struct {
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
 	Excludes   []string `json:"excludes"`
+
+	BuildOptions *options.Compiler `json:"build_options"`
+
+	Copy []CopyField `json:"copy"`
 }
 
-func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
+func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
 	var spec LuetCompilationSpec
 	err := yaml.Unmarshal(b, &spec)
 	if err != nil {
@@ -114,12 +124,16 @@ func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
 	spec.Package = p.(*pkg.DefaultPackage)
 	return &spec, nil
 }
-func (a *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
-	return a.SourceAssertion
+func (cs *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
+	return cs.SourceAssertion
 }
 
-func (a *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) {
-	a.SourceAssertion = as
+func (cs *LuetCompilationSpec) SetBuildOptions(b options.Compiler) {
+	cs.BuildOptions = &b
+}
+
+func (cs *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) {
+	cs.SourceAssertion = as
 }
 func (cs *LuetCompilationSpec) GetPackage() pkg.Package {
 	return cs.Package
@@ -157,6 +171,12 @@ func (cs *LuetCompilationSpec) GetRetrieve() []string {
 	return cs.Retrieve
 }
 
+// IsVirtual returns true if the spec is virtual.
+// A spec is virtual if the package is empty, and it has no image source to unpack from.
+func (cs *LuetCompilationSpec) IsVirtual() bool {
+	return cs.EmptyPackage() && !cs.HasImageSource()
+}
+
 func (cs *LuetCompilationSpec) GetSeedImage() string {
 	return cs.Seed
 }
@@ -185,6 +205,27 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 	cs.Seed = s
 }
 
+func (cs *LuetCompilationSpec) EmptyPackage() bool {
+	return len(cs.BuildSteps()) == 0 && len(cs.GetPreBuildSteps()) == 0 && !cs.UnpackedPackage()
+}
+
+func (cs *LuetCompilationSpec) UnpackedPackage() bool {
+	// If package_dir was specified in the spec, we want to treat the content of the directory
+	// as the root of our archive.  ImageUnpack is implied to be true. override it
+	unpack := cs.ImageUnpack()
+	if cs.GetPackageDir() != "" {
+		unpack = true
+	}
+	return unpack
+}
+
+// HasImageSource returns true when the compilation spec has an image source.
+// a compilation spec has an image source when it depends on other packages or have a source image
+// explictly supplied
+func (cs *LuetCompilationSpec) HasImageSource() bool {
+	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != ""
+}
+
 func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
 	var err error
 	if len(cs.Retrieve) > 0 {
@@ -203,10 +244,9 @@ func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
 	return err
 }
 
-// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
-func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
+func (cs *LuetCompilationSpec) genDockerfile(image string, steps []string) string {
 	spec := `
-FROM ` + cs.GetSeedImage() + `
+FROM ` + image + `
 COPY . /luetbuild
 WORKDIR /luetbuild
 ENV PACKAGE_NAME=` + cs.Package.GetName() + `
@@ -226,36 +266,33 @@ ADD ` + s + ` /luetbuild/`
 		}
 	}
 
+	for _, c := range cs.Copy {
+		if c.Image != "" {
+			spec = spec + fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
+		}
+	}
+
 	for _, s := range cs.Env {
 		spec = spec + `
 ENV ` + s
 	}
 
-	for _, s := range cs.GetPreBuildSteps() {
+	for _, s := range steps {
 		spec = spec + `
 RUN ` + s
 	}
-	return spec, nil
+	return spec
 }
 
-// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
+// RenderBuildImage renders the dockerfile of the image used as a pre-build step
+func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
+	return cs.genDockerfile(cs.GetSeedImage(), cs.GetPreBuildSteps()), nil
+
+}
+
+// RenderStepImage renders the dockerfile used for the image used for building the package
 func (cs *LuetCompilationSpec) RenderStepImage(image string) (string, error) {
-	spec := `
-FROM ` + image + `
-ENV PACKAGE_NAME=` + cs.Package.GetName() + `
-ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
-ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
-
-	for _, s := range cs.Env {
-		spec = spec + `
-ENV ` + s
-	}
-	for _, s := range cs.BuildSteps() {
-		spec = spec + `
-RUN ` + s
-	}
-
-	return spec, nil
+	return cs.genDockerfile(image, cs.BuildSteps()), nil
 }
 
 func (cs *LuetCompilationSpec) WriteBuildImageDefinition(path string) error {

pkg/compiler/types/spec/spec_test.go

@@ -13,17 +13,18 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package compiler_test
+package compilerspec_test
 
 import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -32,25 +33,45 @@ import (
 var _ = Describe("Spec", func() {
 	Context("Luet specs", func() {
 		It("Allows normal operations", func() {
-			testSpec := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
-			testSpec2 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}}
-			testSpec3 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}}
-			testSpec4 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
+			testSpec := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
+			testSpec2 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}}
+			testSpec3 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}}
+			testSpec4 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
 
-			specs := NewLuetCompilationspecs(testSpec, testSpec2)
+			specs := compilerspec.NewLuetCompilationspecs(testSpec, testSpec2)
 			Expect(specs.Len()).To(Equal(2))
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2}))
+			Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2}))
 			specs.Add(testSpec3)
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
+			Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
 			specs.Add(testSpec4)
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
+			Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
 			newSpec := specs.Unique()
-			Expect(newSpec.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
+			Expect(newSpec.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
 
-			newSpec2 := specs.Remove(NewLuetCompilationspecs(testSpec, testSpec2))
-			Expect(newSpec2.All()).To(Equal([]CompilationSpec{testSpec3}))
+			newSpec2 := specs.Remove(compilerspec.NewLuetCompilationspecs(testSpec, testSpec2))
+			Expect(newSpec2.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec3}))
 
 		})
+		Context("virtuals", func() {
+			When("is empty", func() {
+				It("is virtual", func() {
+					spec := &compilerspec.LuetCompilationSpec{}
+					Expect(spec.IsVirtual()).To(BeTrue())
+				})
+			})
+			When("has defined steps", func() {
+				It("is not a virtual", func() {
+					spec := &compilerspec.LuetCompilationSpec{Steps: []string{"foo"}}
+					Expect(spec.IsVirtual()).To(BeFalse())
+				})
+			})
+			When("has defined image", func() {
+				It("is not a virtual", func() {
+					spec := &compilerspec.LuetCompilationSpec{Image: "foo"}
+					Expect(spec.IsVirtual()).To(BeFalse())
+				})
+			})
+		})
 	})
 
 	Context("Simple package build definition", func() {
@@ -62,13 +83,10 @@ var _ = Describe("Spec", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
 			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 			Expect(lspec.Image).To(Equal("luet/base"))
 			Expect(lspec.Seed).To(Equal("alpine"))
@@ -96,6 +114,8 @@ ENV test=1`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin
@@ -115,13 +135,10 @@ RUN echo bar > /test2`))
 
 		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-		spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+		lspec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
 		Expect(err).ToNot(HaveOccurred())
 
-		lspec, ok := spec.(*LuetCompilationSpec)
-		Expect(ok).To(BeTrue())
-
 		Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 		Expect(lspec.Image).To(Equal("luet/base"))
 		Expect(lspec.Seed).To(Equal("alpine"))
@@ -168,9 +185,13 @@ ENV test=1`))
 
 		Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
 ENV PACKAGE_NAME=a
 ENV PACKAGE_VERSION=1.0
 ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
 ENV test=1
 RUN echo foo > /test
 RUN echo bar > /test2`))

pkg/config/config.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/mudler/luet/pkg/helpers"
+	pkg "github.com/mudler/luet/pkg/package"
 	solver "github.com/mudler/luet/pkg/solver"
 
 	v "github.com/spf13/viper"
@@ -64,6 +65,7 @@ type LuetGeneralConfig struct {
 }
 
 type LuetSolverOptions struct {
+	solver.Options
 	Type           string            `mapstructure:"type"`
 	LearnRate      float32           `mapstructure:"rate"`
 	Discount       float32           `mapstructure:"discount"`
@@ -71,6 +73,15 @@ type LuetSolverOptions struct {
 	Implementation solver.SolverType `mapstructure:"implementation"`
 }
 
+func (opts LuetSolverOptions) ResolverIsSet() bool {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		return true
+	default:
+		return false
+	}
+}
+
 func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
 	switch opts.Type {
 	case solver.QLearningResolverType:
@@ -149,8 +160,9 @@ type LuetRepository struct {
 	Enable         bool              `json:"enable" yaml:"enable" mapstructure:"enable"`
 	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
 	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
-	TreePath       string            `json:"tree_path,omitempty" yaml:"tree_path,omitempty" mapstructure:"tree_path"`
-	MetaPath       string            `json:"meta_path,omitempty" yaml:"meta_path,omitempty" mapstructure:"meta_path"`
+	TreePath       string            `json:"treepath,omitempty" yaml:"treepath,omitempty" mapstructure:"treepath"`
+	MetaPath       string            `json:"metapath,omitempty" yaml:"metapath,omitempty" mapstructure:"metapath"`
+	Verify         bool              `json:"verify,omitempty" yaml:"verify,omitempty" mapstructure:"verify"`
 
 	// Serialized options not used in repository configuration
 
@@ -267,6 +279,16 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("solver.max_attempts", 9000)
 }
 
+func (c *LuetConfig) GetSystemDB() pkg.PackageDatabase {
+	switch LuetCfg.GetSystem().DatabaseEngine {
+	case "boltdb":
+		return pkg.NewBoltDatabase(
+			filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+	default:
+		return pkg.NewInMemoryDatabase(true)
+	}
+}
+
 func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
 	c.SystemRepositories = append(c.SystemRepositories, r)
 }

pkg/helpers/archive.go

@@ -90,14 +90,11 @@ func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modi
 	}
 
 	if sameOwner {
-		// PRE: i have root privileged.
-
+		// we do have root permissions, so we can extract keeping the same permissions.
 		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
 
 		opts := &archive.TarOptions{
-			// NOTE: NoLchown boolean is used for chmod of the symlink
-			// Probably it's needed set this always to true.
-			NoLchown:        true,
+			NoLchown:        false,
 			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
 			ContinueOnError: true,
 		}
@@ -201,12 +198,8 @@ func Untar(src, dest string, sameOwner bool) error {
 	defer in.Close()
 
 	if sameOwner {
-		// PRE: i have root privileged.
-
 		opts := &archive.TarOptions{
-			// NOTE: NoLchown boolean is used for chmod of the symlink
-			// Probably it's needed set this always to true.
-			NoLchown:        true,
+			NoLchown:        false,
 			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
 			ContinueOnError: true,
 		}

pkg/helpers/docker.go

@@ -0,0 +1,129 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"context"
+	"encoding/hex"
+	"os"
+	"strings"
+
+	"github.com/docker/cli/cli/trust"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/registry"
+	"github.com/mudler/luet/pkg/helpers/imgworker"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"github.com/theupdateframework/notary/tuf/data"
+)
+
+// See also https://github.com/docker/cli/blob/88c6089300a82d3373892adf6845a4fed1a4ba8d/cli/command/image/trust.go#L171
+
+func verifyImage(image string, authConfig *types.AuthConfig) (string, error) {
+	ref, err := reference.ParseAnyReference(image)
+	if err != nil {
+		return "", errors.Wrapf(err, "invalid reference %s", image)
+	}
+
+	// only check if image ref doesn't contain hashes
+	if _, ok := ref.(reference.Digested); !ok {
+		namedRef, ok := ref.(reference.Named)
+		if !ok {
+			return "", errors.New("failed to resolve image digest using content trust: reference is not named")
+		}
+		namedRef = reference.TagNameOnly(namedRef)
+		taggedRef, ok := namedRef.(reference.NamedTagged)
+		if !ok {
+			return "", errors.New("failed to resolve image digest using content trust: reference is not tagged")
+		}
+
+		resolvedImage, err := trustedResolveDigest(context.Background(), taggedRef, authConfig, "luet")
+		if err != nil {
+			return "", errors.Wrap(err, "failed to resolve image digest using content trust")
+		}
+		resolvedFamiliar := reference.FamiliarString(resolvedImage)
+		return resolvedFamiliar, nil
+	}
+
+	return "", nil
+}
+
+func trustedResolveDigest(ctx context.Context, ref reference.NamedTagged, authConfig *types.AuthConfig, useragent string) (reference.Canonical, error) {
+	repoInfo, err := registry.ParseRepositoryInfo(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	notaryRepo, err := trust.GetNotaryRepository(os.Stdin, os.Stdout, useragent, repoInfo, authConfig, "pull")
+	if err != nil {
+		return nil, errors.Wrap(err, "error establishing connection to trust repository")
+	}
+
+	t, err := notaryRepo.GetTargetByName(ref.Tag(), trust.ReleasesRole, data.CanonicalTargetsRole)
+	if err != nil {
+		return nil, trust.NotaryError(repoInfo.Name.Name(), err)
+	}
+	// Only get the tag if it's in the top level targets role or the releases delegation role
+	// ignore it if it's in any other delegation roles
+	if t.Role != trust.ReleasesRole && t.Role != data.CanonicalTargetsRole {
+		return nil, trust.NotaryError(repoInfo.Name.Name(), errors.Errorf("No trust data for %s", reference.FamiliarString(ref)))
+	}
+
+	h, ok := t.Hashes["sha256"]
+	if !ok {
+		return nil, errors.New("no valid hash, expecting sha256")
+	}
+
+	dgst := digest.NewDigestFromHex("sha256", hex.EncodeToString(h))
+
+	// Allow returning canonical reference with tag
+	return reference.WithDigest(ref, dgst)
+}
+
+// DownloadAndExtractDockerImage is a re-adaption
+// from genuinetools/img https://github.com/genuinetools/img/blob/54d0ca981c1260546d43961a538550eef55c87cf/pull.go
+func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) {
+
+	if verify {
+		img, err := verifyImage(image, auth)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed verifying image")
+		}
+		image = img
+	}
+
+	defer os.RemoveAll(temp)
+	c, err := imgworker.New(temp, auth)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed creating client")
+	}
+	defer c.Close()
+
+	listedImage, err := c.Pull(image)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed listing images")
+
+	}
+
+	os.RemoveAll(dest)
+	err = c.Unpack(image, dest)
+	return listedImage, err
+}
+
+func StripInvalidStringsFromImage(s string) string {
+	return strings.ReplaceAll(s, "+", "-")
+}

pkg/helpers/docker_test.go

@@ -0,0 +1,30 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("StripInvalidStringsFromImage", func() {
+	Context("Image names", func() {
+		It("strips invalid chars", func() {
+			Expect(StripInvalidStringsFromImage("foo+bar")).To(Equal("foo-bar"))
+		})
+	})
+})

pkg/helpers/file.go

@@ -16,14 +16,53 @@
 package helpers
 
 import (
+	"fmt"
+	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path/filepath"
+	"sort"
+	"strings"
+	"syscall"
 	"time"
 
+	"github.com/docker/docker/pkg/system"
+	"github.com/google/renameio"
 	copy "github.com/otiai10/copy"
+	"github.com/pkg/errors"
 )
 
+var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+func RandStringRunes(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letterRunes[rand.Intn(len(letterRunes))]
+	}
+	return string(b)
+}
+
+func Move(src, dst string) error {
+	f, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	t, err := renameio.TempFile("", dst)
+	if err != nil {
+		return err
+	}
+	defer t.Cleanup()
+
+	_, err = io.Copy(t, f)
+	if err != nil {
+		return err
+	}
+	return t.CloseAtomicallyReplace()
+}
+
 func OrderFiles(target string, files []string) ([]string, []string) {
 
 	var newFiles []string
@@ -41,6 +80,8 @@ func OrderFiles(target string, files []string) ([]string, []string) {
 		}
 	}
 
+	dirs := []string{}
+
 	for _, f := range files {
 		target := filepath.Join(target, f)
 		fi, err := os.Lstat(target)
@@ -48,11 +89,16 @@ func OrderFiles(target string, files []string) ([]string, []string) {
 			continue
 		}
 		if m := fi.Mode(); m.IsDir() {
-			newFiles = append(newFiles, f)
+			dirs = append(dirs, f)
 		}
 	}
 
-	return newFiles, notPresent
+	// Compare how many sub paths there are, and push at the end the ones that have less subpaths
+	sort.Slice(dirs, func(i, j int) bool {
+		return len(strings.Split(dirs[i], string(os.PathSeparator))) > len(strings.Split(dirs[j], string(os.PathSeparator)))
+	})
+
+	return append(newFiles, dirs...), notPresent
 }
 
 func ListDir(dir string) ([]string, error) {
@@ -71,6 +117,20 @@ func ListDir(dir string) ([]string, error) {
 	return content, err
 }
 
+// DirectoryIsEmpty Checks wether the directory is empty or not
+func DirectoryIsEmpty(dir string) (bool, error) {
+	f, err := os.Open(dir)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	if _, err = f.Readdirnames(1); err == io.EOF {
+		return true, nil
+	}
+	return false, nil
+}
+
 // Touch creates an empty file
 func Touch(f string) error {
 	_, err := os.Stat(f)
@@ -108,23 +168,105 @@ func Read(file string) (string, error) {
 	return string(dat), nil
 }
 
-func ensureDir(fileName string) {
-	dirName := filepath.Dir(fileName)
-	if _, serr := os.Stat(dirName); serr != nil {
-		merr := os.MkdirAll(dirName, os.ModePerm) // FIXME: It should preserve permissions from src to dst instead
-		if merr != nil {
-			panic(merr)
+func EnsureDirPerm(src, dst string) {
+	if info, err := os.Lstat(filepath.Dir(src)); err == nil {
+		if _, err := os.Lstat(filepath.Dir(dst)); os.IsNotExist(err) {
+			err := os.MkdirAll(filepath.Dir(dst), info.Mode().Perm())
+			if err != nil {
+				fmt.Println("warning: failed creating", filepath.Dir(dst), err.Error())
+			}
+			if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+				if err := os.Lchown(filepath.Dir(dst), int(stat.Uid), int(stat.Gid)); err != nil {
+					fmt.Println("warning: failed chowning", filepath.Dir(dst), err.Error())
+				}
+			}
 		}
+	} else {
+		EnsureDir(dst)
 	}
 }
 
-// CopyFile copies the contents of the file named src to the file named
+func EnsureDir(fileName string) error {
+	dirName := filepath.Dir(fileName)
+	if _, serr := os.Stat(dirName); os.IsNotExist(serr) {
+		merr := os.MkdirAll(dirName, os.ModePerm) // FIXME: It should preserve permissions from src to dst instead
+		if merr != nil {
+			return merr
+		}
+	}
+	return nil
+}
+
+func CopyFile(src, dst string) (err error) {
+	return copy.Copy(src, dst, copy.Options{
+		Sync:      true,
+		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
+}
+
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
+// DeepCopyFile copies the contents of the file named src to the file named
 // by dst. The file will be created if it does not already exist. If the
 // destination file exists, all it's contents will be replaced by the contents
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
-func CopyFile(src, dst string) (err error) {
-	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
+func DeepCopyFile(src, dst string) (err error) {
+	// Workaround for https://github.com/otiai10/copy/issues/47
+	fi, err := os.Lstat(src)
+	if err != nil {
+		return errors.Wrap(err, "error reading file info")
+	}
+
+	fm := fi.Mode()
+	switch {
+	case fm&os.ModeNamedPipe != 0:
+		EnsureDirPerm(src, dst)
+		if err := syscall.Mkfifo(dst, uint32(fi.Mode())); err != nil {
+			return errors.Wrap(err, "failed creating pipe")
+		}
+		if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
+			if err := os.Chown(dst, int(stat.Uid), int(stat.Gid)); err != nil {
+				return errors.Wrap(err, "failed chowning file")
+			}
+		}
+		return nil
+	}
+
+	//filepath.Dir(src)
+	EnsureDirPerm(src, dst)
+
+	err = copy.Copy(src, dst, copy.Options{
+		Sync:      true,
+		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
+	if err != nil {
+		return err
+	}
+	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
+		if err := os.Lchown(dst, int(stat.Uid), int(stat.Gid)); err != nil {
+			fmt.Println("warning: failed chowning", dst, err.Error())
+		}
+	}
+
+	return doCopyXattrs(src, dst)
 }
 
 func IsDirectory(path string) (bool, error) {
@@ -141,5 +283,7 @@ func IsDirectory(path string) (bool, error) {
 func CopyDir(src string, dst string) (err error) {
 	src = filepath.Clean(src)
 	dst = filepath.Clean(dst)
-	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
+	return copy.Copy(src, dst, copy.Options{
+		Sync:      true,
+		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }

pkg/helpers/file_test.go

@@ -33,6 +33,23 @@ var _ = Describe("Helpers", func() {
 		})
 	})
 
+	Context("DirectoryIsEmpty", func() {
+		It("Detects empty directory", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+			Expect(DirectoryIsEmpty(testDir)).To(BeTrue())
+		})
+		It("Detects directory with files", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+			err = Touch(filepath.Join(testDir, "foo"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(DirectoryIsEmpty(testDir)).To(BeFalse())
+		})
+	})
+
 	Context("Orders dir and files correctly", func() {
 		It("puts files first and folders at end", func() {
 			testDir, err := ioutil.TempDir(os.TempDir(), "test")
@@ -60,5 +77,27 @@ var _ = Describe("Helpers", func() {
 			Expect(ordered).To(Equal([]string{"baz", "bar/foo", "foo", "baz2/foo", "bar", "baz2"}))
 			Expect(notExisting).To(Equal([]string{"notexisting"}))
 		})
+
+		It("orders correctly when there are folders with folders", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			err = os.MkdirAll(filepath.Join(testDir, "bar"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+			err = os.MkdirAll(filepath.Join(testDir, "foo"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "bar"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz", "fa"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			ordered, _ := OrderFiles(testDir, []string{"foo", "foo/bar", "bar", "foo/baz/fa", "foo/baz"})
+			Expect(ordered).To(Equal([]string{"foo/baz/fa", "foo/bar", "foo/baz", "foo", "bar"}))
+		})
 	})
 })

pkg/helpers/helm.go

@@ -3,6 +3,7 @@ package helpers
 import (
 	"io/ioutil"
 
+	"github.com/imdario/mergo"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 	"helm.sh/helm/v3/pkg/chart"
@@ -11,7 +12,7 @@ import (
 )
 
 // RenderHelm renders the template string with helm
-func RenderHelm(template string, values map[string]interface{}) (string, error) {
+func RenderHelm(template string, values, d map[string]interface{}) (string, error) {
 	c := &chart.Chart{
 		Metadata: &chart.Metadata{
 			Name:    "",
@@ -23,7 +24,7 @@ func RenderHelm(template string, values map[string]interface{}) (string, error)
 		Values: map[string]interface{}{"Values": values},
 	}
 
-	v, err := chartutil.CoalesceValues(c, map[string]interface{}{})
+	v, err := chartutil.CoalesceValues(c, map[string]interface{}{"Values": d})
 	if err != nil {
 		return "", errors.Wrap(err, "while rendering template")
 	}
@@ -37,7 +38,38 @@ func RenderHelm(template string, values map[string]interface{}) (string, error)
 
 type templatedata map[string]interface{}
 
-func RenderFiles(toTemplate, valuesFile string) (string, error) {
+// UnMarshalValues unmarshal values files and joins them into a unique templatedata
+// the join happens from right to left, so any rightmost value file overwrites the content of the ones before it.
+func UnMarshalValues(values []string) (templatedata, error) {
+	dst := templatedata{}
+	if len(values) > 0 {
+		for _, bv := range reverse(values) {
+			current := templatedata{}
+
+			defBuild, err := ioutil.ReadFile(bv)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+bv)
+			}
+			err = yaml.Unmarshal(defBuild, &current)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+bv)
+			}
+			if err := mergo.Merge(&dst, current); err != nil {
+				return nil, errors.Wrap(err, "merging values file "+bv)
+			}
+		}
+	}
+	return dst, nil
+}
+
+func reverse(s []string) []string {
+	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+		s[i], s[j] = s[j], s[i]
+	}
+	return s
+}
+
+func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string, error) {
 	raw, err := ioutil.ReadFile(toTemplate)
 	if err != nil {
 		return "", errors.Wrap(err, "reading file "+toTemplate)
@@ -46,14 +78,20 @@ func RenderFiles(toTemplate, valuesFile string) (string, error) {
 	if !Exists(valuesFile) {
 		return "", errors.Wrap(err, "file not existing "+valuesFile)
 	}
-	def, err := ioutil.ReadFile(valuesFile)
+	val, err := ioutil.ReadFile(valuesFile)
 	if err != nil {
 		return "", errors.Wrap(err, "reading file "+valuesFile)
 	}
 
 	var values templatedata
-	if err = yaml.Unmarshal(def, &values); err != nil {
+	if err = yaml.Unmarshal(val, &values); err != nil {
 		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
 	}
-	return RenderHelm(string(raw), values)
+
+	dst, err := UnMarshalValues(defaultFile)
+	if err != nil {
+		return "", errors.Wrap(err, "unmarshalling values")
+	}
+
+	return RenderHelm(string(raw), values, dst)
 }

pkg/helpers/helm_test.go

@@ -16,17 +16,167 @@
 package helpers_test
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
+func writeFile(path string, content string) {
+	err := ioutil.WriteFile(path, []byte(content), 0644)
+	Expect(err).ToNot(HaveOccurred())
+}
+
 var _ = Describe("Helpers", func() {
 	Context("RenderHelm", func() {
 		It("Renders templates", func() {
-			out, err := RenderHelm("{{.Values.Test}}", map[string]interface{}{"Test": "foo"})
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(out).To(Equal("foo"))
+			Expect(out).To(Equal("foobar"))
+		})
+		It("Renders templates with overrides", func() {
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foobar"))
+		})
+
+		It("Renders templates", func() {
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foobar"))
+		})
+
+		It("Render files default overrides", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+			writeFile(d, `
+foo: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("baz"))
+
+		})
+
+		It("Render files from values", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+			writeFile(d, `
+faa: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bar"))
+
+		})
+
+		It("Render files from values if no default", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bar"))
+		})
+
+		It("Render files merging defaults", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+			d2 := filepath.Join(testDir, "default2.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}{{.Values.bar}}{{.Values.b}}`)
+			writeFile(values, `
+foo: "bar"
+b: "f"
+`)
+			writeFile(d, `
+foo: "baz"
+`)
+
+			writeFile(d2, `
+foo: "do"
+bar: "nei"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d2, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bazneif"))
+
+			res, err = RenderFiles(toTemplate, values, d, d2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("doneif"))
+		})
+
+		It("doesn't interpolate if no one provides the values", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foao: "bar"
+`)
+			writeFile(d, `
+faa: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal(""))
+
 		})
 	})
 })

pkg/helpers/imgworker/auth.go

@@ -0,0 +1,36 @@
+package imgworker
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types"
+
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/session/auth"
+	"google.golang.org/grpc"
+)
+
+func NewDockerAuthProvider(auth *types.AuthConfig) session.Attachable {
+	return &authProvider{
+		config: auth,
+	}
+}
+
+type authProvider struct {
+	config *types.AuthConfig
+}
+
+func (ap *authProvider) Register(server *grpc.Server) {
+	// no-op
+}
+
+func (ap *authProvider) Credentials(ctx context.Context, req *auth.CredentialsRequest) (*auth.CredentialsResponse, error) {
+	res := &auth.CredentialsResponse{}
+	if ap.config.IdentityToken != "" {
+		res.Secret = ap.config.IdentityToken
+	} else {
+		res.Username = ap.config.Username
+		res.Secret = ap.config.Password
+	}
+	return res, nil
+}

pkg/helpers/imgworker/client.go

@@ -0,0 +1,81 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/containerd/namespaces"
+	dockertypes "github.com/docker/docker/api/types"
+	"github.com/genuinetools/img/types"
+	"github.com/moby/buildkit/control"
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/util/appcontext"
+	"github.com/moby/buildkit/worker/base"
+	"github.com/pkg/errors"
+)
+
+// Client holds the information for the client we will use for communicating
+// with the buildkit controller.
+type Client struct {
+	backend   string
+	localDirs map[string]string
+	root      string
+
+	sessionManager *session.Manager
+	controller     *control.Controller
+	opts           *base.WorkerOpt
+
+	sess *session.Session
+	ctx  context.Context
+	auth *dockertypes.AuthConfig
+}
+
+// New returns a new client for communicating with the buildkit controller.
+func New(root string, auth *dockertypes.AuthConfig) (*Client, error) {
+	// Native backend is fine, our images have just one layer. No need to depend on anything
+	backend := types.NativeBackend
+
+	// Create the root/
+	root = filepath.Join(root, "runc", backend)
+	if err := os.MkdirAll(root, 0700); err != nil {
+		return nil, err
+	}
+	c := &Client{
+		backend:   types.NativeBackend,
+		root:      root,
+		localDirs: nil,
+		auth:      auth,
+	}
+
+	if err := c.prepare(); err != nil {
+		return nil, errors.Wrapf(err, "failed preparing client")
+	}
+
+	// Create the start of the client.
+	return c, nil
+}
+
+func (c *Client) Close() {
+	c.sess.Close()
+}
+
+func (c *Client) prepare() error {
+	ctx := appcontext.Context()
+	sess, sessDialer, err := c.Session(ctx)
+	if err != nil {
+		return errors.Wrapf(err, "failed creating Session")
+	}
+	ctx = session.NewContext(ctx, sess.ID())
+	ctx = namespaces.WithNamespace(ctx, "buildkit")
+
+	c.ctx = ctx
+	c.sess = sess
+
+	go func() {
+		sess.Run(ctx, sessDialer)
+	}()
+	return nil
+}

pkg/helpers/imgworker/pull.go

@@ -0,0 +1,129 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"fmt"
+
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
+	"github.com/docker/distribution/reference"
+	"github.com/moby/buildkit/cache"
+	"github.com/moby/buildkit/exporter"
+	imageexporter "github.com/moby/buildkit/exporter/containerimage"
+	"github.com/moby/buildkit/source"
+	"github.com/moby/buildkit/source/containerimage"
+)
+
+// ListedImage represents an image structure returuned from ListImages.
+// It extends containerd/images.Image with extra fields.
+type ListedImage struct {
+	images.Image
+	ContentSize int64
+}
+
+// Pull retrieves an image from a remote registry.
+func (c *Client) Pull(image string) (*ListedImage, error) {
+
+	ctx := c.ctx
+
+	sm, err := c.getSessionManager()
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse the image name and tag.
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return nil, fmt.Errorf("parsing image name %q failed: %v", image, err)
+	}
+	// Add the latest lag if they did not provide one.
+	named = reference.TagNameOnly(named)
+	image = named.String()
+
+	// Get the identifier for the image.
+	identifier, err := source.NewImageIdentifier(image)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the worker opts.
+	opt, err := c.createWorkerOpt()
+	if err != nil {
+		return nil, fmt.Errorf("creating worker opt failed: %v", err)
+	}
+
+	cm, err := cache.NewManager(cache.ManagerOpt{
+		Snapshotter:    opt.Snapshotter,
+		MetadataStore:  opt.MetadataStore,
+		ContentStore:   opt.ContentStore,
+		LeaseManager:   opt.LeaseManager,
+		GarbageCollect: opt.GarbageCollect,
+		Applier:        opt.Applier,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the source for the pull.
+	srcOpt := containerimage.SourceOpt{
+		Snapshotter:   opt.Snapshotter,
+		ContentStore:  opt.ContentStore,
+		Applier:       opt.Applier,
+		CacheAccessor: cm,
+		ImageStore:    opt.ImageStore,
+		RegistryHosts: opt.RegistryHosts,
+		LeaseManager:  opt.LeaseManager,
+	}
+	src, err := containerimage.NewSource(srcOpt)
+	if err != nil {
+		return nil, err
+	}
+	s, err := src.Resolve(ctx, identifier, sm)
+	if err != nil {
+		return nil, err
+	}
+	ref, err := s.Snapshot(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the exporter for the pull.
+	iw, err := imageexporter.NewImageWriter(imageexporter.WriterOpt{
+		Snapshotter:  opt.Snapshotter,
+		ContentStore: opt.ContentStore,
+		Differ:       opt.Differ,
+	})
+	if err != nil {
+		return nil, err
+	}
+	expOpt := imageexporter.Opt{
+		SessionManager: sm,
+		ImageWriter:    iw,
+		Images:         opt.ImageStore,
+		RegistryHosts:  opt.RegistryHosts,
+		LeaseManager:   opt.LeaseManager,
+	}
+	exp, err := imageexporter.New(expOpt)
+	if err != nil {
+		return nil, err
+	}
+	e, err := exp.Resolve(ctx, map[string]string{"name": image})
+	if err != nil {
+		return nil, err
+	}
+	if _, err := e.Export(ctx, exporter.Source{Ref: ref}); err != nil {
+		return nil, err
+	}
+	// Get the image.
+	img, err := opt.ImageStore.Get(ctx, image)
+	if err != nil {
+		return nil, fmt.Errorf("getting image %s from image store failed: %v", image, err)
+	}
+	size, err := img.Size(ctx, opt.ContentStore, platforms.Default())
+	if err != nil {
+		return nil, fmt.Errorf("calculating size of image %s failed: %v", img.Name, err)
+	}
+
+	return &ListedImage{Image: img, ContentSize: size}, nil
+}

pkg/helpers/imgworker/session.go

@@ -0,0 +1,49 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"context"
+
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/session/filesync"
+	"github.com/moby/buildkit/session/testutil"
+	"github.com/pkg/errors"
+)
+
+func (c *Client) getSessionManager() (*session.Manager, error) {
+	if c.sessionManager == nil {
+		var err error
+		c.sessionManager, err = session.NewManager()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return c.sessionManager, nil
+}
+
+// Session creates the session manager and returns the session and it's
+// dialer.
+func (c *Client) Session(ctx context.Context) (*session.Session, session.Dialer, error) {
+	m, err := c.getSessionManager()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to create session manager")
+	}
+	sessionName := "luet"
+	s, err := session.NewSession(ctx, sessionName, "")
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to create session")
+	}
+	syncedDirs := make([]filesync.SyncedDir, 0, len(c.localDirs))
+	for name, d := range c.localDirs {
+		syncedDirs = append(syncedDirs, filesync.SyncedDir{Name: name, Dir: d})
+	}
+	s.Allow(filesync.NewFSSyncProvider(syncedDirs))
+	s.Allow(NewDockerAuthProvider(c.auth))
+	return s, sessionDialer(s, m), err
+}
+
+func sessionDialer(s *session.Session, m *session.Manager) session.Dialer {
+	// FIXME: rename testutil
+	return session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn)))
+}

pkg/helpers/imgworker/unpack.go

@@ -0,0 +1,82 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+)
+
+// TODO: this requires root permissions to mount/unmount layers, althrought it shouldn't be required.
+// See how backends are unpacking images without asking for root permissions.
+
+// Unpack exports an image to a rootfs destination directory.
+func (c *Client) Unpack(image, dest string) error {
+
+	ctx := c.ctx
+	if len(dest) < 1 {
+		return errors.New("destination directory for rootfs cannot be empty")
+	}
+
+	if _, err := os.Stat(dest); err == nil {
+		return fmt.Errorf("destination directory already exists: %s", dest)
+	}
+
+	// Parse the image name and tag.
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return fmt.Errorf("parsing image name %q failed: %v", image, err)
+	}
+	// Add the latest lag if they did not provide one.
+	named = reference.TagNameOnly(named)
+	image = named.String()
+
+	// Create the worker opts.
+	opt, err := c.createWorkerOpt()
+	if err != nil {
+		return fmt.Errorf("creating worker opt failed: %v", err)
+	}
+
+	if opt.ImageStore == nil {
+		return errors.New("image store is nil")
+	}
+
+	img, err := opt.ImageStore.Get(ctx, image)
+	if err != nil {
+		return fmt.Errorf("getting image %s from image store failed: %v", image, err)
+	}
+
+	manifest, err := images.Manifest(ctx, opt.ContentStore, img.Target, platforms.Default())
+	if err != nil {
+		return fmt.Errorf("getting image manifest failed: %v", err)
+	}
+
+	for _, desc := range manifest.Layers {
+		logrus.Debugf("Unpacking layer %s", desc.Digest.String())
+
+		// Read the blob from the content store.
+		layer, err := opt.ContentStore.ReaderAt(ctx, desc)
+		if err != nil {
+			return fmt.Errorf("getting reader for digest %s failed: %v", desc.Digest.String(), err)
+		}
+
+		// Unpack the tarfile to the rootfs path.
+		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
+		if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{
+			NoLchown:        false,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+		}); err != nil {
+			return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err)
+		}
+	}
+
+	return nil
+}

pkg/helpers/imgworker/workeropts.go

@@ -0,0 +1,106 @@
+package imgworker
+
+// FROM Slightly adapted from genuinetools/img worker
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/containerd/containerd/content/local"
+	"github.com/containerd/containerd/diff/apply"
+	"github.com/containerd/containerd/diff/walking"
+	ctdmetadata "github.com/containerd/containerd/metadata"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/remotes/docker"
+	ctdsnapshot "github.com/containerd/containerd/snapshots"
+	"github.com/containerd/containerd/snapshots/native"
+	"github.com/moby/buildkit/cache/metadata"
+	containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
+	"github.com/moby/buildkit/util/binfmt_misc"
+	"github.com/moby/buildkit/util/leaseutil"
+	"github.com/moby/buildkit/worker/base"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	bolt "go.etcd.io/bbolt"
+)
+
+// createWorkerOpt creates a base.WorkerOpt to be used for a new worker.
+func (c *Client) createWorkerOpt() (opt base.WorkerOpt, err error) {
+
+	if c.opts != nil {
+		return *c.opts, nil
+	}
+
+	// Create the metadata store.
+	md, err := metadata.NewStore(filepath.Join(c.root, "metadata.db"))
+	if err != nil {
+		return opt, err
+	}
+
+	snapshotRoot := filepath.Join(c.root, "snapshots")
+
+	s, err := native.NewSnapshotter(snapshotRoot)
+	if err != nil {
+		return opt, fmt.Errorf("creating %s snapshotter failed: %v", c.backend, err)
+	}
+
+	// Create the content store locally.
+	contentStore, err := local.NewStore(filepath.Join(c.root, "content"))
+	if err != nil {
+		return opt, err
+	}
+
+	// Open the bolt database for metadata.
+	db, err := bolt.Open(filepath.Join(c.root, "containerdmeta.db"), 0644, nil)
+	if err != nil {
+		return opt, err
+	}
+
+	// Create the new database for metadata.
+	mdb := ctdmetadata.NewDB(db, contentStore, map[string]ctdsnapshot.Snapshotter{
+		c.backend: s,
+	})
+	if err := mdb.Init(context.TODO()); err != nil {
+		return opt, err
+	}
+
+	// Create the image store.
+	imageStore := ctdmetadata.NewImageStore(mdb)
+
+	contentStore = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit")
+
+	id, err := base.ID(c.root)
+	if err != nil {
+		return opt, err
+	}
+
+	xlabels := base.Labels("oci", c.backend)
+
+	var supportedPlatforms []specs.Platform
+	for _, p := range binfmt_misc.SupportedPlatforms(false) {
+		parsed, err := platforms.Parse(p)
+		if err != nil {
+			return opt, err
+		}
+		supportedPlatforms = append(supportedPlatforms, platforms.Normalize(parsed))
+	}
+
+	opt = base.WorkerOpt{
+		ID:             id,
+		Labels:         xlabels,
+		MetadataStore:  md,
+		Snapshotter:    containerdsnapshot.NewSnapshotter(c.backend, mdb.Snapshotter(c.backend), "buildkit", nil),
+		ContentStore:   contentStore,
+		Applier:        apply.NewFileSystemApplier(contentStore),
+		Differ:         walking.NewWalkingDiff(contentStore),
+		ImageStore:     imageStore,
+		Platforms:      supportedPlatforms,
+		RegistryHosts:  docker.ConfigureDefaultRegistries(),
+		LeaseManager:   leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"),
+		GarbageCollect: mdb.GarbageCollect,
+	}
+
+	c.opts = &opt
+
+	return opt, err
+}

pkg/helpers/match.go

@@ -17,9 +17,18 @@
 package helpers
 
 import (
+	"reflect"
 	"regexp"
 )
 
+func ReverseAny(s interface{}) {
+	n := reflect.ValueOf(s).Len()
+	swap := reflect.Swapper(s)
+	for i, j := 0, n-1; i < j; i, j = i+1, j-1 {
+		swap(i, j)
+	}
+}
+
 func MapMatchRegex(m *map[string]string, r *regexp.Regexp) bool {
 	ans := false
 

pkg/helpers/references.go

@@ -0,0 +1,30 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       David Cassany <dcassany@suse.com>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"github.com/asaskevich/govalidator"
+	"strings"
+)
+
+func StripRegistryFromImage(image string) string {
+	img := strings.SplitN(image, "/", 2)
+	if len(img) == 2 && govalidator.IsURL(img[0]) {
+		return img[1]
+	}
+	return image
+}

pkg/helpers/references_test.go

@@ -0,0 +1,44 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       David Cassany <dcassany@suse.com>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Helpers", func() {
+	Context("StripRegistryFromImage", func() {
+		It("Strips the domain name", func() {
+			out := StripRegistryFromImage("valid.domain.org/base/image:tag")
+			Expect(out).To(Equal("base/image:tag"))
+		})
+		It("Strips the domain name when port is included", func() {
+			out := StripRegistryFromImage("valid.domain.org:5000/base/image:tag")
+			Expect(out).To(Equal("base/image:tag"))
+		})
+		It("Does not strip the domain name", func() {
+			out := StripRegistryFromImage("not-a-domain/base/image:tag")
+			Expect(out).To(Equal("not-a-domain/base/image:tag"))
+		})
+		It("Does not strip the domain name on invalid domains", func() {
+			out := StripRegistryFromImage("-invaliddomain.org/base/image:tag")
+			Expect(out).To(Equal("-invaliddomain.org/base/image:tag"))
+		})
+	})
+})

pkg/installer/client/docker.go

@@ -0,0 +1,190 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/go-units"
+	"github.com/pkg/errors"
+
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/helpers/imgworker"
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+const (
+	errImageDownloadMsg = "failed downloading image %s: %s"
+)
+
+type DockerClient struct {
+	RepoData RepoData
+	auth     *types.AuthConfig
+	verify   bool
+}
+
+func NewDockerClient(r RepoData) *DockerClient {
+	auth := &types.AuthConfig{}
+
+	dat, _ := json.Marshal(r.Authentication)
+	json.Unmarshal(dat, auth)
+
+	return &DockerClient{RepoData: r, auth: auth}
+}
+
+func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
+	//var u *url.URL = nil
+	var err error
+	var temp string
+
+	Spinner(22)
+	defer SpinnerStop()
+
+	var resultingArtifact *artifact.PackageArtifact
+	artifactName := path.Base(a.Path)
+	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
+	Debug("Cache file", cacheFile)
+	if err := helpers.EnsureDir(cacheFile); err != nil {
+		return nil, errors.Wrapf(err, "could not create cache folder %s for %s", config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), cacheFile)
+	}
+	ok := false
+
+	// TODO:
+	// Files are in URI/packagename:version (GetPackageImageName() method)
+	// use downloadAndExtract .. and egenrate an archive to consume. Checksum should be already checked while downloading the image
+	// with the above functions, because Docker images already contain such metadata
+	// - Check how verification is done when calling DownloadArtifact outside, similarly we need to check DownloadFile, and how verification
+	// is done in such cases (see repository.go)
+
+	// Check if file is already in cache
+	if helpers.Exists(cacheFile) {
+		Debug("Cache hit for artifact", artifactName)
+		resultingArtifact = a
+		resultingArtifact.Path = cacheFile
+		resultingArtifact.Checksums = artifact.Checksums{}
+	} else {
+
+		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
+		if err != nil {
+			return nil, err
+		}
+		defer os.RemoveAll(temp)
+
+		for _, uri := range c.RepoData.Urls {
+
+			imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID())
+			Info("Downloading image", imageName)
+
+			contentstore, err := config.LuetCfg.GetSystem().TempDir("contentstore")
+			if err != nil {
+				Warning("Cannot create contentstore", err.Error())
+				continue
+			}
+
+			// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
+			info, err := helpers.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
+			if err != nil {
+				Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
+				continue
+			}
+
+			Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
+			Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
+			Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile)
+
+			newart := a
+			// We discard checksum, that are checked while during pull and unpack
+			newart.Checksums = artifact.Checksums{}
+			newart.Path = cacheFile                    // First set to cache file
+			newart.Path = newart.GetUncompressedName() // Calculate the real path from cacheFile
+			err = newart.Compress(temp, 1)
+			if err != nil {
+				Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))
+				continue
+			}
+			resultingArtifact = newart
+
+			ok = true
+			break
+		}
+
+		if !ok {
+			return nil, err
+		}
+	}
+
+	return resultingArtifact, nil
+}
+
+func (c *DockerClient) DownloadFile(name string) (string, error) {
+	var file *os.File = nil
+	var err error
+	var temp, contentstore string
+	var info *imgworker.ListedImage
+	// Files should be in URI/repository:<file>
+	ok := false
+
+	temp, err = config.LuetCfg.GetSystem().TempDir("tree")
+	if err != nil {
+		return "", err
+	}
+
+	for _, uri := range c.RepoData.Urls {
+		file, err = config.LuetCfg.GetSystem().TempFile("DockerClient")
+		if err != nil {
+			continue
+		}
+
+		contentstore, err = config.LuetCfg.GetSystem().TempDir("contentstore")
+		if err != nil {
+			Warning("Cannot create contentstore", err.Error())
+			continue
+		}
+
+		imageName := fmt.Sprintf("%s:%s", uri, helpers.StripInvalidStringsFromImage(name))
+		Info("Downloading", imageName)
+
+		info, err = helpers.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
+		if err != nil {
+			Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
+			continue
+		}
+
+		Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
+		Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
+
+		Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name())
+		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+		if err != nil {
+			continue
+		}
+		ok = true
+		break
+	}
+
+	if !ok {
+		return "", err
+	}
+
+	return file.Name(), err
+}

pkg/installer/client/docker_test.go

@@ -0,0 +1,79 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+
+	helpers "github.com/mudler/luet/pkg/helpers"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/mudler/luet/pkg/installer/client"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// This test expect that the repository defined in UNIT_TEST_DOCKER_IMAGE is in zstd format.
+// the repository is built by the 01_simple_docker.sh integration test file.
+// This test also require root. At the moment, unpacking docker images with 'img' requires root permission to
+// mount/unmount layers.
+var _ = Describe("Docker client", func() {
+	Context("With repository", func() {
+		repoImage := os.Getenv("UNIT_TEST_DOCKER_IMAGE")
+		var repoURL []string
+		var c *DockerClient
+		BeforeEach(func() {
+			if repoImage == "" {
+				Skip("UNIT_TEST_DOCKER_IMAGE not specified")
+			}
+			repoURL = []string{repoImage}
+			c = NewDockerClient(RepoData{Urls: repoURL})
+		})
+
+		It("Downloads single files", func() {
+			f, err := c.DownloadFile("repository.yaml")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(f)).To(ContainSubstring("Test Repo"))
+			os.RemoveAll(f)
+		})
+
+		It("Downloads artifacts", func() {
+			f, err := c.DownloadArtifact(&artifact.PackageArtifact{
+				Path: "test.tar",
+				CompileSpec: &compilerspec.LuetCompilationSpec{
+					Package: &pkg.DefaultPackage{
+						Name:     "c",
+						Category: "test",
+						Version:  "1.0",
+					},
+				},
+			})
+			Expect(err).ToNot(HaveOccurred())
+			tmpdir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			Expect(f.Unpack(tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Read(filepath.Join(tmpdir, "c"))).To(Equal("c\n"))
+			Expect(helpers.Read(filepath.Join(tmpdir, "cd"))).To(Equal("c\n"))
+			os.RemoveAll(f.Path)
+		})
+	})
+})

pkg/installer/client/http.go

@@ -24,9 +24,9 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	. "github.com/mudler/luet/pkg/logger"
 
-	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 
@@ -66,19 +66,19 @@ func Round(input float64) float64 {
 	return math.Floor(input + 0.5)
 }
 
-func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	var u *url.URL = nil
 	var err error
 	var req *grab.Request
 	var temp string
 
-	artifactName := path.Base(artifact.GetPath())
+	artifactName := path.Base(a.Path)
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 	ok := false
 
 	// Check if file is already in cache
 	if helpers.Exists(cacheFile) {
-		Info("Use artifact", artifactName, "from cache.")
+		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 
 		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
@@ -90,7 +90,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 		client := grab.NewClient()
 
 		for _, uri := range c.RepoData.Urls {
-			Info("Downloading artifact", artifactName, "from", uri)
+			Debug("Downloading artifact", artifactName, "from", uri)
 
 			u, err = url.Parse(uri)
 			if err != nil {
@@ -168,8 +168,8 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 		}
 	}
 
-	newart := artifact
-	newart.SetPath(cacheFile)
+	newart := a
+	newart.Path = cacheFile
 	return newart, nil
 }
 
@@ -202,7 +202,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 		}
 		u.Path = path.Join(u.Path, name)
 
-		Info("Downloading", u.String())
+		Debug("Downloading", u.String())
 
 		req, err = c.PrepareReq(temp, u.String())
 		if err != nil {

pkg/installer/client/http_test.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"path/filepath"
 
-	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	helpers "github.com/mudler/luet/pkg/helpers"
 
 	. "github.com/mudler/luet/pkg/installer/client"
@@ -63,10 +63,10 @@ var _ = Describe("Http client", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
-			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
+			path, err := c.DownloadArtifact(&artifact.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path.GetPath())).To(Equal("test"))
-			os.RemoveAll(path.GetPath())
+			Expect(helpers.Read(path.Path)).To(Equal("test"))
+			os.RemoveAll(path.Path)
 		})
 
 	})

pkg/installer/client/interface.go

@@ -18,4 +18,5 @@ package client
 type RepoData struct {
 	Urls           []string
 	Authentication map[string]string
+	Verify         bool
 }

pkg/installer/client/local.go

@@ -20,10 +20,10 @@ import (
 	"path"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 
-	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/helpers"
 )
 
@@ -35,11 +35,11 @@ func NewLocalClient(r RepoData) *LocalClient {
 	return &LocalClient{RepoData: r}
 }
 
-func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+func (c *LocalClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	var err error
 
 	rootfs := ""
-	artifactName := path.Base(artifact.GetPath())
+	artifactName := path.Base(a.Path)
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 
 	if !config.LuetCfg.ConfigFromHost {
@@ -51,7 +51,7 @@ func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Art
 
 	// Check if file is already in cache
 	if helpers.Exists(cacheFile) {
-		Info("Use artifact", artifactName, "from cache.")
+		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 		ok := false
 		for _, uri := range c.RepoData.Urls {
@@ -74,8 +74,8 @@ func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Art
 		}
 	}
 
-	newart := artifact
-	newart.SetPath(cacheFile)
+	newart := a
+	newart.Path = cacheFile
 	return newart, nil
 }
 

pkg/installer/client/local_test.go

@@ -20,7 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
-	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	helpers "github.com/mudler/luet/pkg/helpers"
 
 	. "github.com/mudler/luet/pkg/installer/client"
@@ -56,10 +56,10 @@ var _ = Describe("Local client", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
-			path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"})
+			path, err := c.DownloadArtifact(&artifact.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path.GetPath())).To(Equal("test"))
-			os.RemoveAll(path.GetPath())
+			Expect(helpers.Read(path.Path)).To(Equal("test"))
+			os.RemoveAll(path.Path)
 		})
 
 	})

pkg/installer/finalizer.go

@@ -16,6 +16,7 @@
 package installer
 
 import (
+	"os"
 	"os/exec"
 
 	"github.com/ghodss/yaml"
@@ -48,7 +49,7 @@ func (f *LuetFinalizer) RunInstall(s *System) error {
 	for _, c := range f.Install {
 		toRun := append(args, c)
 		Info(":shell: Executing finalizer on ", s.Target, cmd, toRun)
-		if s.Target == "/" {
+		if s.Target == string(os.PathSeparator) {
 			cmd := exec.Command(cmd, toRun...)
 			stdoutStderr, err := cmd.CombinedOutput()
 			if err != nil {

pkg/installer/installer_test.go

@@ -23,8 +23,10 @@ import (
 	//	. "github.com/mudler/luet/pkg/installer"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/helpers"
-	solver "github.com/mudler/luet/pkg/solver"
 
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -33,7 +35,20 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+func stubRepo(tmpdir, tree string) (*LuetSystemRepository, error) {
+	return GenerateRepository(
+		"test",
+		"description",
+		"disk",
+		[]string{tmpdir},
+		1,
+		tmpdir,
+		[]string{tree},
+		pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
+}
+
 var _ = Describe("Installer", func() {
+
 	Context("Writes a repository definition", func() {
 		It("Writes a repo and can install packages from it", func() {
 			//repo:=NewLuetSystemRepository()
@@ -49,7 +64,9 @@ var _ = Describe("Installer", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
+				generalRecipe.GetDatabase(),
+				options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -60,16 +77,15 @@ var _ = Describe("Installer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
-			artifact, err := c.Compile(false, spec)
+			a, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(a.Path)).To(BeTrue())
+			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -84,13 +100,13 @@ var _ = Describe("Installer", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -134,7 +150,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p.GetName()).To(Equal("b"))
 
-			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			err = inst.Uninstall(system, &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
@@ -165,7 +181,7 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
 			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
-				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+				generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -176,16 +192,15 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -200,16 +215,18 @@ urls:
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
 			treeFile := NewDefaultTreeRepositoryFile()
-			treeFile.SetCompressionType(compiler.None)
+			treeFile.SetCompressionType(compression.None)
 			repo.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -253,7 +270,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p.GetName()).To(Equal("b"))
 
-			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			err = inst.Uninstall(system, &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
@@ -283,7 +300,8 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -294,16 +312,15 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -318,13 +335,20 @@ urls:
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository(
+				"test",
+				"description",
+				"disk",
+				[]string{tmpdir}, 1,
+				tmpdir,
+				[]string{"../../tests/fixtures/buildable"},
+				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -373,7 +397,7 @@ urls:
 			Expect(files).To(Equal([]string{"artifact42", "test5", "test6"}))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			err = inst.Uninstall(system, &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
@@ -401,7 +425,8 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -412,16 +437,15 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -436,13 +460,21 @@ urls:
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := GenerateRepository(
+				"test",
+				"description",
+				"disk",
+				[]string{tmpdir},
+				1,
+				tmpdir,
+				[]string{"../../tests/fixtures/buildable"},
+				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -488,7 +520,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), options.Concurrency(2))
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -496,15 +528,14 @@ urls:
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
 
 			spec.SetOutputPath(tmpdir2)
-			c.SetConcurrency(2)
 
 			artifact, err = c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
 
-			repo, err = GenerateRepository("test", "description", "disk", []string{tmpdir2}, 1, tmpdir2, []string{"../../tests/fixtures/alpine"}, pkg.NewInMemoryDatabase(false))
+			repo, err = stubRepo(tmpdir2, "../../tests/fixtures/alpine")
 			Expect(err).ToNot(HaveOccurred())
-			err = repo.Write(tmpdir2, false)
+			err = repo.Write(tmpdir2, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			fakeroot, err = ioutil.TempDir("", "fakeroot")
@@ -547,7 +578,7 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -565,19 +596,18 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -665,8 +695,8 @@ urls:
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 			Expect(len(generalRecipeNewRepo.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
-			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase())
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -688,27 +718,26 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdirnewrepo)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
 
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec3))
 
 			Expect(errs).To(BeEmpty())
 
-			_, errs = c2.CompileParallel(false, compiler.NewLuetCompilationspecs(spec2))
+			_, errs = c2.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade_old_repo"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			repoupgrade, err := GenerateRepository("test", "description", "disk", []string{tmpdirnewrepo}, 1, tmpdirnewrepo, []string{"../../tests/fixtures/upgrade_new_repo"}, pkg.NewInMemoryDatabase(false))
+			repoupgrade, err := stubRepo(tmpdirnewrepo, "../../tests/fixtures/upgrade_new_repo")
 			Expect(err).ToNot(HaveOccurred())
-			err = repoupgrade.Write(tmpdirnewrepo, false)
+			err = repoupgrade.Write(tmpdirnewrepo, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			fakeroot, err := ioutil.TempDir("", "fakeroot")
@@ -797,7 +826,12 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(
+				backend.NewSimpleDockerBackend(),
+				generalRecipe.GetDatabase(),
+				options.Concurrency(2),
+				options.WithCompressionType(compression.GZip),
+			)
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -814,19 +848,18 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
-			c.SetCompressionType(compiler.GZip)
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
@@ -899,6 +932,47 @@ urls:
 
 	})
 
+	Context("Uninstallation", func() {
+		It("fails if package is required by others which are installed", func() {
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1, CheckConflicts: true})
+
+			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B := pkg.NewPackage("calamares", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+			C := pkg.NewPackage("kpmcore", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+			Z := pkg.NewPackage("chromium", "", []*pkg.DefaultPackage{A}, []*pkg.DefaultPackage{})
+			F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{Z, B}, []*pkg.DefaultPackage{})
+
+			Z.SetVersion("86.0.4240.193+2")
+			Z.SetCategory("www-client")
+			B.SetVersion("3.2.32.1+5")
+			B.SetCategory("app-admin")
+			C.SetVersion("4.2.0+2")
+			C.SetCategory("sys-libs-5")
+			D.SetVersion("5.19.5+9")
+			D.SetCategory("layers")
+
+			for _, p := range []pkg.Package{A, B, C, D, Z, F} {
+				_, err := systemDB.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = inst.Uninstall(system, D)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
 	Context("Existing files", func() {
 		It("Reclaims them", func() {
 			//repo:=NewLuetSystemRepository()
@@ -914,7 +988,9 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.Concurrency(2),
+				options.WithCompressionType(compression.GZip))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -931,19 +1007,17 @@ urls:
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(2)
-			c.SetCompressionType(compiler.GZip)
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
@@ -1019,7 +1093,8 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(),
+				options.WithCompressionType(compression.GZip))
 
 			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1033,19 +1108,17 @@ urls:
 			defer os.RemoveAll(tmpdir) // clean up
 			spec.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
-			c.SetConcurrency(1)
-			c.SetCompressionType(compiler.GZip)
-			_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec3))
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec3))
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/upgrade_old_repo"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
@@ -1111,7 +1184,7 @@ urls:
 
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(3))
 
-			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase())
 
 			spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -1121,14 +1194,14 @@ urls:
 			defer os.RemoveAll(tmpdir2) // clean up
 			spec.SetOutputPath(tmpdir2)
 
-			_, errs = c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec))
+			_, errs = c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 
 			Expect(errs).To(BeEmpty())
 
-			repo, err = GenerateRepository("test", "description", "disk", []string{tmpdir2}, 1, tmpdir2, []string{"../../tests/fixtures/upgrade_new_repo"}, pkg.NewInMemoryDatabase(false))
+			repo, err = stubRepo(tmpdir2, "../../tests/fixtures/upgrade_new_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			err = repo.Write(tmpdir2, false)
+			err = repo.Write(tmpdir2, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			inst = NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})

pkg/installer/interface.go

@@ -16,60 +16,13 @@
 package installer
 
 import (
-	compiler "github.com/mudler/luet/pkg/compiler"
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/tree"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	//"github.com/mudler/luet/pkg/solver"
 )
 
-type Installer interface {
-	Install(pkg.Packages, *System) error
-	Uninstall(pkg.Package, *System) error
-	Upgrade(s *System) error
-	Reclaim(s *System) error
-
-	Repositories([]Repository)
-	SyncRepositories(bool) (Repositories, error)
-	Swap(pkg.Packages, pkg.Packages, *System) error
-}
-
 type Client interface {
-	DownloadArtifact(compiler.Artifact) (compiler.Artifact, error)
+	DownloadArtifact(*artifact.PackageArtifact) (*artifact.PackageArtifact, error)
 	DownloadFile(string) (string, error)
 }
 
-type Repositories []Repository
-
-type Repository interface {
-	GetName() string
-	GetDescription() string
-	GetUrls() []string
-	SetUrls([]string)
-	AddUrl(string)
-	GetPriority() int
-	GetIndex() compiler.ArtifactIndex
-	SetIndex(i compiler.ArtifactIndex)
-	GetTree() tree.Builder
-	SetTree(tree.Builder)
-	Write(path string, resetRevision bool) error
-	Sync(bool) (Repository, error)
-	GetTreePath() string
-	SetTreePath(string)
-	GetMetaPath() string
-	SetMetaPath(string)
-	GetType() string
-	SetType(string)
-	SetAuthentication(map[string]string)
-	GetAuthentication() map[string]string
-	GetRevision() int
-	IncrementRevision()
-	GetLastUpdate() string
-	SetLastUpdate(string)
-	Client() Client
-
-	SetPriority(int)
-	GetRepositoryFile(string) (LuetRepositoryFile, error)
-	SetRepositoryFile(string, LuetRepositoryFile)
-	SetName(p string)
-	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
-}
+type Repositories []*LuetSystemRepository

pkg/installer/repository_docker.go

@@ -0,0 +1,270 @@
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+
+	"github.com/mudler/luet/pkg/compiler/backend"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/mudler/luet/pkg/bus"
+	compiler "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	"github.com/pkg/errors"
+)
+
+type dockerRepositoryGenerator struct {
+	b                compiler.CompilerBackend
+	imagePrefix      string
+	imagePush, force bool
+}
+
+func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
+
+	Info("Generating docker images for packages in", l.imagePrefix)
+	var art []*artifact.PackageArtifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			Debug("Skipping", info.Name(), err.Error())
+			return nil
+		}
+		if info.IsDir() {
+			Debug("Skipping directories")
+			return nil
+		}
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil
+		}
+
+		if err := l.pushImageFromArtifact(artifact.NewPackageArtifact(currentpath), l.b); err != nil {
+			return errors.Wrap(err, "while pushing metadata file associated to the artifact")
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		a, err := artifact.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+		// Set the path relative to the file.
+		// The metadata contains the full path where the file was located during buildtime.
+		a.Path = filepath.Join(filepath.Dir(currentpath), filepath.Base(a.Path))
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(a.CompileSpec.Package); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				a.CompileSpec.Package.HumanReadableString()))
+			return nil
+		}
+
+		packageImage := fmt.Sprintf("%s:%s", l.imagePrefix, a.CompileSpec.GetPackage().ImageID())
+
+		if l.imagePush && l.b.ImageAvailable(packageImage) && !l.force {
+			Info("Image", packageImage, "already present, skipping. use --force-push to override")
+		} else {
+			Info("Generating final image", packageImage,
+				"for package ", a.CompileSpec.GetPackage().HumanReadableString())
+			if opts, err := a.GenerateFinalImage(packageImage, l.b, true); err != nil {
+				return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+			}
+		}
+		if l.imagePush {
+			if err := pushImage(l.b, packageImage, l.force); err != nil {
+				return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage)
+			}
+		}
+
+		art = append(art, a)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
+func pushImage(b compiler.CompilerBackend, image string, force bool) error {
+	if b.ImageAvailable(image) && !force {
+		Debug("Image", image, "already present, skipping")
+		return nil
+	}
+	return b.Push(backend.Options{ImageName: image})
+}
+
+func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string) error {
+	Debug("Generating image", imageTree)
+	if opts, err := a.GenerateFinalImage(imageTree, d.b, false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
+	}
+	if d.imagePush {
+		if err := pushImage(d.b, imageTree, true); err != nil {
+			return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree)
+		}
+	}
+	return nil
+}
+
+func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec string, r *LuetSystemRepository) error {
+	// create temp dir for metafile
+	metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaDir) // clean up
+
+	tempRepoFile := filepath.Join(metaDir, REPOSITORY_SPECFILE+".tar")
+	if err := helpers.Tar(repospec, tempRepoFile); err != nil {
+		return errors.Wrap(err, "Error met while archiving repository file")
+	}
+
+	a := artifact.NewPackageArtifact(tempRepoFile)
+	imageRepo := fmt.Sprintf("%s:%s", d.imagePrefix, REPOSITORY_SPECFILE)
+
+	if err := d.pushFileFromArtifact(a, imageRepo); err != nil {
+		return errors.Wrap(err, "while pushing file from artifact")
+	}
+	return nil
+}
+
+func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, b compiler.CompilerBackend) error {
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	treeArchive, err := artifact.CreateArtifactForFile(a.Path)
+	if err != nil {
+		return errors.Wrap(err, "failed generating checksums for tree")
+	}
+	imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, helpers.StripInvalidStringsFromImage(a.GetFileName()))
+
+	return d.pushFileFromArtifact(treeArchive, imageTree)
+}
+
+// Generate creates a Docker luet repository
+func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefix string, resetRevision bool) error {
+	// - Iterate over meta, build final images, push them if necessary
+	//   - while pushing, check if image already exists, and if exist push them only if --force is supplied
+	// - Generate final images for metadata and push
+
+	imageRepository := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
+
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
+
+	repoTemp, err := config.LuetCfg.GetSystem().TempDir("repo")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for repository")
+	}
+	defer os.RemoveAll(repoTemp) // clean up
+
+	if r.GetBackend().ImageAvailable(imageRepository) {
+		if err := r.GetBackend().DownloadImage(backend.Options{ImageName: imageRepository}); err != nil {
+			return errors.Wrapf(err, "while downloading '%s'", imageRepository)
+		}
+
+		if err := r.GetBackend().ExtractRootfs(backend.Options{ImageName: imageRepository, Destination: repoTemp}, false); err != nil {
+			return errors.Wrapf(err, "while extracting '%s'", imageRepository)
+		}
+	}
+
+	repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE)
+
+	// Increment the internal revision version by reading the one which is already available (if any)
+	if err := r.BumpRevision(repospec, resetRevision); err != nil {
+		return err
+	}
+
+	Info(fmt.Sprintf(
+		"For repository %s creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imageRepository,
+	})
+
+	// Create tree and repository file
+	a, err := r.AddTree(r.GetTree(), repoTemp, REPOFILE_TREE_KEY, NewDefaultTreeRepositoryFile())
+	if err != nil {
+		return errors.Wrap(err, "error met while adding runtime tree to repository")
+	}
+
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+		return errors.Wrap(err, "error met while pushing runtime tree")
+	}
+
+	a, err = r.AddTree(r.BuildTree, repoTemp, REPOFILE_COMPILER_TREE_KEY, NewDefaultCompilerTreeRepositoryFile())
+	if err != nil {
+		return errors.Wrap(err, "error met while adding compiler tree to repository")
+	}
+	// we generate a new archive containing the required compressed file.
+	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
+	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+		return errors.Wrap(err, "error met while pushing compiler tree")
+	}
+
+	// create temp dir for metafile
+	metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaDir) // clean up
+
+	a, err = r.AddMetadata(repospec, metaDir)
+	if err != nil {
+		return errors.Wrap(err, "failed adding Metadata file to repository")
+	}
+
+	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+		return errors.Wrap(err, "error met while pushing docker image from artifact")
+	}
+
+	if err := d.pushRepoMetadata(repospec, r); err != nil {
+		return errors.Wrap(err, "while pushing repository metadata tree")
+	}
+
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imagePrefix,
+	})
+	return nil
+}

pkg/installer/repository_local.go

@@ -0,0 +1,132 @@
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/mudler/luet/pkg/bus"
+	"github.com/pkg/errors"
+)
+
+type localRepositoryGenerator struct{}
+
+func (l *localRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
+	return buildPackageIndex(path, db)
+}
+
+func buildPackageIndex(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
+
+	var art []*artifact.PackageArtifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			Debug("Failed walking", err.Error())
+			return err
+		}
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil // Skip with no errors
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		a, err := artifact.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(a.CompileSpec.GetPackage()); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				a.CompileSpec.GetPackage().HumanReadableString()))
+			return nil
+		}
+
+		art = append(art, a)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
+// Generate creates a Local luet repository
+func (*localRepositoryGenerator) Generate(r *LuetSystemRepository, dst string, resetRevision bool) error {
+	err := os.MkdirAll(dst, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
+
+	repospec := filepath.Join(dst, REPOSITORY_SPECFILE)
+	// Increment the internal revision version by reading the one which is already available (if any)
+	if err := r.BumpRevision(repospec, resetRevision); err != nil {
+		return err
+	}
+
+	Info(fmt.Sprintf(
+		"Repository %s: creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
+
+	if _, err := r.AddTree(r.GetTree(), dst, REPOFILE_TREE_KEY, NewDefaultTreeRepositoryFile()); err != nil {
+		return errors.Wrap(err, "error met while adding runtime tree to repository")
+	}
+
+	if _, err := r.AddTree(r.BuildTree, dst, REPOFILE_COMPILER_TREE_KEY, NewDefaultCompilerTreeRepositoryFile()); err != nil {
+		return errors.Wrap(err, "error met while adding compiler tree to repository")
+	}
+
+	if _, err := r.AddMetadata(repospec, dst); err != nil {
+		return errors.Wrap(err, "failed adding Metadata file to repository")
+	}
+
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: dst,
+	})
+	return nil
+}

pkg/installer/repository_test.go

@@ -1,4 +1,4 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+// Copyright © 2019 Ettore Di Giacinto <mudler@sabayon.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -19,21 +19,37 @@ import (
 
 	//	. "github.com/mudler/luet/pkg/installer"
 
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
 	"github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	config "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
+func dockerStubRepo(tmpdir, tree, image string, push, force bool) (*LuetSystemRepository, error) {
+	return GenerateRepository(
+		"test",
+		"description",
+		"docker",
+		[]string{image},
+		1,
+		tmpdir,
+		[]string{tree},
+		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force, false, nil)
+}
+
 var _ = Describe("Repository", func() {
 	Context("Generation", func() {
 		It("Generate repository metadata", func() {
@@ -49,7 +65,7 @@ var _ = Describe("Repository", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -60,16 +76,15 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -84,13 +99,13 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
@@ -117,11 +132,11 @@ var _ = Describe("Repository", func() {
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase())
 			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -133,23 +148,21 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(1)
-			compiler2.SetConcurrency(1)
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			artifact2, err := compiler2.Compile(false, spec2)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact2.GetPath())).To(BeTrue())
-			Expect(helpers.Untar(artifact2.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact2.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact2.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -166,13 +179,13 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
 			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
 
-			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, pkg.NewInMemoryDatabase(false))
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
@@ -216,6 +229,239 @@ urls:
 			Expect(matches).To(Equal([]PackageMatch{{Repo: repo1, Package: package1}}))
 
 		})
+	})
+	Context("Docker repository", func() {
+		repoImage := os.Getenv("UNIT_TEST_DOCKER_IMAGE_REPOSITORY")
 
+		BeforeEach(func() {
+			if repoImage == "" {
+				Skip("UNIT_TEST_DOCKER_IMAGE_REPOSITORY not specified")
+			}
+		})
+
+		It("generates images", func() {
+			b := backend.NewSimpleDockerBackend()
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+
+			spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			a, err := localcompiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(a.Path)).To(BeTrue())
+			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/buildable", repoImage, true, true)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(repoImage, false, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "tree.tar.gz"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.meta.yaml.tar"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.yaml"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "b-test-1.0"))).To(BeTrue())
+
+			extracted, err := ioutil.TempDir("", "extracted")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(extracted) // clean up
+
+			c := repo.Client()
+
+			f, err := c.DownloadFile("repository.yaml")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(f)).To(ContainSubstring("name: test"))
+
+			a, err = c.DownloadArtifact(&artifact.PackageArtifact{
+				Path: "test.tar",
+				CompileSpec: &compilerspec.LuetCompilationSpec{
+					Package: &pkg.DefaultPackage{
+						Name:     "b",
+						Category: "test",
+						Version:  "1.0",
+					},
+				},
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(a.Unpack(extracted, false)).ToNot(HaveOccurred())
+			Expect(helpers.Read(filepath.Join(extracted, "test6"))).To(Equal("artifact6\n"))
+		})
+
+		It("generates images of virtual packages", func() {
+			b := backend.NewSimpleDockerBackend()
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/virtuals")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(5))
+
+			localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+
+			spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.99"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			a, err := localcompiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(a.Path)).To(BeTrue())
+			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/virtuals", repoImage, true, true)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(repoImage, false, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "tree.tar.gz"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.meta.yaml.tar"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "repository.yaml"))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", repoImage, "a-test-1.99"))).To(BeTrue())
+
+			extracted, err := ioutil.TempDir("", "extracted")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(extracted) // clean up
+
+			c := repo.Client()
+
+			f, err := c.DownloadFile("repository.yaml")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Read(f)).To(ContainSubstring("name: test"))
+
+			a, err = c.DownloadArtifact(&artifact.PackageArtifact{
+				Path: "test.tar",
+				CompileSpec: &compilerspec.LuetCompilationSpec{
+					Package: &pkg.DefaultPackage{
+						Name:     "a",
+						Category: "test",
+						Version:  "1.99",
+					},
+				},
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(a.Unpack(extracted, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.DirectoryIsEmpty(extracted)).To(BeFalse())
+			content, err := ioutil.ReadFile(filepath.Join(extracted, ".virtual"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(content)).To(Equal(""))
+		})
+
+		It("Searches files", func() {
+			repos := Repositories{
+				&LuetSystemRepository{
+					Index: compiler.ArtifactIndex{
+						&artifact.PackageArtifact{
+							CompileSpec: &compilerspec.LuetCompilationSpec{
+								Package: &pkg.DefaultPackage{},
+							},
+							Path:  "bar",
+							Files: []string{"boo"},
+						},
+						&artifact.PackageArtifact{
+							Path:  "d",
+							Files: []string{"baz"},
+						},
+					},
+				},
+			}
+
+			matches := repos.SearchPackages("bo", FileSearch)
+			Expect(len(matches)).To(Equal(1))
+			Expect(matches[0].Artifact.Path).To(Equal("bar"))
+		})
+
+		It("Searches packages", func() {
+			repo := &LuetSystemRepository{
+				Index: compiler.ArtifactIndex{
+					&artifact.PackageArtifact{
+						Path: "foo",
+						CompileSpec: &compilerspec.LuetCompilationSpec{
+							Package: &pkg.DefaultPackage{
+								Name:     "foo",
+								Category: "bar",
+								Version:  "1.0",
+							},
+						},
+					},
+					&artifact.PackageArtifact{
+						Path: "baz",
+						CompileSpec: &compilerspec.LuetCompilationSpec{
+							Package: &pkg.DefaultPackage{
+								Name:     "foo",
+								Category: "baz",
+								Version:  "1.0",
+							},
+						},
+					},
+				},
+			}
+
+			a, err := repo.SearchArtefact(&pkg.DefaultPackage{
+				Name:     "foo",
+				Category: "baz",
+				Version:  "1.0",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(a.Path).To(Equal("baz"))
+
+			a, err = repo.SearchArtefact(&pkg.DefaultPackage{
+				Name:     "foo",
+				Category: "bar",
+				Version:  "1.0",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(a.Path).To(Equal("foo"))
+
+			// Doesn't exist. so must fail
+			_, err = repo.SearchArtefact(&pkg.DefaultPackage{
+				Name:     "foo",
+				Category: "bar",
+				Version:  "1.1",
+			})
+			Expect(err).To(HaveOccurred())
+		})
 	})
 })

pkg/installer/system.go

@@ -1,12 +1,11 @@
 package installer
 
 import (
-	. "github.com/mudler/luet/pkg/logger"
-
+	"github.com/hashicorp/go-multierror"
 	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
-	"github.com/pkg/errors"
 )
 
 type System struct {
@@ -20,28 +19,35 @@ func (s *System) World() (pkg.Packages, error) {
 
 type templatedata map[string]interface{}
 
-func (s *System) ExecuteFinalizers(packs []pkg.Package, force bool) error {
+func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
+	var errs error
 	executedFinalizer := map[string]bool{}
 	for _, p := range packs {
 		if helpers.Exists(p.Rel(tree.FinalizerFile)) {
 			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile))
-			if err != nil && !force {
-				return errors.Wrap(err, "reading file "+p.Rel(tree.FinalizerFile))
+			if err != nil {
+				Warning("Failed rendering finalizer for ", p.HumanReadableString(), err.Error())
+				errs = multierror.Append(errs, err)
+				continue
 			}
 
 			if _, exists := executedFinalizer[p.GetFingerPrint()]; !exists {
+				executedFinalizer[p.GetFingerPrint()] = true
 				Info("Executing finalizer for " + p.HumanReadableString())
 				finalizer, err := NewLuetFinalizerFromYaml([]byte(out))
-				if err != nil && !force {
-					return errors.Wrap(err, "Error reading finalizer "+p.Rel(tree.FinalizerFile))
+				if err != nil {
+					Warning("Failed reading finalizer for ", p.HumanReadableString(), err.Error())
+					errs = multierror.Append(errs, err)
+					continue
 				}
 				err = finalizer.RunInstall(s)
-				if err != nil && !force {
-					return errors.Wrap(err, "Error executing install finalizer "+p.Rel(tree.FinalizerFile))
+				if err != nil {
+					Warning("Failed running finalizer for ", p.HumanReadableString(), err.Error())
+					errs = multierror.Append(errs, err)
+					continue
 				}
-				executedFinalizer[p.GetFingerPrint()] = true
 			}
 		}
 	}
-	return nil
+	return errs
 }

pkg/logger/logger.go

@@ -3,9 +3,12 @@ package logger
 import (
 	"fmt"
 	"os"
+	"path"
 	"regexp"
+	"runtime"
 	"strings"
 
+	"sync"
 	. "github.com/mudler/luet/pkg/config"
 
 	"github.com/briandowns/spinner"
@@ -18,7 +21,7 @@ import (
 var s *spinner.Spinner = nil
 var z *zap.Logger = nil
 var aurora Aurora = nil
-
+var spinnerLock = sync.Mutex{}
 func NewSpinner() {
 	if s == nil {
 		s = spinner.New(
@@ -82,6 +85,8 @@ func ZapLogger() error {
 }
 
 func Spinner(i int) {
+	spinnerLock.Lock()
+	defer spinnerLock.Unlock()
 	var confLevel int
 	if LuetCfg.GetGeneral().Debug {
 		confLevel = 3
@@ -118,6 +123,8 @@ func SpinnerText(suffix, prefix string) {
 }
 
 func SpinnerStop() {
+	spinnerLock.Lock()
+	defer spinnerLock.Unlock()
 	var confLevel int
 	if LuetCfg.GetGeneral().Debug {
 		confLevel = 3
@@ -171,7 +178,7 @@ func level2AtomicLevel(level string) zap.AtomicLevel {
 	}
 }
 
-func msg(level string, withoutColor bool, msg ...interface{}) {
+func Msg(level string, withoutColor, ln bool, msg ...interface{}) {
 	var message string
 	var confLevel, msgLevel int
 
@@ -196,13 +203,13 @@ func msg(level string, withoutColor bool, msg ...interface{}) {
 	} else {
 		switch level {
 		case "warning":
-			levelMsg = Bold(Yellow(":construction: " + message)).BgBlack().String()
+			levelMsg = Yellow(":construction: warning" + message).BgBlack().String()
 		case "debug":
 			levelMsg = White(message).BgBlack().String()
 		case "info":
 			levelMsg = message
 		case "error":
-			levelMsg = Bold(Red(":bomb: " + message + ":fire:")).BgBlack().String()
+			levelMsg = Red(message).String()
 		}
 	}
 
@@ -217,37 +224,47 @@ func msg(level string, withoutColor bool, msg ...interface{}) {
 		log2File(level, message)
 	}
 
-	fmt.Println(levelMsg)
+	if ln {
+		fmt.Println(levelMsg)
+	} else {
+		fmt.Print(levelMsg)
+	}
+
 }
 
 func Warning(mess ...interface{}) {
-	msg("warning", false, mess...)
+	Msg("warning", false, true, mess...)
 	if LuetCfg.GetGeneral().FatalWarns {
 		os.Exit(2)
 	}
 }
 
 func Debug(mess ...interface{}) {
-	msg("debug", false, mess...)
+	pc, file, line, ok := runtime.Caller(1)
+	if ok {
+		mess = append([]interface{}{fmt.Sprintf("DEBUG (%s:#%d:%v)",
+			path.Base(file), line, runtime.FuncForPC(pc).Name())}, mess...)
+	}
+	Msg("debug", false, true, mess...)
 }
 
 func DebugC(mess ...interface{}) {
-	msg("debug", true, mess...)
+	Msg("debug", true, true, mess...)
 }
 
 func Info(mess ...interface{}) {
-	msg("info", false, mess...)
+	Msg("info", false, true, mess...)
 }
 
 func InfoC(mess ...interface{}) {
-	msg("info", true, mess...)
+	Msg("info", true, true, mess...)
 }
 
 func Error(mess ...interface{}) {
-	msg("error", false, mess...)
+	Msg("error", false, true, mess...)
 }
 
 func Fatal(mess ...interface{}) {
-	Error(mess)
+	Error(mess...)
 	os.Exit(1)
 }

pkg/package/database.go

@@ -28,6 +28,9 @@ type PackageDatabase interface {
 }
 
 type PackageSet interface {
+	Clone(PackageDatabase) error
+	Copy() (PackageDatabase, error)
+
 	GetRevdeps(p Package) (Packages, error)
 	GetPackages() []string //Ids
 	CreatePackage(pkg Package) (string, error)
@@ -49,6 +52,7 @@ type PackageSet interface {
 	FindPackageLabel(labelKey string) (Packages, error)
 	FindPackageLabelMatch(pattern string) (Packages, error)
 	FindPackageMatch(pattern string) (Packages, error)
+	FindPackageByFile(pattern string) (Packages, error)
 }
 
 type PackageFile struct {

pkg/package/database_boltdb.go

@@ -47,6 +47,14 @@ func NewBoltDatabase(path string) PackageDatabase {
 	return &BoltDatabase{Path: path, ProvidesDatabase: map[string]map[string]Package{}}
 }
 
+func (db *BoltDatabase) Clone(to PackageDatabase) error {
+	return clone(db, to)
+}
+
+func (db *BoltDatabase) Copy() (PackageDatabase, error) {
+	return copy(db)
+}
+
 func (db *BoltDatabase) Get(s string) (string, error) {
 	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
@@ -90,9 +98,9 @@ func (db *BoltDatabase) Retrieve(ID string) ([]byte, error) {
 // TODO: Have a memory instance for boltdb, so we don't compute each time we get called
 // as this is REALLY expensive. But we don't perform usually those operations in a file db.
 func (db *BoltDatabase) GetRevdeps(p Package) (Packages, error) {
-	memory := NewInMemoryDatabase(false)
-	for _, p := range db.World() {
-		memory.CreatePackage(p)
+	memory, err := db.Copy()
+	if err != nil {
+		return nil, errors.New("Failed copying bolt db to memory")
 	}
 	return memory.GetRevdeps(p)
 }
@@ -341,17 +349,18 @@ func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
 
 	required, err := db.FindPackage(p)
 	if err != nil {
-
+		err = nil
 		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
 		packages, err := p.Expand(db)
 		//	Info("Expanded", packages, err)
 		if err != nil || len(packages) == 0 {
 			required = p
+			err = errors.Wrap(err, "Candidate not found")
 		} else {
 			required = packages.Best(nil)
 
 		}
-		return required, nil
+		return required, err
 		//required = &DefaultPackage{Name: "test"}
 	}
 
@@ -362,10 +371,22 @@ func (db *BoltDatabase) FindPackageCandidate(p Package) (Package, error) {
 // FindPackages return the list of the packages beloging to cat/name  (any versions in requested range)
 // FIXME: Optimize, see inmemorydb
 func (db *BoltDatabase) FindPackages(p Package) (Packages, error) {
+	if !p.IsSelector() {
+		pack, err := db.FindPackage(p)
+		if err != nil {
+			return []Package{}, err
+		}
+		return []Package{pack}, nil
+	}
+
 	// Provides: Treat as the replaced package here
 	if provided, err := db.getProvide(p); err == nil {
 		p = provided
+		if !provided.IsSelector() {
+			return Packages{provided}, nil
+		}
 	}
+
 	var versionsInWorld []Package
 	for _, w := range db.World() {
 		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {
@@ -415,9 +436,9 @@ func (db *BoltDatabase) FindPackageLabel(labelKey string) (Packages, error) {
 func (db *BoltDatabase) FindPackageLabelMatch(pattern string) (Packages, error) {
 	var ans []Package
 
-	re := regexp.MustCompile(pattern)
-	if re == nil {
-		return nil, errors.New("Invalid regex " + pattern + "!")
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, errors.Wrap(err, "Invalid regex "+pattern+"!")
 	}
 
 	for _, pack := range db.World() {
@@ -429,12 +450,15 @@ func (db *BoltDatabase) FindPackageLabelMatch(pattern string) (Packages, error)
 	return Packages(ans), nil
 }
 
+func (db *BoltDatabase) FindPackageByFile(pattern string) (Packages, error) {
+	return findPackageByFile(db, pattern)
+}
 func (db *BoltDatabase) FindPackageMatch(pattern string) (Packages, error) {
 	var ans []Package
 
-	re := regexp.MustCompile(pattern)
-	if re == nil {
-		return nil, errors.New("Invalid regex " + pattern + "!")
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, errors.Wrap(err, "Invalid regex "+pattern+"!")
 	}
 
 	for _, pack := range db.World() {

pkg/package/database_boltdb_test.go

@@ -57,6 +57,31 @@ var _ = Describe("BoltDB Database", func() {
 
 		})
 
+		It("Find package files", func() {
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = db.SetPackageFiles(&PackageFile{PackageFingerprint: a.GetFingerPrint(), Files: []string{"foo"}})
+			Expect(err).ToNot(HaveOccurred())
+
+			err = db.SetPackageFiles(&PackageFile{PackageFingerprint: a1.GetFingerPrint(), Files: []string{"bar"}})
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.FindPackageByFile("fo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(pack)).To(Equal(1))
+			Expect(pack[0]).To(Equal(a))
+		})
+
 		It("Expands correctly", func() {
 
 			a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
@@ -81,7 +106,6 @@ var _ = Describe("BoltDB Database", func() {
 		})
 
 		It("Find best package candidate", func() {
-			db := NewInMemoryDatabase(false)
 			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
 			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
 			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
@@ -102,7 +126,6 @@ var _ = Describe("BoltDB Database", func() {
 		})
 
 		It("Find specific package candidate", func() {
-			db := NewInMemoryDatabase(false)
 			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
 			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
 			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
@@ -123,7 +146,6 @@ var _ = Describe("BoltDB Database", func() {
 		})
 
 		It("Provides replaces definitions", func() {
-			db := NewInMemoryDatabase(false)
 			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
 			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
 			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
@@ -147,6 +169,144 @@ var _ = Describe("BoltDB Database", func() {
 			Expect(pack).To(Equal(a3))
 		})
 
+		Context("Provides", func() {
+
+			It("replaces definitions", func() {
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				pack, err := db.FindPackage(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(pack).To(Equal(a3))
+			})
+
+			It("replaces definitions", func() {
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(a3))
+			})
+
+			It("replaces definitions", func() {
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of unexisting packages", func() {
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of a required package", func() {
+
+				c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = db.CreatePackage(c)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			When("Searching with selectors", func() {
+				It("replaces definitions of a required package", func() {
+
+					c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+					z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+					z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+					Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+					_, err := db.CreatePackage(z)
+					Expect(err).ToNot(HaveOccurred())
+					_, err = db.CreatePackage(c)
+					Expect(err).ToNot(HaveOccurred())
+
+					s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					packs, err := db.FindPackages(s)
+					Expect(err).ToNot(HaveOccurred())
+					Expect(packs).To(ContainElement(z))
+				})
+			})
+
+		})
+
 	})
 
 })

pkg/package/database_common.go

@@ -0,0 +1,68 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+import (
+	"regexp"
+
+	"github.com/pkg/errors"
+)
+
+func clone(src, dst PackageDatabase) error {
+	for _, i := range src.World() {
+		_, err := dst.CreatePackage(i)
+		if err != nil {
+			return errors.Wrap(err, "Failed create package "+i.HumanReadableString())
+		}
+	}
+	return nil
+}
+
+func copy(src PackageDatabase) (PackageDatabase, error) {
+	dst := NewInMemoryDatabase(false)
+
+	if err := clone(src, dst); err != nil {
+		return dst, errors.Wrap(err, "Failed create temporary in-memory db")
+	}
+
+	return dst, nil
+}
+
+func findPackageByFile(db PackageDatabase, pattern string) (Packages, error) {
+
+	var ans []Package
+
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, errors.Wrap(err, "Invalid regex "+pattern+"!")
+	}
+
+PACKAGE:
+	for _, pack := range db.World() {
+		files, err := db.GetPackageFiles(pack)
+		if err == nil {
+			for _, f := range files {
+				if re.MatchString(f) {
+					ans = append(ans, pack)
+					continue PACKAGE
+				}
+			}
+		}
+	}
+
+	return Packages(ans), nil
+
+}

pkg/package/database_mem.go

@@ -32,6 +32,7 @@ var DBInMemoryInstance = &InMemoryDatabase{
 	CacheNoVersion:   map[string]map[string]interface{}{},
 	ProvidesDatabase: map[string]map[string]Package{},
 	RevDepsDatabase:  map[string]map[string]Package{},
+	cached:           map[string]interface{}{},
 }
 
 type InMemoryDatabase struct {
@@ -41,6 +42,7 @@ type InMemoryDatabase struct {
 	CacheNoVersion   map[string]map[string]interface{}
 	ProvidesDatabase map[string]map[string]Package
 	RevDepsDatabase  map[string]map[string]Package
+	cached           map[string]interface{}
 }
 
 func NewInMemoryDatabase(singleton bool) PackageDatabase {
@@ -53,6 +55,7 @@ func NewInMemoryDatabase(singleton bool) PackageDatabase {
 			CacheNoVersion:   map[string]map[string]interface{}{},
 			ProvidesDatabase: map[string]map[string]Package{},
 			RevDepsDatabase:  map[string]map[string]Package{},
+			cached:           map[string]interface{}{},
 		}
 	}
 	return DBInMemoryInstance
@@ -197,6 +200,15 @@ func (db *InMemoryDatabase) populateCaches(p Package) {
 
 	// Create extra cache between package -> []versions
 	db.Lock()
+	if db.cached == nil {
+		db.cached = map[string]interface{}{}
+	}
+
+	if _, ok := db.cached[p.GetFingerPrint()]; ok {
+		db.Unlock()
+		return
+	}
+	db.cached[p.GetFingerPrint()] = nil
 
 	// Provides: Store package provides, we will reuse this when walking deps
 	for _, provide := range pd.Provides {
@@ -213,6 +225,25 @@ func (db *InMemoryDatabase) populateCaches(p Package) {
 		db.CacheNoVersion[p.GetPackageName()] = make(map[string]interface{})
 	}
 	db.CacheNoVersion[p.GetPackageName()][p.GetVersion()] = nil
+
+	// Updating Revdeps
+	// Given that when we populate the cache we don't have the full db at hand
+	// We cycle over reverse dependency of a package to update their entry if they are matching
+	// the version selector
+	toUpdate, ok := db.RevDepsDatabase[pd.GetPackageName()]
+	if ok {
+		for _, pp := range toUpdate {
+			for _, re := range pp.GetRequires() {
+				if match, _ := pd.VersionMatchSelector(re.GetVersion(), nil); match {
+					_, ok = db.RevDepsDatabase[pd.GetFingerPrint()]
+					if !ok {
+						db.RevDepsDatabase[pd.GetFingerPrint()] = make(map[string]Package)
+					}
+					db.RevDepsDatabase[pd.GetFingerPrint()][pp.GetFingerPrint()] = pp
+				}
+			}
+		}
+	}
 	db.Unlock()
 
 	for _, re := range pd.GetRequires() {
@@ -225,12 +256,23 @@ func (db *InMemoryDatabase) populateCaches(p Package) {
 				db.RevDepsDatabase[pa.GetFingerPrint()] = make(map[string]Package)
 			}
 			db.RevDepsDatabase[pa.GetFingerPrint()][pd.GetFingerPrint()] = pd
+			_, ok = db.RevDepsDatabase[pa.GetPackageName()]
+			if !ok {
+				db.RevDepsDatabase[pa.GetPackageName()] = make(map[string]Package)
+			}
+			db.RevDepsDatabase[pa.GetPackageName()][pd.GetPackageName()] = pd
 		}
 		_, ok := db.RevDepsDatabase[re.GetFingerPrint()]
 		if !ok {
 			db.RevDepsDatabase[re.GetFingerPrint()] = make(map[string]Package)
 		}
 		db.RevDepsDatabase[re.GetFingerPrint()][pd.GetFingerPrint()] = pd
+		_, ok = db.RevDepsDatabase[re.GetPackageName()]
+		if !ok {
+			db.RevDepsDatabase[re.GetPackageName()] = make(map[string]Package)
+		}
+		db.RevDepsDatabase[re.GetPackageName()][pd.GetPackageName()] = pd
+
 		db.Unlock()
 	}
 }
@@ -270,6 +312,14 @@ func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
 	return db.FindPackage(pa)
 }
 
+func (db *InMemoryDatabase) Clone(to PackageDatabase) error {
+	return clone(db, to)
+}
+
+func (db *InMemoryDatabase) Copy() (PackageDatabase, error) {
+	return copy(db)
+}
+
 func (db *InMemoryDatabase) encodePackage(p Package) (string, string, error) {
 	pd, ok := p.(*DefaultPackage)
 	if !ok {
@@ -330,6 +380,9 @@ func (db *InMemoryDatabase) FindPackages(p Package) (Packages, error) {
 	// Provides: Treat as the replaced package here
 	if provided, err := db.getProvide(p); err == nil {
 		p = provided
+		if !provided.IsSelector() {
+			return Packages{provided}, nil
+		}
 	}
 
 	db.Lock()
@@ -431,16 +484,17 @@ func (db *InMemoryDatabase) FindPackageCandidate(p Package) (Package, error) {
 
 	required, err := db.FindPackage(p)
 	if err != nil {
+		err = nil
 		//	return nil, errors.Wrap(err, "Couldn't find required package in db definition")
 		packages, err := p.Expand(db)
 		//	Info("Expanded", packages, err)
 		if err != nil || len(packages) == 0 {
 			required = p
+			err = errors.Wrap(err, "Candidate not found")
 		} else {
 			required = packages.Best(nil)
-
 		}
-		return required, nil
+		return required, err
 		//required = &DefaultPackage{Name: "test"}
 	}
 
@@ -506,3 +560,7 @@ func (db *InMemoryDatabase) FindPackageMatch(pattern string) (Packages, error) {
 
 	return Packages(ans), nil
 }
+
+func (db *InMemoryDatabase) FindPackageByFile(pattern string) (Packages, error) {
+	return findPackageByFile(db, pattern)
+}

pkg/package/database_mem_test.go

@@ -77,6 +77,32 @@ var _ = Describe("Database", func() {
 
 		})
 
+		It("Find package files", func() {
+			db := NewInMemoryDatabase(false)
+			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+			_, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a1)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = db.CreatePackage(a3)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = db.SetPackageFiles(&PackageFile{PackageFingerprint: a.GetFingerPrint(), Files: []string{"foo"}})
+			Expect(err).ToNot(HaveOccurred())
+
+			err = db.SetPackageFiles(&PackageFile{PackageFingerprint: a1.GetFingerPrint(), Files: []string{"bar"}})
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.FindPackageByFile("fo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(pack)).To(Equal(1))
+			Expect(pack[0]).To(Equal(a))
+		})
+
 		It("Find specific package candidate", func() {
 			db := NewInMemoryDatabase(false)
 			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
@@ -98,30 +124,150 @@ var _ = Describe("Database", func() {
 
 		})
 
-		It("Provides replaces definitions", func() {
-			db := NewInMemoryDatabase(false)
-			a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
-			a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
-			a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+		Context("Provides", func() {
 
-			a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
-			Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+			It("replaces definitions", func() {
+				db := NewInMemoryDatabase(false)
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
 
-			_, err := db.CreatePackage(a)
-			Expect(err).ToNot(HaveOccurred())
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
 
-			_, err = db.CreatePackage(a1)
-			Expect(err).ToNot(HaveOccurred())
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
 
-			_, err = db.CreatePackage(a3)
-			Expect(err).ToNot(HaveOccurred())
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
 
-			s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				pack, err := db.FindPackage(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(pack).To(Equal(a3))
+			})
+
+			It("replaces definitions", func() {
+				db := NewInMemoryDatabase(false)
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				a3 := NewPackage("A", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				a3.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}})
+				Expect(a3.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: "1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a3)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(a3))
+			})
+
+			It("replaces definitions", func() {
+				db := NewInMemoryDatabase(false)
+				a := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of unexisting packages", func() {
+				db := NewInMemoryDatabase(false)
+				a1 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(a1)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			It("replaces definitions of a required package", func() {
+				db := NewInMemoryDatabase(false)
+
+				c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+				z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+				z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+				Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+				_, err := db.CreatePackage(z)
+				Expect(err).ToNot(HaveOccurred())
+				_, err = db.CreatePackage(c)
+				Expect(err).ToNot(HaveOccurred())
+
+				s := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+				packs, err := db.FindPackages(s)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(packs).To(ContainElement(z))
+			})
+
+			When("Searching with selectors", func() {
+				It("replaces definitions of a required package", func() {
+					db := NewInMemoryDatabase(false)
+
+					c := NewPackage("C", "1.1", []*DefaultPackage{{Name: "A", Category: "", Version: ">=0"}}, []*DefaultPackage{})
+					z := NewPackage("Z", "1.3", []*DefaultPackage{}, []*DefaultPackage{})
+
+					z.SetProvides([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}})
+					Expect(z.GetProvides()).To(Equal([]*DefaultPackage{{Name: "A", Category: "", Version: ">=1.0"}}))
+
+					_, err := db.CreatePackage(z)
+					Expect(err).ToNot(HaveOccurred())
+					_, err = db.CreatePackage(c)
+					Expect(err).ToNot(HaveOccurred())
+
+					s := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
+					packs, err := db.FindPackages(s)
+					Expect(err).ToNot(HaveOccurred())
+					Expect(packs).To(ContainElement(z))
+				})
+			})
 
-			pack, err := db.FindPackage(s)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(pack).To(Equal(a3))
 		})
+
 	})
 
 })

pkg/package/package.go

@@ -46,6 +46,7 @@ type Package interface {
 
 	GetFingerPrint() string
 	GetPackageName() string
+	ImageID() string
 	Requires([]*DefaultPackage) Package
 	Conflicts([]*DefaultPackage) Package
 	Revdeps(PackageDatabase) Packages
@@ -112,6 +113,12 @@ type Package interface {
 	GetBuildTimestamp() string
 
 	Clone() Package
+
+	GetMetadataFilePath() string
+	SetTreeDir(s string)
+	GetTreeDir() string
+
+	JSON() ([]byte, error)
 }
 
 type Tree interface {
@@ -209,6 +216,11 @@ func (t *DefaultPackage) JSON() ([]byte, error) {
 	return buffer.Bytes(), err
 }
 
+// GetMetadataFilePath returns the canonical name of an artifact metadata file
+func (d *DefaultPackage) GetMetadataFilePath() string {
+	return d.GetFingerPrint() + ".metadata.yaml"
+}
+
 // DefaultPackage represent a standard package definition
 type DefaultPackage struct {
 	ID               int               `storm:"id,increment" json:"id"` // primary key with auto increment
@@ -234,6 +246,8 @@ type DefaultPackage struct {
 	BuildTimestamp string   `json:"buildtimestamp,omitempty"`
 
 	Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
+
+	TreeDir string `json:"treedir,omitempty"`
 }
 
 // State represent the package state
@@ -250,6 +264,12 @@ func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*D
 	}
 }
 
+func (p *DefaultPackage) SetTreeDir(s string) {
+	p.TreeDir = s
+}
+func (p *DefaultPackage) GetTreeDir() string {
+	return p.TreeDir
+}
 func (p *DefaultPackage) String() string {
 	b, err := p.JSON()
 	if err != nil {
@@ -288,6 +308,10 @@ func (p *DefaultPackage) GetPackageName() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.Category)
 }
 
+func (p *DefaultPackage) ImageID() string {
+	return helpers.StripInvalidStringsFromImage(p.GetFingerPrint())
+}
+
 // GetBuildTimestamp returns the package build timestamp
 func (p *DefaultPackage) GetBuildTimestamp() string {
 	return p.BuildTimestamp
@@ -621,6 +645,16 @@ func (set Packages) Best(v version.Versioner) Package {
 	return versionsMap[sorted[len(sorted)-1]]
 }
 
+func (set Packages) Find(packageName string) (Package, error) {
+	for _, p := range set {
+		if p.GetPackageName() == packageName {
+			return p, nil
+		}
+	}
+
+	return &DefaultPackage{}, errors.New("package not found")
+}
+
 func (set Packages) Unique() Packages {
 	var result Packages
 	uniq := make(map[string]Package)

pkg/package/package_test.go

@@ -66,6 +66,17 @@ var _ = Describe("Package", func() {
 		})
 	})
 
+	Context("ImageID", func() {
+		It("Returns a correct ImageID escaping unsupported chars", func() {
+			p := NewPackage("A", "1.0+p1", []*DefaultPackage{}, []*DefaultPackage{})
+			Expect(p.ImageID()).To(Equal("A--1.0-p1"))
+		})
+		It("Returns a correct ImageID", func() {
+			p := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			Expect(p.ImageID()).To(Equal("A--1.0"))
+		})
+	})
+
 	Context("Find label on packages", func() {
 		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a.AddLabel("project1", "test1")

pkg/solver/parallel.go

@@ -76,7 +76,7 @@ func (s *Parallel) noRulesInstalled() bool {
 	return true
 }
 
-func (s *Parallel) buildParallelFormula(formulas []bf.Formula, packages pkg.Packages) (bf.Formula, error) {
+func (s *Parallel) buildParallelFormula(db pkg.PackageDatabase, formulas []bf.Formula, packages pkg.Packages) (bf.Formula, error) {
 	var wg = new(sync.WaitGroup)
 	var wg2 = new(sync.WaitGroup)
 
@@ -87,7 +87,7 @@ func (s *Parallel) buildParallelFormula(formulas []bf.Formula, packages pkg.Pack
 		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
 			defer wg.Done()
 			for p := range c {
-				solvable, err := p.BuildFormula(s.DefinitionDatabase, s.ParallelDatabase)
+				solvable, err := p.BuildFormula(db, s.ParallelDatabase)
 				if err != nil {
 					panic(err)
 				}
@@ -126,13 +126,13 @@ func (s *Parallel) BuildInstalled() (bf.Formula, error) {
 	var packages pkg.Packages
 	for _, p := range s.Installed() {
 		packages = append(packages, p)
-		for _, dep := range p.Related(s.DefinitionDatabase) {
+		for _, dep := range p.Related(s.InstalledDatabase) {
 			packages = append(packages, dep)
 		}
 
 	}
 
-	return s.buildParallelFormula(formulas, packages)
+	return s.buildParallelFormula(s.InstalledDatabase, formulas, packages)
 }
 
 // BuildWorld builds the formula which olds the requirements from the package definitions
@@ -148,7 +148,7 @@ func (s *Parallel) BuildWorld(includeInstalled bool) (bf.Formula, error) {
 		//f = bf.And(f, solvable)
 		formulas = append(formulas, solvable)
 	}
-	return s.buildParallelFormula(formulas, s.World())
+	return s.buildParallelFormula(s.DefinitionDatabase, formulas, s.World())
 }
 
 // BuildWorld builds the formula which olds the requirements from the package definitions
@@ -200,7 +200,7 @@ func (s *Parallel) BuildPartialWorld(includeInstalled bool) (bf.Formula, error)
 	close(results)
 	wg2.Wait()
 
-	return s.buildParallelFormula(formulas, packages)
+	return s.buildParallelFormula(s.DefinitionDatabase, formulas, packages)
 
 	//return s.buildParallelFormula(formulas, s.World())
 }
@@ -446,10 +446,9 @@ func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAsse
 	removed := pkg.Packages{}
 
 	// TODO: this is memory expensive, we need to optimize this
-	universe := pkg.NewInMemoryDatabase(false)
-
-	for _, p := range s.DefinitionDatabase.World() {
-		universe.CreatePackage(p)
+	universe, err := s.DefinitionDatabase.Copy()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't build world copy")
 	}
 	for _, p := range s.Installed() {
 		universe.CreatePackage(p)
@@ -471,23 +470,22 @@ func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAsse
 		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
 			defer wg.Done()
 			for p := range c {
-				available, err := universe.FindPackageVersions(p)
-				if err != nil {
-					removed = append(removed, p) /// FIXME: Racy
-				}
-				if len(available) == 0 {
+				available, err := s.DefinitionDatabase.FindPackageVersions(p)
+				if len(available) == 0 || err != nil {
+					removed = append(removed, p)
 					continue
 				}
 
 				bestmatch := available.Best(nil)
 				// Found a better version available
 				if !bestmatch.Matches(p) {
-					encodedP, _ := p.Encode(universe)
-					P := bf.Var(encodedP)
-					results <- bf.And(bf.Not(P), r)
-					encodedP, _ = bestmatch.Encode(universe)
-					P = bf.Var(encodedP)
-					results <- bf.And(P, r)
+					oldP, _ := p.Encode(universe)
+					toreplaceP := bf.Var(oldP)
+					best, _ := bestmatch.Encode(universe)
+					toUpgrade := bf.Var(best)
+
+					solvablenew, _ := bestmatch.BuildFormula(s.DefinitionDatabase, s.ParallelDatabase)
+					results <- bf.And(bf.Not(toreplaceP), bf.And(append(solvablenew, toUpgrade)...))
 				}
 			}
 		}(wg, all)
@@ -514,8 +512,7 @@ func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAsse
 	// Treat removed packages from universe as marked for deletion
 	if dropremoved {
 
-		// SAT encode the clauses against the world
-		for _, p := range removed {
+		for _, p := range removed.Unique() {
 			encodedP, err := p.Encode(universe)
 			if err != nil {
 				return nil, nil, errors.Wrap(err, "couldn't encode package")
@@ -560,9 +557,9 @@ func (s *Parallel) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAss
 	toInstall := pkg.Packages{}
 
 	// we do this in memory so we take into account of provides
-	universe := pkg.NewInMemoryDatabase(false)
-	for _, p := range s.DefinitionDatabase.World() {
-		universe.CreatePackage(p)
+	universe, err := s.DefinitionDatabase.Copy()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Could not copy def db")
 	}
 
 	installedcopy := pkg.NewInMemoryDatabase(false)
@@ -699,7 +696,7 @@ func (s *Parallel) Uninstall(checkconflicts, full bool, packs ...pkg.Package) (p
 		}
 	}
 
-	s2 := &Parallel{Concurrency: s.Concurrency, InstalledDatabase: pkg.NewInMemoryDatabase(false), DefinitionDatabase: s.DefinitionDatabase, ParallelDatabase: pkg.NewInMemoryDatabase(false)}
+	s2 := &Parallel{Concurrency: s.Concurrency, InstalledDatabase: pkg.NewInMemoryDatabase(false), DefinitionDatabase: s.InstalledDatabase, ParallelDatabase: pkg.NewInMemoryDatabase(false)}
 	s2.SetResolver(s.Resolver)
 	// Get the requirements to install the candidate
 	asserts, err := s2.Install(toRemove)
@@ -744,7 +741,7 @@ func (s *Parallel) Uninstall(checkconflicts, full bool, packs ...pkg.Package) (p
 func (s *Parallel) BuildFormula() (bf.Formula, error) {
 	var formulas []bf.Formula
 
-	r, err := s.BuildPartialWorld(false)
+	r, err := s.BuildWorld(false)
 	if err != nil {
 		return nil, err
 	}

pkg/solver/parallel_test.go

@@ -111,7 +111,7 @@ var _ = Describe("Parallel", func() {
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: true}))
 
-			Expect(len(solution)).To(Equal(3))
+			Expect(len(solution)).To(Equal(5))
 		})
 
 		It("Solves correctly if the selected package to install has requirements", func() {
@@ -401,7 +401,7 @@ var _ = Describe("Parallel", func() {
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(3))
+			Expect(len(solution)).To(Equal(4))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -529,7 +529,7 @@ var _ = Describe("Parallel", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(5))
+			Expect(len(solution)).To(Equal(6))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -570,7 +570,7 @@ var _ = Describe("Parallel", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(5))
+			Expect(len(solution)).To(Equal(6))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -671,8 +671,9 @@ var _ = Describe("Parallel", func() {
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
 
-				Expect(len(solution)).To(Equal(1))
+				Expect(len(solution)).To(Equal(2))
 			})
 
 			It("Uninstalls complex packages correctly, even if shared deps are required by system packages", func() {
@@ -1285,12 +1286,172 @@ var _ = Describe("Parallel", func() {
 			Expect(uninstall[0].GetName()).To(Equal("a"))
 			Expect(uninstall[0].GetVersion()).To(Equal("1.1"))
 
-			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: C, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
 
-			Expect(len(solution)).To(Equal(4))
+			Expect(len(solution)).To(Equal(3))
+		})
+
+		It("UpgradeUniverse upgrades correctly", func() {
+
+			D := pkg.NewPackage("d", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D.SetCategory("test")
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{
+				&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"},
+				&pkg.DefaultPackage{Name: "d", Version: ">=1.0", Category: "test"},
+			}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			C1 := pkg.NewPackage("c", "1.6", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			C1.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+
+			B1 := pkg.NewPackage("b", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "c", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			B1.SetCategory("test")
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+			A1 = pkg.NewPackage("a", "1.2", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A1.SetCategory("test")
+
+			for _, p := range []pkg.Package{A1, B, B1, C, C1, D} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B, D} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.UpgradeUniverse(false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(2))
+			Expect(uninstall).To(ContainElement(A))
+			Expect(uninstall).To(ContainElement(B))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: C1, Value: true}))
+
+			Expect(len(solution)).To(Equal(6))
+		})
+
+		It("UpgradeUniverse upgrades correctly", func() {
+
+			D := pkg.NewPackage("d", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D.SetCategory("test")
+			D1 := pkg.NewPackage("d", "1.6", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D1.SetCategory("test")
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{
+				&pkg.DefaultPackage{Name: "c", Version: ">=1.0", Category: "test"},
+			}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+
+			for _, p := range []pkg.Package{A, B, C, D, D1} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B, C, D} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.UpgradeUniverse(false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(1))
+			Expect(uninstall[0].GetName()).To(Equal("d"))
+			Expect(uninstall[0].GetVersion()).To(Equal("1.5"))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: true}))
+
+			Expect(len(solution)).To(Equal(3))
+		})
+
+		It("Upgrade upgrades correctly", func() {
+
+			D := pkg.NewPackage("d", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D.SetCategory("test")
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{
+				&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"},
+				&pkg.DefaultPackage{Name: "d", Version: ">=1.0", Category: "test"},
+			}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			C1 := pkg.NewPackage("c", "1.6", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			C1.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+
+			B1 := pkg.NewPackage("b", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "c", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			B1.SetCategory("test")
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+			A1 = pkg.NewPackage("a", "1.2", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A1.SetCategory("test")
+
+			for _, p := range []pkg.Package{A1, B, B1, C, C1, D} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B, D} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.Upgrade(false, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(2))
+			Expect(uninstall).To(ContainElement(A))
+			Expect(uninstall).To(ContainElement(B))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: C1, Value: true}))
+
+			Expect(len(solution)).To(Equal(6))
+		})
+
+		It("Upgrade upgrades correctly", func() {
+
+			D := pkg.NewPackage("d", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D.SetCategory("test")
+			D1 := pkg.NewPackage("d", "1.6", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D1.SetCategory("test")
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{
+				&pkg.DefaultPackage{Name: "c", Version: ">=1.0", Category: "test"},
+			}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+
+			for _, p := range []pkg.Package{A, B, C, D, D1} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B, C, D} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.Upgrade(false, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(1))
+			Expect(uninstall[0].GetName()).To(Equal("d"))
+			Expect(uninstall[0].GetVersion()).To(Equal("1.5"))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: true}))
+
+			Expect(len(solution)).To(Equal(5))
 		})
 	})
 })