Tag 0.19.2

Fixes https://github.com/mudler/luet/issues/250 as now it is documented
in the cli --help too.

.chglog/CHANGELOG.tpl.md

@@ -0,0 +1,49 @@
+{{ if .Versions -}}
+{{ if .Unreleased.CommitGroups -}}
+{{ range .Unreleased.CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ else }}
+{{ range .Unreleased.Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+
+{{ if .CommitGroups -}}
+{{ range .CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ else }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} (https://github.com/mudler/luet/commit/{{.Hash.Short}})
+{{ end }}
+{{ end -}}
+
+{{- if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+{{ range .Notes }}
+{{ .Body }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}
+
+{{- if .Versions }}
+[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD
+{{ range .Versions -}}
+{{ if .Tag.Previous -}}
+[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}
+{{ end -}}
+{{ end -}}
+{{ end -}}

.chglog/config.yml

@@ -0,0 +1,27 @@
+style: github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://github.com/mudler/luet
+options:
+  commits:
+    # filters:
+    #   Type:
+    #     - feat
+    #     - fix
+    #     - perf
+    #     - refactor
+  commit_groups:
+     title_maps:
+       feat: Features
+       fix: Bug Fixes
+       perf: Performance Improvements
+       refactor: Code Refactoring
+       ci: Continous Integration
+  header:
+    pattern: "(.*)"
+    pattern_maps:
+      - Subject
+  notes:
+    keywords:
+      - BREAKING CHANGE

.github/workflows/image.yml

@@ -0,0 +1,67 @@
+---
+name: 'build container images'
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - '*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=quay.io/luet/base
+          VERSION=latest
+          SHORTREF=${GITHUB_SHA::8}
+
+          # If this is git tag, use the tag name as a docker tag
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          fi
+          TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:${SHORTREF}"
+
+          # If the VERSION looks like a version number, assume that
+          # this is the most recent version of the image and also
+          # tag it 'latest'.
+          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
+            TAGS="$TAGS,${DOCKER_IMAGE}:latest"
+          fi
+
+          # Set output parameters.
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=docker_image::${DOCKER_IMAGE}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@master
+        with:
+          platforms: all
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
+
+      - name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          registry: quay.io
+          username: ${{ secrets.DOCKER_RELEASE_USERNAME }}
+          password: ${{ secrets.DOCKER_RELEASE_PASSWORD }}
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm
+          push: true
+          tags: ${{ steps.prep.outputs.tags }}

.github/workflows/pr.yml

@@ -0,0 +1,58 @@
+
+on: pull_request
+name: Build and Test
+jobs:
+  tests-integration:
+    strategy:
+      matrix:
+        go-version: [1.16.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+    - name: Tests with Img backend
+      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make test-integration
+  tests-unit:
+    strategy:
+      matrix:
+        go-version: [1.16.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+            make build
+            sudo cp -rf luet /usr/bin/luet
+    - name: Install GoReleaser
+      uses: goreleaser/goreleaser-action@v2
+      with:
+        install-only: true
+    - name: Build
+      run: sudo -E env "PATH=$PATH" make multiarch-build-small
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make test-coverage

.github/workflows/push.yml

@@ -0,0 +1,76 @@
+on: push
+concurrency:
+  group: registries-tests
+
+name: Build on push
+jobs:
+  tests-integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+      - name: Login to quay with img
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Tests with Img backend
+        run: |
+            sudo -E env "PATH=$PATH" \
+                    env "LUET_BACKEND=img" \
+                    make test-integration
+      - name: Tests
+        run: |
+              sudo -E \
+                    env "PATH=$PATH" \
+                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                    make test-integration
+
+  tests-unit:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+            make build
+            sudo cp -rf luet /usr/bin/luet
+      - name: Install GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          install-only: true
+      - name: Build test
+        run: sudo -E env "PATH=$PATH" make multiarch-build-small
+      - name: Tests
+        run: |
+              sudo -E \
+                    env "PATH=$PATH" \
+                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                    make test-coverage

.github/workflows/release.yml

@@ -1,14 +1,20 @@
-on: push
-name: Build and release on push
+on:
+  push:
+    tags:
+      - '*'  # only test and release when a tag is pushed
+concurrency:
+  group: registries-tests
+
+name: Test and Release on tag
 jobs:
-  release:
-    name: Test and Release
+  tests-integration:
+    name: Integration tests
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14.x
+          go-version: 1.16.x
       - name: Checkout code
         uses: actions/checkout@v2
       - name: setup-docker
@@ -17,31 +23,72 @@ jobs:
         run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
       - name: Install deps
         run: |
-            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
-            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
-            sudo chmod a+x "/usr/bin/img"
-      - name: Build test
-        run: sudo -E env "PATH=$PATH" make multiarch-build-small
+          sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+          sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+          sudo chmod a+x "/usr/bin/img"
       - name: Login to quay with img
         run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
       - name: Tests with Img backend
         run: |
-            sudo -E env "PATH=$PATH" \
-                    env "LUET_BACKEND=img" \
-                    make test-integration
+          sudo -E env "PATH=$PATH" \
+                  env "LUET_BACKEND=img" \
+                  make test-integration
       - name: Tests
         run: |
-              sudo -E \
-                    env "PATH=$PATH" \
-                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
-                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
-                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
-                    make test-integration test-coverage
-      - name: Build
-        run: sudo -E env "PATH=$PATH" make multiarch-build-small && sudo chmod -R 777 release/
-      - name: Release
-        uses: fnkr/github-action-ghr@v1
-        if: startsWith(github.ref, 'refs/tags/')
+          sudo -E \
+                env "PATH=$PATH" \
+                env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                make test-integration
+
+  tests-unit:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+          sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+          sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+          sudo chmod a+x "/usr/bin/img"
+          make build
+          sudo cp -rf luet /usr/bin/luet
+      - name: Tests
+        run: |
+          sudo -E \
+                env "PATH=$PATH" \
+                env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                make test-coverage
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    needs: [ "tests-integration","tests-unit" ]
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          version: latest
+          args: release --rm-dist
         env:
-          GHR_PATH: release/
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -1,30 +0,0 @@
-
-on: pull_request
-name: Build and Test
-jobs:
-  test:
-    strategy:
-      matrix:
-        go-version: [1.14.x]
-        platform: [ubuntu-latest]
-    runs-on: ${{ matrix.platform }}
-    steps:
-    - name: Install Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ matrix.go-version }}
-    - name: Checkout code
-      uses: actions/checkout@v2
-    - name: setup-docker
-      uses: docker-practice/actions-setup-docker@0.0.1
-    - name: Install deps
-      run: |
-            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
-            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
-            sudo chmod a+x "/usr/bin/img"
-    - name: Build
-      run: sudo -E env "PATH=$PATH" make multiarch-build-small
-    - name: Tests with Img backend
-      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
-    - name: Tests
-      run: sudo -E env "PATH=$PATH" make test-integration test-coverage

.gitignore

@@ -1,4 +1,5 @@
 *.swp
 luet
 tests/integration/shunit2
-tests/integration/bin
+tests/integration/bin
+release/

.goreleaser.yml

@@ -0,0 +1,43 @@
+before:
+  hooks:
+    - go mod tidy
+dist: release
+source:
+  enabled: true
+  name_template: '{{ .ProjectName }}-{{ .Tag }}-source'
+checksum:
+  name_template: '{{ .ProjectName }}-{{ .Tag }}-checksums.txt'
+builds:
+  -
+    env:
+      - CGO_ENABLED=0
+    ldflags:
+      - -s -w
+      - -X "github.com/mudler/luet/cmd.BuildTime={{ time "2006-01-02 15:04:05 MST" }}"
+      - -X "github.com/mudler/luet/cmd.BuildCommit={{ .FullCommit }}"
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm
+      - arm64
+      - 386
+    goarm:
+      - 6
+      - 7
+archives:
+  - format: binary # this removes the tar of the archives, leaving the binaries alone
+    name_template: luet-{{ .Tag }}-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+      - '^Merge pull request'
+release:
+  header: |
+    Luet is a multi-platform Package Manager based off from containers - it uses Docker (and others) to build packages.
+
+    It has zero dependencies and it is well suitable for "from scratch" environments.
+    It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded device

CONTRIBUTING.md

@@ -0,0 +1,53 @@
+
+We love your input! We want to make contributing to this project as easy and transparent as possible, whether it's:
+
+- Reporting a bug
+- Discussing the current state of the code
+- Submitting a fix
+- Proposing new features
+- Becoming a maintainer
+
+## We Develop with Github
+We use github to host code, to track issues and feature requests, as well as accept pull requests.
+
+## Stay in touch
+
+Join us in [slack](https://luet.slack.com/join/shared_invite/enQtOTQxMjcyNDQ0MDUxLWQ5ODVlNTI1MTYzNDRkYzkyYmM1YWE5YjM0NTliNDEzNmQwMTkxNDRhNDIzM2Y5NDBlOTZjZTYxYWQyNDE4YzY#/) and hang out with the community! It will be much easier to get started and do your first steps in contributing to the project.
+
+## All Code Changes Happen Through Pull Requests
+Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests:
+
+1. Fork the repo you want to contribute to and create your branch from `develop`.
+2. If you've added code that should be tested, add tests.
+3. If you've changed APIs, update the [documentation](https://github.com/Luet-lab/docs).
+4. Ensure the test suite passes.
+5. Make sure your code lints.
+6. Issue that pull request!
+
+## Any contributions you make will be under the Software License of the repository
+In short, when you submit code changes, your submissions are understood to be under the same License that covers the project. Feel free to contact the maintainers if that's a concern.
+
+## Report bugs using Github's [issues](https://github.com/mudler/luet/issues)
+We use GitHub issues to track public bugs. Report a bug by [opening a new issue](https://github.com/mudler/luet/issues/new); it's that easy!
+
+## Write bug reports with detail, background, and sample code
+Try to be as more descriptive as possible. When opening a new issue you will be prompted to choose between a bug or a feature request, with a small template to fill details with. Be specific!
+
+**Great Bug Reports** tend to have:
+
+- A quick summary and/or background
+- Steps to reproduce
+  - Be specific!
+  - Give sample code if you can. 
+- What you expected would happen
+- What actually happens
+- Notes (possibly including why you think this might be happening, or stuff you tried that didn't work)
+
+People *love* thorough bug reports.
+
+
+## License
+By contributing, you agree that your contributions will be licensed under the project Licenses.
+
+## References
+This document was adapted from the open-source contribution guidelines from https://gist.github.com/briandk/3d2e8b3ec8daf5a27a62

Makefile

@@ -6,10 +6,6 @@ override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildCommit=$(shell git rev-p
 NAME ?= luet
 PACKAGE_NAME ?= $(NAME)
 PACKAGE_CONFLICT ?= $(PACKAGE_NAME)-beta
-REVISION := $(shell git rev-parse --short HEAD || echo dev)
-VERSION := $(shell git describe --tags || echo $(REVISION))
-VERSION := $(shell echo $(VERSION) | sed -e 's/^v//g')
-BUILD_PLATFORMS ?= -osarch="linux/amd64" -osarch="linux/386" -osarch="linux/arm"
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
 .PHONY: all
@@ -89,10 +85,10 @@ test-docker:
 				bash -c "make test"
 
 multiarch-build:
-	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+	goreleaser build --snapshot --rm-dist
 
 multiarch-build-small:
-	@$(MAKE) LDFLAGS+="-s -w" multiarch-build
-	for file in $(ROOT_DIR)/release/* ; do \
+	@$(MAKE) multiarch-build
+	for file in $(ROOT_DIR)/release/**/* ; do \
 		upx --brute -1 $${file} ; \
-	done
+	done

README.md

@@ -1,19 +1,24 @@
+
+<p align="center">
+  <img width=150 height=150 src="https://user-images.githubusercontent.com/2420543/119691600-0293d700-be4b-11eb-827f-49ff1174a07a.png">
+</p>
+
 # luet - Container-based Package manager
 
 [![Docker Repository on Quay](https://quay.io/repository/luet/base/status "Docker Repository on Quay")](https://quay.io/repository/luet/base)
-[![Go Report Card](https://goreportcard.com/badge/github.com/mudler/luet)](https://goreportcard.com/report/github.com/mudler/luet)
-[![Build Status](https://travis-ci.org/mudler/luet.svg?branch=master)](https://travis-ci.org/mudler/luet)
+[![Build and release on push](https://github.com/mudler/luet/actions/workflows/release.yml/badge.svg)](https://github.com/mudler/luet/actions/workflows/release.yml)
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
 [![codecov](https://codecov.io/gh/mudler/luet/branch/master/graph/badge.svg)](https://codecov.io/gh/mudler/luet)
 
-[![asciicast](https://asciinema.org/a/388348.svg)](https://asciinema.org/a/388348)
-
 Luet is a multi-platform Package Manager based off from containers - it uses Docker (and others) to build packages. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
 
 It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/) in YAML notation to define both [packages](https://luet-lab.github.io/docs/docs/concepts/packages/) and [rootfs](https://luet-lab.github.io/docs/docs/concepts/packages/#package-layers). As it is based on containers, it can be also used to build stages for Linux From Scratch installations and it can build and track updates for those systems.
 
 It is written entirely in Golang and where used as package manager, it can run in from scratch environment, with zero dependencies.
 
+[![asciicast](https://asciinema.org/a/388348.svg)](https://asciinema.org/a/388348)
+
+
 ## In a glance
 
 - Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.

cmd/box/exec.go

@@ -21,8 +21,8 @@ import (
 
 	b64 "encoding/base64"
 
+	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/box"
-	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/spf13/cobra"
 )
@@ -54,18 +54,18 @@ func NewBoxExecCommand() *cobra.Command {
 
 				args = ss
 			}
-			Info("Executing", args, "in", rootfs)
+			util.DefaultContext.Info("Executing", args, "in", rootfs)
 
 			b := box.NewBox(entrypoint, args, mounts, envs, rootfs, stdin, stdout, stderr)
 			err := b.Run()
 			if err != nil {
-				Fatal(err)
+				util.DefaultContext.Fatal(err)
 			}
 		},
 	}
 	path, err := os.Getwd()
 	if err != nil {
-		Fatal(err)
+		util.DefaultContext.Fatal(err)
 	}
 	ans.Flags().String("rootfs", path, "Rootfs path")
 	ans.Flags().Bool("stdin", false, "Attach to stdin")

cmd/build.go

@@ -21,15 +21,14 @@ import (
 
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/installer"
 
 	"github.com/mudler/luet/pkg/compiler/types/compression"
 	"github.com/mudler/luet/pkg/compiler/types/options"
-	. "github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
@@ -74,7 +73,7 @@ Build packages specifying multiple definition trees:
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
 		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
-		viper.BindPFlag("values", cmd.Flags().Lookup("values"))
+		util.BindValuesFlags(cmd)
 		viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
 
 		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
@@ -83,26 +82,24 @@ Build packages specifying multiple definition trees:
 		viper.BindPFlag("wait", cmd.Flags().Lookup("wait"))
 		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
 
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
-		LuetCfg.Viper.BindPFlag("general.show_build_output", cmd.Flags().Lookup("live-output"))
-		LuetCfg.Viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
+		util.BindSolverFlags(cmd)
+
+		viper.BindPFlag("general.show_build_output", cmd.Flags().Lookup("live-output"))
+		viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
 
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
 		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("destination")
-		concurrency := LuetCfg.GetGeneral().Concurrency
+		concurrency := util.DefaultContext.Config.GetGeneral().Concurrency
 		backendType := viper.GetString("backend")
 		privileged := viper.GetBool("privileged")
 		revdeps := viper.GetBool("revdeps")
 		all := viper.GetBool("all")
 		compressionType := viper.GetString("compression")
 		imageRepository := viper.GetString("image-repository")
-		values := viper.GetStringSlice("values")
+		values := util.ValuesFlags()
 		wait := viper.GetBool("wait")
 		push := viper.GetBool("push")
 		pull := viper.GetBool("pull")
@@ -113,13 +110,12 @@ Build packages specifying multiple definition trees:
 		full, _ := cmd.Flags().GetBool("full")
 		rebuild, _ := cmd.Flags().GetBool("rebuild")
 
-		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		var results Results
 		backendArgs := viper.GetStringSlice("backend-args")
 
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
-			LuetCfg.GetLogging().SetLogLevel("error")
+			util.DefaultContext.Config.GetLogging().SetLogLevel("error")
 		}
 		pretend, _ := cmd.Flags().GetBool("pretend")
 		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
@@ -127,7 +123,7 @@ Build packages specifying multiple definition trees:
 		compilerSpecs := compilerspec.NewLuetCompilationspecs()
 		var db pkg.PackageDatabase
 
-		compilerBackend, err := compiler.NewBackend(backendType)
+		compilerBackend, err := compiler.NewBackend(util.DefaultContext, backendType)
 		helpers.CheckErr(err)
 
 		db = pkg.NewInMemoryDatabase(false)
@@ -136,40 +132,26 @@ Build packages specifying multiple definition trees:
 		generalRecipe := tree.NewCompilerRecipe(db)
 
 		if fromRepo {
-			if err := installer.LoadBuildTree(generalRecipe, db, LuetCfg); err != nil {
-				Warning("errors while loading trees from repositories", err.Error())
+			if err := installer.LoadBuildTree(generalRecipe, db, util.DefaultContext); err != nil {
+				util.DefaultContext.Warning("errors while loading trees from repositories", err.Error())
 			}
 		}
 
 		for _, src := range treePaths {
-			Info("Loading tree", src)
+			util.DefaultContext.Info("Loading tree", src)
 			helpers.CheckErr(generalRecipe.Load(src))
 		}
 
-		Info("Building in", dst)
+		util.DefaultContext.Info("Building in", dst)
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		opts := util.SetSolverConfig(util.DefaultContext)
 		pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
-		LuetCfg.GetGeneral().ShowBuildOutput = LuetCfg.Viper.GetBool("general.show_build_output")
+		util.DefaultContext.Config.GetGeneral().ShowBuildOutput = viper.GetBool("general.show_build_output")
 
-		opts := &LuetSolverOptions{
-			Type:        stype,
-			LearnRate:   float32(rate),
-			Discount:    float32(discount),
-			MaxAttempts: attempts,
-		}
+		util.DefaultContext.Debug("Solver", opts.CompactString())
 
-		Debug("Solver", opts.CompactString())
-
-		if concurrent {
-			opts.Options = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
-		} else {
-			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
-		}
+		opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
 
 		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
 			options.NoDeps(nodeps),
@@ -179,12 +161,14 @@ Build packages specifying multiple definition trees:
 			options.WithPullRepositories(pullRepo),
 			options.WithPushRepository(imageRepository),
 			options.Rebuild(rebuild),
+			options.WithTemplateFolder(util.TemplateFolders(util.DefaultContext, fromRepo, treePaths)),
 			options.WithSolverOptions(*opts),
 			options.Wait(wait),
 			options.OnlyTarget(onlyTarget),
 			options.PullFirst(pull),
 			options.KeepImg(keepImages),
 			options.OnlyDeps(onlydeps),
+			options.WithContext(util.DefaultContext),
 			options.BackendArgs(backendArgs),
 			options.Concurrency(concurrency),
 			options.WithCompressionType(compression.Implementation(compressionType)),
@@ -193,10 +177,10 @@ Build packages specifying multiple definition trees:
 		if full {
 			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
 			if err != nil {
-				Fatal(err.Error())
+				util.DefaultContext.Fatal(err.Error())
 			}
 			for _, spec := range specs {
-				Info(":package: Selecting ", spec.GetPackage().GetName(), spec.GetPackage().GetVersion())
+				util.DefaultContext.Info(":package: Selecting ", spec.GetPackage().GetName(), spec.GetPackage().GetVersion())
 
 				compilerSpecs.Add(spec)
 			}
@@ -204,12 +188,12 @@ Build packages specifying multiple definition trees:
 			for _, a := range args {
 				pack, err := helpers.ParsePackageStr(a)
 				if err != nil {
-					Fatal("Invalid package string ", a, ": ", err.Error())
+					util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
 				}
 
 				spec, err := luetCompiler.FromPackage(pack)
 				if err != nil {
-					Fatal("Error: " + err.Error())
+					util.DefaultContext.Fatal("Error: " + err.Error())
 				}
 
 				spec.SetOutputPath(dst)
@@ -221,9 +205,9 @@ Build packages specifying multiple definition trees:
 			for _, p := range w {
 				spec, err := luetCompiler.FromPackage(p)
 				if err != nil {
-					Fatal("Error: " + err.Error())
+					util.DefaultContext.Fatal("Error: " + err.Error())
 				}
-				Info(":package: Selecting ", p.GetName(), p.GetVersion())
+				util.DefaultContext.Info(":package: Selecting ", p.GetName(), p.GetVersion())
 				spec.SetOutputPath(dst)
 				compilerSpecs.Add(spec)
 			}
@@ -247,11 +231,12 @@ Build packages specifying multiple definition trees:
 			}
 
 			for _, sp := range toCalculate {
-				packs, err := luetCompiler.ComputeDepTree(sp)
+				ht := compiler.NewHashTree(generalRecipe.GetDatabase())
+				hashTree, err := ht.Query(luetCompiler, sp)
 				if err != nil {
 					errs = append(errs, err)
 				}
-				for _, p := range packs {
+				for _, p := range hashTree.Dependencies {
 					results.Packages = append(results.Packages,
 						PackageResult{
 							Name:       p.Package.GetName(),
@@ -281,7 +266,7 @@ Build packages specifying multiple definition trees:
 				fmt.Println(string(j2))
 			case "terminal":
 				for _, p := range results.Packages {
-					Info(p.String())
+					util.DefaultContext.Info(p.String())
 				}
 			}
 		} else {
@@ -290,12 +275,12 @@ Build packages specifying multiple definition trees:
 		}
 		if len(errs) != 0 {
 			for _, e := range errs {
-				Error("Error: " + e.Error())
+				util.DefaultContext.Error("Error: " + e.Error())
 			}
-			Fatal("Bailing out")
+			util.DefaultContext.Fatal("Bailing out")
 		}
 		for _, a := range artifact {
-			Info("Artifact generated:", a.Path)
+			util.DefaultContext.Info("Artifact generated:", a.Path)
 		}
 	},
 }
@@ -303,7 +288,7 @@ Build packages specifying multiple definition trees:
 func init() {
 	path, err := os.Getwd()
 	if err != nil {
-		Fatal(err)
+		util.DefaultContext.Fatal(err)
 	}
 
 	buildCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the tree to use.")
@@ -330,7 +315,7 @@ func init() {
 	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
-	buildCmd.Flags().Bool("live-output", LuetCfg.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
+	buildCmd.Flags().Bool("live-output", util.DefaultContext.Config.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
 	buildCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
 	buildCmd.Flags().Bool("rebuild", false, "To combine with --pull. Allows to rebuild the target package even if an image is available, against a local values file")
 	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")

cmd/cleanup.go

@@ -16,14 +16,13 @@
 package cmd
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
-	. "github.com/mudler/luet/pkg/config"
-	config "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/cmd/util"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	"github.com/spf13/cobra"
 )
@@ -33,46 +32,35 @@ var cleanupCmd = &cobra.Command{
 	Short: "Clean packages cache.",
 	Long:  `remove downloaded packages tarballs and clean cache directory`,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+		util.BindSystemFlags(cmd)
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var cleaned int = 0
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := config.LuetCfg.Viper.GetString("system.rootfs")
-		engine := config.LuetCfg.Viper.GetString("system.database_engine")
-
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig(util.DefaultContext)
 		// Check if cache dir exists
-		if helpers.Exists(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
+		if fileHelper.Exists(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath()) {
 
-			files, err := ioutil.ReadDir(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
+			files, err := ioutil.ReadDir(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath())
 			if err != nil {
-				Fatal("Error on read cachedir ", err.Error())
+				util.DefaultContext.Fatal("Error on read cachedir ", err.Error())
 			}
 
 			for _, file := range files {
-				if file.IsDir() {
-					continue
-				}
 
-				if LuetCfg.GetGeneral().Debug {
-					Info("Removing ", file.Name())
+				if util.DefaultContext.Config.GetGeneral().Debug {
+					util.DefaultContext.Info("Removing ", file.Name())
 				}
 
 				err := os.RemoveAll(
-					filepath.Join(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
+					filepath.Join(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
 				if err != nil {
-					Fatal("Error on removing", file.Name())
+					util.DefaultContext.Fatal("Error on removing", file.Name())
 				}
 				cleaned++
 			}
 		}
 
-		Info("Cleaned: ", cleaned, "packages.")
+		util.DefaultContext.Info(fmt.Sprintf("Cleaned: %d files from %s", cleaned, util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath()))
 
 	},
 }

cmd/config.go

@@ -18,8 +18,7 @@ package cmd
 import (
 	"fmt"
 
-	config "github.com/mudler/luet/pkg/config"
-	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/cmd/util"
 
 	"github.com/spf13/cobra"
 )
@@ -30,46 +29,12 @@ var configCmd = &cobra.Command{
 	Long:    `Show luet configuration`,
 	Aliases: []string{"c"},
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println(config.LuetCfg.GetLogging())
-		fmt.Println(config.LuetCfg.GetGeneral())
-		fmt.Println(config.LuetCfg.GetSystem())
-		if len(config.LuetCfg.CacheRepositories) > 0 {
-			fmt.Println("repetitors:")
-			for _, r := range config.LuetCfg.CacheRepositories {
-				fmt.Println("  - ", r.String())
-			}
-		}
-		if len(config.LuetCfg.SystemRepositories) > 0 {
-			fmt.Println("repositories:")
-			for _, r := range config.LuetCfg.SystemRepositories {
-				fmt.Println("  - ", r.String())
-			}
+		data, err := util.DefaultContext.Config.YAML()
+		if err != nil {
+			util.DefaultContext.Fatal(err.Error())
 		}
 
-		if len(config.LuetCfg.RepositoriesConfDir) > 0 {
-			fmt.Println("repos_confdir:")
-			for _, dir := range config.LuetCfg.RepositoriesConfDir {
-				fmt.Println("  - ", dir)
-			}
-		}
-
-		if len(config.LuetCfg.ConfigProtectConfDir) > 0 {
-
-			// Load config protect configs
-			installer.LoadConfigProtectConfs(config.LuetCfg)
-
-			fmt.Println("config_protect_confdir:")
-			for _, dir := range config.LuetCfg.ConfigProtectConfDir {
-				fmt.Println("  - ", dir)
-			}
-
-			if len(config.LuetCfg.GetConfigProtectConfFiles()) > 0 {
-				fmt.Println("protect_conf_files:")
-				for _, file := range config.LuetCfg.GetConfigProtectConfFiles() {
-					fmt.Println("  - ", file.String())
-				}
-			}
-		}
+		fmt.Println(string(data))
 	},
 }
 

cmd/create-repo.go

@@ -19,9 +19,9 @@ import (
 	"path/filepath"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/types/compression"
-	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 
 	//	. "github.com/mudler/luet/pkg/logger"
@@ -69,7 +69,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename"))
 		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
 		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
-
+		viper.BindPFlag("from-metadata", cmd.Flags().Lookup("from-metadata"))
 		viper.BindPFlag("force-push", cmd.Flags().Lookup("force-push"))
 		viper.BindPFlag("push-images", cmd.Flags().Lookup("push-images"))
 
@@ -80,7 +80,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 
 		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("output")
-		packages := viper.GetString("packages")
+
 		name := viper.GetString("name")
 		descr := viper.GetString("descr")
 		urls := viper.GetStringSlice("urls")
@@ -96,14 +96,26 @@ Create a repository from the metadata description defined in the luet.yaml confi
 
 		treeFile := installer.NewDefaultTreeRepositoryFile()
 		metaFile := installer.NewDefaultMetaRepositoryFile()
-		compilerBackend, err := compiler.NewBackend(backendType)
+		compilerBackend, err := compiler.NewBackend(util.DefaultContext, backendType)
 		helpers.CheckErr(err)
 		force := viper.GetBool("force-push")
 		imagePush := viper.GetBool("push-images")
 
+		opts := []installer.RepositoryOption{
+			installer.WithSource(viper.GetString("packages")),
+			installer.WithPushImages(imagePush),
+			installer.WithForce(force),
+			installer.FromRepository(fromRepo),
+			installer.WithImagePrefix(dst),
+			installer.WithDatabase(pkg.NewInMemoryDatabase(false)),
+			installer.WithCompilerBackend(compilerBackend),
+			installer.FromMetadata(viper.GetBool("from-metadata")),
+			installer.WithContext(util.DefaultContext),
+		}
+
 		if source_repo != "" {
 			// Search for system repository
-			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
+			lrepo, err := util.DefaultContext.Config.GetSystemRepository(source_repo)
 			helpers.CheckErr(err)
 
 			if len(treePaths) <= 0 {
@@ -114,27 +126,28 @@ Create a repository from the metadata description defined in the luet.yaml confi
 				t = lrepo.Type
 			}
 
-			repo, err = installer.GenerateRepository(lrepo.Name,
-				lrepo.Description, t,
-				lrepo.Urls,
-				lrepo.Priority,
-				packages,
-				treePaths,
-				pkg.NewInMemoryDatabase(false),
-				compilerBackend,
-				dst,
-				imagePush,
-				force,
-				fromRepo,
-				LuetCfg)
-			helpers.CheckErr(err)
+			opts = append(opts,
+				installer.WithName(lrepo.Name),
+				installer.WithDescription(lrepo.Description),
+				installer.WithType(t),
+				installer.WithUrls(lrepo.Urls...),
+				installer.WithPriority(lrepo.Priority),
+				installer.WithTree(treePaths...),
+			)
 
 		} else {
-			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
-				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force, fromRepo, LuetCfg)
-			helpers.CheckErr(err)
+			opts = append(opts,
+				installer.WithName(name),
+				installer.WithDescription(descr),
+				installer.WithType(t),
+				installer.WithUrls(urls...),
+				installer.WithTree(treePaths...),
+			)
 		}
 
+		repo, err = installer.GenerateRepository(opts...)
+		helpers.CheckErr(err)
+
 		if treetype != "" {
 			treeFile.SetCompressionType(compression.Implementation(treetype))
 		}
@@ -154,7 +167,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
-		err = repo.Write(dst, reset, true)
+		err = repo.Write(util.DefaultContext, dst, reset, true)
 		helpers.CheckErr(err)
 
 	},
@@ -177,6 +190,7 @@ func init() {
 
 	createrepoCmd.Flags().Bool("force-push", false, "Force overwrite of docker images if already present online")
 	createrepoCmd.Flags().Bool("push-images", false, "Enable/Disable docker image push for docker repositories")
+	createrepoCmd.Flags().Bool("from-metadata", false, "Consider metadata files from the packages folder while indexing the new tree")
 
 	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")

cmd/database/create.go

@@ -18,13 +18,11 @@ package cmd_database
 import (
 	"io/ioutil"
 
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/cmd/util"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 
-	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
-	. "github.com/mudler/luet/pkg/config"
-
 	"github.com/spf13/cobra"
 )
 
@@ -45,43 +43,39 @@ The yaml must contain the package definition, and the file list at least.
 For reference, inspect a "metadata.yaml" file generated while running "luet build"`,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 
-			dbpath := LuetCfg.Viper.GetString("system.database_path")
-			rootfs := LuetCfg.Viper.GetString("system.rootfs")
-			engine := LuetCfg.Viper.GetString("system.database_engine")
-
-			LuetCfg.System.DatabaseEngine = engine
-			LuetCfg.System.DatabasePath = dbpath
-			LuetCfg.System.Rootfs = rootfs
-
-			systemDB := LuetCfg.GetSystemDB()
+			util.SetSystemConfig(util.DefaultContext)
+			systemDB := util.DefaultContext.Config.GetSystemDB()
 
 			for _, a := range args {
 				dat, err := ioutil.ReadFile(a)
 				if err != nil {
-					Fatal("Failed reading ", a, ": ", err.Error())
+					util.DefaultContext.Fatal("Failed reading ", a, ": ", err.Error())
 				}
 				art, err := artifact.NewPackageArtifactFromYaml(dat)
 				if err != nil {
-					Fatal("Failed reading yaml ", a, ": ", err.Error())
+					util.DefaultContext.Fatal("Failed reading yaml ", a, ": ", err.Error())
 				}
 
 				files := art.Files
 
-				if _, err := systemDB.CreatePackage(art.CompileSpec.GetPackage()); err != nil {
-					Fatal("Failed to create ", a, ": ", err.Error())
-				}
-				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.CompileSpec.GetPackage().GetFingerPrint(), Files: files}); err != nil {
-					Fatal("Failed setting package files for ", a, ": ", err.Error())
+				// Check if the package is already present
+				if p, err := systemDB.FindPackage(art.CompileSpec.GetPackage()); err == nil && p.GetName() != "" {
+					util.DefaultContext.Fatal("Package", art.CompileSpec.GetPackage().HumanReadableString(),
+						" already present.")
 				}
 
-				Info(art.CompileSpec.GetPackage().HumanReadableString(), " created")
+				if _, err := systemDB.CreatePackage(art.CompileSpec.GetPackage()); err != nil {
+					util.DefaultContext.Fatal("Failed to create ", a, ": ", err.Error())
+				}
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.CompileSpec.GetPackage().GetFingerPrint(), Files: files}); err != nil {
+					util.DefaultContext.Fatal("Failed setting package files for ", a, ": ", err.Error())
+				}
+
+				util.DefaultContext.Info(art.CompileSpec.GetPackage().HumanReadableString(), " created")
 			}
 
 		},

cmd/database/get.go

@@ -19,10 +19,9 @@ import (
 	"fmt"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	"gopkg.in/yaml.v2"
 
-	. "github.com/mudler/luet/pkg/config"
-
 	"github.com/spf13/cobra"
 )
 
@@ -38,22 +37,13 @@ To return also files:
 		$ luet database get --files system/foo`,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			showFiles, _ := cmd.Flags().GetBool("files")
-			dbpath := LuetCfg.Viper.GetString("system.database_path")
-			rootfs := LuetCfg.Viper.GetString("system.rootfs")
-			engine := LuetCfg.Viper.GetString("system.database_engine")
+			util.SetSystemConfig(util.DefaultContext)
 
-			LuetCfg.System.DatabaseEngine = engine
-			LuetCfg.System.DatabasePath = dbpath
-			LuetCfg.System.Rootfs = rootfs
-
-			systemDB := LuetCfg.GetSystemDB()
+			systemDB := util.DefaultContext.Config.GetSystemDB()
 
 			for _, a := range args {
 				pack, err := helpers.ParsePackageStr(a)

cmd/database/remove.go

@@ -16,10 +16,8 @@
 package cmd_database
 
 import (
-	. "github.com/mudler/luet/pkg/logger"
-
 	helpers "github.com/mudler/luet/cmd/helpers"
-	. "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/cmd/util"
 
 	"github.com/spf13/cobra"
 )
@@ -36,34 +34,25 @@ This commands takes multiple packages as arguments and prunes their entries from
 `,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			dbpath := LuetCfg.Viper.GetString("system.database_path")
-			rootfs := LuetCfg.Viper.GetString("system.rootfs")
-			engine := LuetCfg.Viper.GetString("system.database_engine")
+			util.SetSystemConfig(util.DefaultContext)
 
-			LuetCfg.System.DatabaseEngine = engine
-			LuetCfg.System.DatabasePath = dbpath
-			LuetCfg.System.Rootfs = rootfs
-
-			systemDB := LuetCfg.GetSystemDB()
+			systemDB := util.DefaultContext.Config.GetSystemDB()
 
 			for _, a := range args {
 				pack, err := helpers.ParsePackageStr(a)
 				if err != nil {
-					Fatal("Invalid package string ", a, ": ", err.Error())
+					util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
 				}
 
 				if err := systemDB.RemovePackage(pack); err != nil {
-					Fatal("Failed removing ", a, ": ", err.Error())
+					util.DefaultContext.Fatal("Failed removing ", a, ": ", err.Error())
 				}
 
 				if err := systemDB.RemovePackageFiles(pack); err != nil {
-					Fatal("Failed removing files for ", a, ": ", err.Error())
+					util.DefaultContext.Fatal("Failed removing files for ", a, ": ", err.Error())
 				}
 			}
 

cmd/exec.go

@@ -20,8 +20,8 @@ import (
 
 	b64 "encoding/base64"
 
+	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/box"
-	. "github.com/mudler/luet/pkg/logger"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
@@ -55,12 +55,12 @@ var execCmd = &cobra.Command{
 
 			args = ss
 		}
-		Info("Executing", args, "in", rootfs)
+		util.DefaultContext.Info("Executing", args, "in", rootfs)
 
 		b := box.NewBox(entrypoint, args, mounts, envs, rootfs, stdin, stdout, stderr)
 		err := b.Exec()
 		if err != nil {
-			Fatal(errors.Wrap(err, fmt.Sprintf("entrypoint: %s rootfs: %s", entrypoint, rootfs)))
+			util.DefaultContext.Fatal(errors.Wrap(err, fmt.Sprintf("entrypoint: %s rootfs: %s", entrypoint, rootfs)))
 		}
 	},
 }
@@ -68,7 +68,7 @@ var execCmd = &cobra.Command{
 func init() {
 	path, err := os.Getwd()
 	if err != nil {
-		Fatal(err)
+		util.DefaultContext.Fatal(err)
 	}
 	execCmd.Hidden = true
 	execCmd.Flags().String("rootfs", path, "Rootfs path")

cmd/helpers/cli.go

@@ -22,11 +22,9 @@ import (
 	"regexp"
 	"strings"
 
-	. "github.com/mudler/luet/pkg/logger"
-
 	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
+	"github.com/mudler/luet/cmd/util"
 	pkg "github.com/mudler/luet/pkg/package"
-	version "github.com/mudler/luet/pkg/versioner"
 )
 
 func CreateRegexArray(rgx []string) ([]*regexp.Regexp, error) {
@@ -56,67 +54,77 @@ func packageData(p string) (string, string) {
 	}
 	return cat, name
 }
-func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
 
-	if !(strings.HasPrefix(p, "=") || strings.HasPrefix(p, ">") ||
-		strings.HasPrefix(p, "<")) {
-		ver := ">=0"
-		cat := ""
-		name := ""
+func packageHasGentooSelector(v string) bool {
+	return (strings.HasPrefix(v, "=") || strings.HasPrefix(v, ">") ||
+		strings.HasPrefix(v, "<"))
+}
 
-		if strings.Contains(p, "@") {
-			packageinfo := strings.Split(p, "@")
-			ver = packageinfo[1]
-			cat, name = packageData(packageinfo[0])
-		} else {
-			cat, name = packageData(p)
-		}
+func gentooVersion(gp *_gentoo.GentooPackage) string {
 
-		return &pkg.DefaultPackage{
-			Name:     name,
-			Category: cat,
-			Version:  ver,
-			Uri:      make([]string, 0),
-		}, nil
+	condition := gp.Condition.String()
+	if condition == "=" {
+		condition = ""
 	}
 
-	gp, err := _gentoo.ParsePackageStr(p)
-	if err != nil {
-		return nil, err
-	}
-	if gp.Version == "" {
-		gp.Version = "0"
-		gp.Condition = _gentoo.PkgCondGreaterEqual
-	}
-
-	pkgVersion := ""
+	pkgVersion := fmt.Sprintf("%s%s%s",
+		condition,
+		gp.Version,
+		gp.VersionSuffix,
+	)
 	if gp.VersionBuild != "" {
 		pkgVersion = fmt.Sprintf("%s%s%s+%s",
-			version.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+			condition,
 			gp.Version,
 			gp.VersionSuffix,
 			gp.VersionBuild,
 		)
+	}
+	return pkgVersion
+}
+
+func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
+
+	if packageHasGentooSelector(p) {
+		gp, err := _gentoo.ParsePackageStr(p)
+		if err != nil {
+			return nil, err
+		}
+		if gp.Version == "" {
+			gp.Version = "0"
+			gp.Condition = _gentoo.PkgCondGreaterEqual
+		}
+
+		return &pkg.DefaultPackage{
+			Name:     gp.Name,
+			Category: gp.Category,
+			Version:  gentooVersion(gp),
+			Uri:      make([]string, 0),
+		}, nil
+	}
+
+	ver := ">=0"
+	cat := ""
+	name := ""
+
+	if strings.Contains(p, "@") {
+		packageinfo := strings.Split(p, "@")
+		ver = packageinfo[1]
+		cat, name = packageData(packageinfo[0])
 	} else {
-		pkgVersion = fmt.Sprintf("%s%s%s",
-			version.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
-			gp.Version,
-			gp.VersionSuffix,
-		)
+		cat, name = packageData(p)
 	}
 
-	pack := &pkg.DefaultPackage{
-		Name:     gp.Name,
-		Category: gp.Category,
-		Version:  pkgVersion,
+	return &pkg.DefaultPackage{
+		Name:     name,
+		Category: cat,
+		Version:  ver,
 		Uri:      make([]string, 0),
-	}
-
-	return pack, nil
+	}, nil
 }
 
 func CheckErr(err error) {
 	if err != nil {
-		Fatal(err)
+		util.DefaultContext.Fatal(err)
 	}
 }

cmd/helpers/cli_suite_test.go

@@ -18,15 +18,11 @@ package cmd_helpers_test
 import (
 	"testing"
 
-	. "github.com/mudler/luet/cmd"
-	config "github.com/mudler/luet/pkg/config"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
-	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "CLI helpers test Suite")
 }

cmd/install.go

@@ -15,15 +15,16 @@
 package cmd
 
 import (
+	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
 	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
-	. "github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/cmd/util"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
 var installCmd = &cobra.Command{
@@ -47,17 +48,12 @@ To force install a package:
 `,
 	Aliases: []string{"i"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
-		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
-		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
-		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
-		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var toInstall pkg.Packages
@@ -65,63 +61,53 @@ To force install a package:
 		for _, a := range args {
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
-				Fatal("Invalid package string ", a, ": ", err.Error())
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
 			}
 			toInstall = append(toInstall, pack)
 		}
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
-		force := LuetCfg.Viper.GetBool("force")
-		nodeps := LuetCfg.Viper.GetBool("nodeps")
-		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
-		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
-		yes := LuetCfg.Viper.GetBool("yes")
+		force := viper.GetBool("force")
+		nodeps := viper.GetBool("nodeps")
+		onlydeps := viper.GetBool("onlydeps")
+		yes := viper.GetBool("yes")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+		finalizerEnvs, _ := cmd.Flags().GetStringArray("finalizer-env")
+		relax, _ := cmd.Flags().GetBool("relax")
 
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
+		util.SetSystemConfig(util.DefaultContext)
+		util.SetSolverConfig(util.DefaultContext)
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-
-		if concurrent {
-			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
-		} else {
-			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
-		}
-
-		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
-		repos := installer.SystemRepositories(LuetCfg)
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
 
 		// Load config protect configs
-		installer.LoadConfigProtectConfs(LuetCfg)
+		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+
+		// Load finalizer runtime environments
+		err := util.SetCliFinalizerEnvs(util.DefaultContext, finalizerEnvs)
+		if err != nil {
+			util.DefaultContext.Fatal(err.Error())
+		}
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
-			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
 			NoDeps:                      nodeps,
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
 			PreserveSystemEssentialData: true,
 			DownloadOnly:                downloadOnly,
 			Ask:                         !yes,
+			Relaxed:                     relax,
+			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			Context:                     util.DefaultContext,
 		})
-		inst.Repositories(repos)
 
-		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
-		err := inst.Install(toInstall, system)
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		err = inst.Install(toInstall, system)
 		if err != nil {
-			Fatal("Error: " + err.Error())
+			util.DefaultContext.Fatal("Error: " + err.Error())
 		}
 	},
 }
@@ -131,16 +117,20 @@ func init() {
 	installCmd.Flags().String("system-target", "", "System rootpath")
 	installCmd.Flags().String("system-engine", "", "System DB engine")
 
-	installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
 	installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	installCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	installCmd.Flags().Bool("relax", false, "Relax installation constraints")
+
 	installCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
 	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
 	installCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	installCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 	installCmd.Flags().Bool("download-only", false, "Download only")
+	installCmd.Flags().StringArray("finalizer-env", []string{},
+		"Set finalizer environment in the format key=value.")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/pack.go

@@ -20,11 +20,10 @@ import (
 	"time"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler/types/compression"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
-	. "github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -53,17 +52,17 @@ Afterwards, you can use the content generated and associate it with a tree and a
 
 		dst := viper.GetString("destination")
 		compressionType := viper.GetString("compression")
-		concurrency := LuetCfg.GetGeneral().Concurrency
+		concurrency := util.DefaultContext.Config.GetGeneral().Concurrency
 
 		if len(args) != 1 {
-			Fatal("You must specify a package name")
+			util.DefaultContext.Fatal("You must specify a package name")
 		}
 
 		packageName := args[0]
 
 		p, err := helpers.ParsePackageStr(packageName)
 		if err != nil {
-			Fatal("Invalid package string ", packageName, ": ", err.Error())
+			util.DefaultContext.Fatal("Invalid package string ", packageName, ": ", err.Error())
 		}
 
 		spec := &compilerspec.LuetCompilationSpec{Package: p}
@@ -71,18 +70,18 @@ Afterwards, you can use the content generated and associate it with a tree and a
 		a.CompressionType = compression.Implementation(compressionType)
 		err = a.Compress(sourcePath, concurrency)
 		if err != nil {
-			Fatal("failed compressing ", packageName, ": ", err.Error())
+			util.DefaultContext.Fatal("failed compressing ", packageName, ": ", err.Error())
 		}
 		a.CompileSpec = spec
 		filelist, err := a.FileList()
 		if err != nil {
-			Fatal("failed generating file list for ", packageName, ": ", err.Error())
+			util.DefaultContext.Fatal("failed generating file list for ", packageName, ": ", err.Error())
 		}
 		a.Files = filelist
 		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
-		err = a.WriteYaml(dst)
+		err = a.WriteYAML(dst)
 		if err != nil {
-			Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
+			util.DefaultContext.Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
 		}
 	},
 }
@@ -90,7 +89,7 @@ Afterwards, you can use the content generated and associate it with a tree and a
 func init() {
 	path, err := os.Getwd()
 	if err != nil {
-		Fatal(err)
+		util.DefaultContext.Fatal(err)
 	}
 	packCmd.Flags().String("source", path, "Source folder")
 	packCmd.Flags().String("destination", path, "Destination folder")

cmd/reclaim.go

@@ -15,22 +15,19 @@
 package cmd
 
 import (
+	"github.com/mudler/luet/cmd/util"
 	installer "github.com/mudler/luet/pkg/installer"
 
-	. "github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
-
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
 var reclaimCmd = &cobra.Command{
 	Use:   "reclaim",
 	Short: "Reclaim packages to Luet database from available repositories",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		util.BindSystemFlags(cmd)
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Long: `Reclaim tries to find association between packages in the online repositories and the system one.
 
@@ -39,39 +36,24 @@ var reclaimCmd = &cobra.Command{
 It scans the target file system, and if finds a match with a package available in the repositories, it marks as installed in the system database.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
+		util.SetSystemConfig(util.DefaultContext)
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		force := viper.GetBool("force")
 
-		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
-		repos := installer.Repositories{}
-		for _, repo := range LuetCfg.SystemRepositories {
-			if !repo.Enable {
-				continue
-			}
-			r := installer.NewSystemRepository(repo)
-			repos = append(repos, r)
-		}
-
-		force := LuetCfg.Viper.GetBool("force")
-
-		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
 			Force:                       force,
 			PreserveSystemEssentialData: true,
+			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			Context:                     util.DefaultContext,
 		})
-		inst.Repositories(repos)
 
-		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
 		err := inst.Reclaim(system)
 		if err != nil {
-			Fatal("Error: " + err.Error())
+			util.DefaultContext.Fatal("Error: " + err.Error())
 		}
 	},
 }

cmd/reinstall.go

@@ -0,0 +1,114 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"github.com/mudler/luet/pkg/api/core/types"
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var reinstallCmd = &cobra.Command{
+	Use:   "reinstall <pkg1> <pkg2> <pkg3>",
+	Short: "reinstall a set of packages",
+	Long: `Reinstall a group of packages in the system:
+
+	$ luet reinstall -y system/busybox shells/bash system/coreutils ...
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("for", cmd.Flags().Lookup("for"))
+
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var toUninstall pkg.Packages
+		var toAdd pkg.Packages
+
+		force := viper.GetBool("force")
+		onlydeps := viper.GetBool("onlydeps")
+		yes := viper.GetBool("yes")
+
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		util.SetSystemConfig(util.DefaultContext)
+
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toUninstall = append(toUninstall, pack)
+			toAdd = append(toAdd, pack)
+		}
+
+		util.SetSolverConfig(util.DefaultContext)
+
+		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			NoDeps:                      true,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
+			Context:                     util.DefaultContext,
+			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+		})
+
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		err := inst.Swap(toUninstall, toAdd, system)
+		if err != nil {
+			util.DefaultContext.Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+
+	reinstallCmd.Flags().String("system-dbpath", "", "System db path")
+	reinstallCmd.Flags().String("system-target", "", "System rootpath")
+	reinstallCmd.Flags().String("system-engine", "", "System DB engine")
+
+	reinstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	reinstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	reinstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	reinstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	reinstallCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	reinstallCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	reinstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	reinstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	reinstallCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(reinstallCmd)
+}

cmd/replace.go

@@ -15,15 +15,17 @@
 package cmd
 
 import (
+	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
 	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
-	. "github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/cmd/util"
+
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
 var replaceCmd = &cobra.Command{
@@ -35,47 +37,32 @@ var replaceCmd = &cobra.Command{
 	$ luet replace -y system/busybox ... --for shells/bash --for system/coreutils ...
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
-		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
-		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
-		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
-		LuetCfg.Viper.BindPFlag("for", cmd.Flags().Lookup("for"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("for", cmd.Flags().Lookup("for"))
 
-		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var toUninstall pkg.Packages
 		var toAdd pkg.Packages
 
-		f := LuetCfg.Viper.GetStringSlice("for")
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
-		force := LuetCfg.Viper.GetBool("force")
-		nodeps := LuetCfg.Viper.GetBool("nodeps")
-		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
-		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
-		yes := LuetCfg.Viper.GetBool("yes")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
+		f := viper.GetStringSlice("for")
+		force := viper.GetBool("force")
+		nodeps := viper.GetBool("nodeps")
+		onlydeps := viper.GetBool("onlydeps")
+		yes := viper.GetBool("yes")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
-
+		util.SetSystemConfig(util.DefaultContext)
+		util.SetSolverConfig(util.DefaultContext)
 		for _, a := range args {
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
-				Fatal("Invalid package string ", a, ": ", err.Error())
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
 			}
 			toUninstall = append(toUninstall, pack)
 		}
@@ -83,53 +70,35 @@ var replaceCmd = &cobra.Command{
 		for _, a := range f {
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
-				Fatal("Invalid package string ", a, ": ", err.Error())
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
 			}
 			toAdd = append(toAdd, pack)
 		}
 
-		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
-		repos := installer.Repositories{}
-		for _, repo := range LuetCfg.SystemRepositories {
-			if !repo.Enable {
-				continue
-			}
-			r := installer.NewSystemRepository(repo)
-			repos = append(repos, r)
-		}
+		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-
-		if concurrent {
-			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
-		} else {
-			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
-		}
-
-		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
 
 		// Load config protect configs
-		installer.LoadConfigProtectConfs(LuetCfg)
+		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
-			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
 			NoDeps:                      nodeps,
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
 			PreserveSystemEssentialData: true,
 			Ask:                         !yes,
 			DownloadOnly:                downloadOnly,
+			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			Context:                     util.DefaultContext,
 		})
-		inst.Repositories(repos)
 
-		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
 		err := inst.Swap(toUninstall, toAdd, system)
 		if err != nil {
-			Fatal("Error: " + err.Error())
+			util.DefaultContext.Fatal("Error: " + err.Error())
 		}
 	},
 }
@@ -140,7 +109,7 @@ func init() {
 	replaceCmd.Flags().String("system-target", "", "System rootpath")
 	replaceCmd.Flags().String("system-engine", "", "System DB engine")
 
-	replaceCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	replaceCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
 	replaceCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	replaceCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	replaceCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")

cmd/repo/list.go

@@ -22,10 +22,10 @@ import (
 	"strconv"
 	"time"
 
-	. "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/cmd/util"
 	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/pterm/pterm"
 
-	. "github.com/logrusorgru/aurora"
 	"github.com/spf13/cobra"
 )
 
@@ -43,7 +43,7 @@ func NewRepoListCommand() *cobra.Command {
 			quiet, _ := cmd.Flags().GetBool("quiet")
 			repoType, _ := cmd.Flags().GetString("type")
 
-			for _, repo := range LuetCfg.SystemRepositories {
+			for _, repo := range util.DefaultContext.Config.SystemRepositories {
 				if enable && !repo.Enable {
 					continue
 				}
@@ -58,17 +58,17 @@ func NewRepoListCommand() *cobra.Command {
 					fmt.Println(repo.Name)
 				} else {
 					if repo.Enable {
-						repoColor = Bold(BrightGreen(repo.Name)).String()
+						repoColor = pterm.LightGreen(repo.Name)
 					} else {
-						repoColor = Bold(BrightRed(repo.Name)).String()
+						repoColor = pterm.LightRed(repo.Name)
 					}
 					if repo.Description != "" {
-						repoText = Yellow(repo.Description).String()
+						repoText = pterm.LightYellow(repo.Description)
 					} else {
-						repoText = Yellow(repo.Urls[0]).String()
+						repoText = pterm.LightYellow(repo.Urls[0])
 					}
 
-					repobasedir := LuetCfg.GetSystem().GetRepoDatabaseDirPath(repo.Name)
+					repobasedir := util.DefaultContext.Config.GetSystem().GetRepoDatabaseDirPath(repo.Name)
 					if repo.Cached {
 
 						r := installer.NewSystemRepository(repo)
@@ -76,8 +76,8 @@ func NewRepoListCommand() *cobra.Command {
 							installer.REPOSITORY_SPECFILE))
 						if localRepo != nil {
 							tsec, _ := strconv.ParseInt(localRepo.GetLastUpdate(), 10, 64)
-							repoRevision = Bold(Red(localRepo.GetRevision())).String() +
-								" - " + Bold(Green(time.Unix(tsec, 0).String())).String()
+							repoRevision = pterm.LightRed(localRepo.GetRevision()) +
+								" - " + pterm.LightGreen(time.Unix(tsec, 0).String())
 						}
 					}
 

cmd/repo/update.go

@@ -17,9 +17,8 @@
 package cmd_repo
 
 import (
-	. "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/cmd/util"
 	installer "github.com/mudler/luet/pkg/installer"
-	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/spf13/cobra"
 )
@@ -45,32 +44,28 @@ $> luet repo update repo1 repo2
 
 			if len(args) > 0 {
 				for _, rname := range args {
-					repo, err := LuetCfg.GetSystemRepository(rname)
+					repo, err := util.DefaultContext.Config.GetSystemRepository(rname)
 					if err != nil && !ignore {
-						Fatal(err.Error())
+						util.DefaultContext.Fatal(err.Error())
 					} else if err != nil {
 						continue
 					}
 
 					r := installer.NewSystemRepository(*repo)
-					Spinner(32)
-					_, err = r.Sync(force)
+					_, err = r.Sync(util.DefaultContext, force)
 					if err != nil && !ignore {
-						Fatal("Error on sync repository " + rname + ": " + err.Error())
+						util.DefaultContext.Fatal("Error on sync repository " + rname + ": " + err.Error())
 					}
-					SpinnerStop()
 				}
 
 			} else {
-				for _, repo := range LuetCfg.SystemRepositories {
+				for _, repo := range util.DefaultContext.Config.SystemRepositories {
 					if repo.Cached && repo.Enable {
 						r := installer.NewSystemRepository(repo)
-						Spinner(32)
-						_, err := r.Sync(force)
+						_, err := r.Sync(util.DefaultContext, force)
 						if err != nil && !ignore {
-							Fatal("Error on sync repository " + r.GetName() + ": " + err.Error())
+							util.DefaultContext.Fatal("Error on sync repository " + r.GetName() + ": " + err.Error())
 						}
-						SpinnerStop()
 					}
 				}
 			}

cmd/root.go

@@ -18,32 +18,28 @@ package cmd
 import (
 	"fmt"
 	"os"
-	"os/user"
-	"path/filepath"
-	"runtime"
-	"strings"
 
-	"github.com/marcsauter/single"
+	"github.com/mudler/luet/cmd/util"
 	bus "github.com/mudler/luet/pkg/bus"
 
-	extensions "github.com/mudler/cobra-extensions"
-	config "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
-	. "github.com/mudler/luet/pkg/logger"
-	repo "github.com/mudler/luet/pkg/repository"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 var cfgFile string
 var Verbose bool
-var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.14.3"
+	LuetCLIVersion = "0.19.2"
 	LuetEnvPrefix  = "LUET"
 )
 
+var license = []string{
+	"Luet Copyright (C) 2019-2021 Ettore Di Giacinto",
+	"This program comes with ABSOLUTELY NO WARRANTY.",
+	"This is free software, and you are welcome to redistribute it under certain conditions.",
+}
+
 // Build time and commit information.
 //
 // ⚠️ WARNING: should only be set by "-ldflags".
@@ -52,6 +48,10 @@ var (
 	BuildCommit string
 )
 
+func version() string {
+	return fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime)
+}
+
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
 	Use:   "luet",
@@ -79,98 +79,48 @@ To build a package, from a tree definition:
 	$ luet build --tree tree/path package
 	
 `,
-	Version: fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime),
+	Version: version(),
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
-		err := LoadConfig(config.LuetCfg)
+		err := util.InitContext(util.DefaultContext)
 		if err != nil {
-			Fatal("failed to load configuration:", err.Error())
+			util.DefaultContext.Error("failed to load configuration:", err.Error())
 		}
+		util.DisplayVersionBanner(util.DefaultContext, util.IntroScreen, version, license)
+
 		// Initialize tmpdir prefix. TODO: Move this with LoadConfig
 		// directly on sub command to ensure the creation only when it's
 		// needed.
-		err = config.LuetCfg.GetSystem().InitTmpDir()
+		err = util.DefaultContext.Config.GetSystem().InitTmpDir()
 		if err != nil {
-			Fatal("failed on init tmp basedir:", err.Error())
+			util.DefaultContext.Fatal("failed on init tmp basedir:", err.Error())
 		}
 
 		viper.BindPFlag("plugin", cmd.Flags().Lookup("plugin"))
 
 		plugin := viper.GetStringSlice("plugin")
 
-		bus.Manager.Load(plugin...).Register()
+		bus.Manager.Initialize(plugin...)
 		if len(bus.Manager.Plugins) != 0 {
-			Info(":lollipop:Enabled plugins:")
+			util.DefaultContext.Info(":lollipop:Enabled plugins:")
 			for _, p := range bus.Manager.Plugins {
-				Info("\t:arrow_right:", p.Name)
+				util.DefaultContext.Info("\t:arrow_right:", p.Name)
 			}
 		}
 	},
 	PersistentPostRun: func(cmd *cobra.Command, args []string) {
 		// Cleanup all tmp directories used by luet
-		err := config.LuetCfg.GetSystem().CleanupTmpDir()
+		err := util.DefaultContext.Config.GetSystem().CleanupTmpDir()
 		if err != nil {
-			Warning("failed on cleanup tmpdir:", err.Error())
+			util.DefaultContext.Warning("failed on cleanup tmpdir:", err.Error())
 		}
 	},
 	SilenceErrors: true,
 }
 
-func LoadConfig(c *config.LuetConfig) error {
-	// If a config file is found, read it in.
-	c.Viper.ReadInConfig()
-
-	err := c.Viper.Unmarshal(&config.LuetCfg)
-	if err != nil {
-		return err
-	}
-
-	noSpinner := c.Viper.GetBool("no_spinner")
-
-	InitAurora()
-	if !noSpinner {
-		NewSpinner()
-	}
-
-	Debug("Using config file:", c.Viper.ConfigFileUsed())
-
-	if c.GetLogging().EnableLogFile && c.GetLogging().Path != "" {
-		// Init zap logger
-		err = ZapLogger()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Load repositories
-	err = repo.LoadRepositories(c)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
-
-	if os.Getenv("LUET_NOLOCK") != "true" {
-		if len(os.Args) > 1 {
-			for _, lockedCmd := range LockedCommands {
-				if os.Args[1] == lockedCmd {
-					s := single.New("luet")
-					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
-						Fatal("another instance of the app is already running, exiting")
-					} else if err != nil {
-						// Another error occurred, might be worth handling it as well
-						Fatal("failed to acquire exclusive app lock:", err.Error())
-					}
-					defer s.TryUnlock()
-					break
-				}
-			}
-		}
-	}
+	util.HandleLock(util.DefaultContext)
 
 	if err := RootCmd.Execute(); err != nil {
 		fmt.Println(err)
@@ -179,95 +129,5 @@ func Execute() {
 }
 
 func init() {
-	cobra.OnInitialize(initConfig)
-	pflags := RootCmd.PersistentFlags()
-	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
-	pflags.BoolP("debug", "d", false, "verbose output")
-	pflags.Bool("fatal", false, "Enables Warnings to exit")
-	pflags.Bool("enable-logfile", false, "Enable log to file")
-	pflags.Bool("no-spinner", false, "Disable spinner.")
-	pflags.Bool("color", config.LuetCfg.GetLogging().Color, "Enable/Disable color.")
-	pflags.Bool("emoji", config.LuetCfg.GetLogging().EnableEmoji, "Enable/Disable emoji.")
-	pflags.Bool("skip-config-protect", config.LuetCfg.ConfigProtectSkip,
-		"Disable config protect analysis.")
-	pflags.StringP("logfile", "l", config.LuetCfg.GetLogging().Path,
-		"Logfile path. Empty value disable log to file.")
-	pflags.StringSlice("plugin", []string{}, "A list of runtime plugins to load")
-
-	// os/user doesn't work in from scratch environments.
-	// Check if i can retrieve user informations.
-	_, err := user.Current()
-	if err != nil {
-		Warning("failed to retrieve user identity:", err.Error())
-	}
-	pflags.Bool("same-owner", config.LuetCfg.GetGeneral().SameOwner, "Maintain same owner on uncompress.")
-	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
-
-	config.LuetCfg.Viper.BindPFlag("logging.color", pflags.Lookup("color"))
-	config.LuetCfg.Viper.BindPFlag("logging.enable_emoji", pflags.Lookup("emoji"))
-	config.LuetCfg.Viper.BindPFlag("logging.enable_logfile", pflags.Lookup("enable-logfile"))
-	config.LuetCfg.Viper.BindPFlag("logging.path", pflags.Lookup("logfile"))
-
-	config.LuetCfg.Viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
-	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("debug"))
-	config.LuetCfg.Viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
-	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
-	config.LuetCfg.Viper.BindPFlag("plugin", pflags.Lookup("plugin"))
-
-	// Currently I maintain this only from cli.
-	config.LuetCfg.Viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))
-	config.LuetCfg.Viper.BindPFlag("config_protect_skip", pflags.Lookup("skip-config-protect"))
-
-	// Extensions must be binary with the "luet-" prefix to be able to be shown in the help.
-	// we also accept extensions in the relative path where luet is being started, "extensions/"
-	exts := extensions.Discover("luet", "extensions")
-	for _, ex := range exts {
-		cobraCmd := ex.CobraCommand()
-		RootCmd.AddCommand(cobraCmd)
-	}
-
-}
-
-// initConfig reads in config file and ENV variables if set.
-func initConfig() {
-	// Luet support these priorities on read configuration file:
-	// - command line option (if available)
-	// - $PWD/.luet.yaml
-	// - $HOME/.luet.yaml
-	// - /etc/luet/luet.yaml
-	//
-	// Note: currently a single viper instance support only one config name.
-
-	viper.SetEnvPrefix(LuetEnvPrefix)
-	viper.SetConfigType("yaml")
-
-	if cfgFile != "" { // enable ability to specify config file via flag
-		viper.SetConfigFile(cfgFile)
-	} else {
-		// Retrieve pwd directory
-		pwdDir, err := os.Getwd()
-		if err != nil {
-			Error(err)
-			os.Exit(1)
-		}
-		homeDir := helpers.GetHomeDir()
-
-		if helpers.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && helpers.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
-			viper.AddConfigPath(".")
-			if homeDir != "" {
-				viper.AddConfigPath(homeDir)
-			}
-			viper.SetConfigName(".luet")
-		} else {
-			viper.SetConfigName("luet")
-			viper.AddConfigPath("/etc/luet")
-		}
-	}
-
-	viper.AutomaticEnv() // read in environment variables that match
-
-	// Create EnvKey Replacer for handle complex structure
-	replacer := strings.NewReplacer(".", "__")
-	viper.SetEnvKeyReplacer(replacer)
-	viper.SetTypeByDefaultValue(true)
+	util.InitViper(util.DefaultContext, RootCmd)
 }

cmd/search.go

@@ -19,13 +19,13 @@ import (
 	"strings"
 
 	"github.com/ghodss/yaml"
-	"github.com/jedib0t/go-pretty/table"
-	"github.com/jedib0t/go-pretty/v6/list"
-	. "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
-	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
 type PackageResult struct {
@@ -46,28 +46,47 @@ func (r PackageResult) String() string {
 	return fmt.Sprintf("%s/%s-%s required for %s", r.Category, r.Name, r.Version, r.Target)
 }
 
-var rows table.Row = table.Row{"Package", "Category", "Name", "Version", "Repository", "Description", "License", "URI"}
+var rows []string = []string{"Package", "Category", "Name", "Version", "Repository", "License"}
 
-func packageToRow(repo string, p pkg.Package) table.Row {
-	return table.Row{p.HumanReadableString(), p.GetCategory(), p.GetName(), p.GetVersion(), repo, p.GetDescription(), p.GetLicense(), strings.Join(p.GetURI(), "\n")}
+func packageToRow(repo string, p pkg.Package) []string {
+	return []string{p.HumanReadableString(), p.GetCategory(), p.GetName(), p.GetVersion(), repo, p.GetLicense()}
 }
 
-func packageToList(l list.Writer, repo string, p pkg.Package) {
-	l.AppendItem(p.HumanReadableString())
-	l.Indent()
-	l.AppendItem(fmt.Sprintf("Category: %s", p.GetCategory()))
-	l.AppendItem(fmt.Sprintf("Name: %s", p.GetName()))
-	l.AppendItem(fmt.Sprintf("Version: %s", p.GetVersion()))
-	l.AppendItem(fmt.Sprintf("Description: %s", p.GetDescription()))
-	l.AppendItem(fmt.Sprintf("Repository: %s ", repo))
-	l.AppendItem(fmt.Sprintf("Uri: %s ", strings.Join(p.GetURI(), "\n")))
-	l.UnIndent()
+func packageToList(l *util.ListWriter, repo string, p pkg.Package) {
+	l.AppendItem(pterm.BulletListItem{
+		Level: 0, Text: p.HumanReadableString(),
+		TextStyle: pterm.NewStyle(pterm.FgCyan), Bullet: ">", BulletStyle: pterm.NewStyle(pterm.FgYellow),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Category: %s", p.GetCategory()),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Name: %s", p.GetName()),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Version: %s", p.GetVersion()),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Description: %s", p.GetDescription()),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Repository: %s ", repo),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Uri: %s ", strings.Join(p.GetURI(), " ")),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
 }
 
-func searchLocally(term string, l list.Writer, t table.Writer, label, labelMatch, revdeps, hidden bool) Results {
+func searchLocally(term string, l *util.ListWriter, t *util.TableWriter, label, labelMatch, revdeps, hidden bool) Results {
 	var results Results
 
-	system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+	system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
 
 	var err error
 	iMatches := pkg.Packages{}
@@ -80,7 +99,7 @@ func searchLocally(term string, l list.Writer, t table.Writer, label, labelMatch
 	}
 
 	if err != nil {
-		Fatal("Error: " + err.Error())
+		util.DefaultContext.Fatal("Error: " + err.Error())
 	}
 
 	for _, pack := range iMatches {
@@ -124,31 +143,23 @@ func searchLocally(term string, l list.Writer, t table.Writer, label, labelMatch
 	return results
 
 }
-func searchOnline(term string, l list.Writer, t table.Writer, label, labelMatch, revdeps, hidden bool) Results {
+func searchOnline(term string, l *util.ListWriter, t *util.TableWriter, label, labelMatch, revdeps, hidden bool) Results {
 	var results Results
 
-	repos := installer.Repositories{}
-	for _, repo := range LuetCfg.SystemRepositories {
-		if !repo.Enable {
-			continue
-		}
-		r := installer.NewSystemRepository(repo)
-		repos = append(repos, r)
-	}
-
 	inst := installer.NewLuetInstaller(
 		installer.LuetInstallerOptions{
-			Concurrency:   LuetCfg.GetGeneral().Concurrency,
-			SolverOptions: *LuetCfg.GetSolverOptions(),
+			Concurrency:         util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:       *util.DefaultContext.Config.GetSolverOptions(),
+			PackageRepositories: util.DefaultContext.Config.SystemRepositories,
+			Context:             util.DefaultContext,
 		},
 	)
-	inst.Repositories(repos)
-	synced, err := inst.SyncRepositories(false)
+	synced, err := inst.SyncRepositories()
 	if err != nil {
-		Fatal("Error: " + err.Error())
+		util.DefaultContext.Fatal("Error: " + err.Error())
 	}
 
-	Info("--- Search results (" + term + "): ---")
+	util.DefaultContext.Info("--- Search results (" + term + "): ---")
 
 	matches := []installer.PackageMatch{}
 	if label {
@@ -199,15 +210,15 @@ func searchOnline(term string, l list.Writer, t table.Writer, label, labelMatch,
 	}
 	return results
 }
-func searchLocalFiles(term string, l list.Writer, t table.Writer) Results {
+func searchLocalFiles(term string, l *util.ListWriter, t *util.TableWriter) Results {
 	var results Results
-	Info("--- Search results (" + term + "): ---")
+	util.DefaultContext.Info("--- Search results (" + term + "): ---")
 
-	matches, _ := LuetCfg.GetSystemDB().FindPackageByFile(term)
+	matches, _ := util.DefaultContext.Config.GetSystemDB().FindPackageByFile(term)
 	for _, pack := range matches {
 		t.AppendRow(packageToRow("system", pack))
 		packageToList(l, "system", pack)
-		f, _ := LuetCfg.GetSystemDB().GetPackageFiles(pack)
+		f, _ := util.DefaultContext.Config.GetSystemDB().GetPackageFiles(pack)
 		results.Packages = append(results.Packages,
 			PackageResult{
 				Name:       pack.GetName(),
@@ -222,31 +233,23 @@ func searchLocalFiles(term string, l list.Writer, t table.Writer) Results {
 	return results
 }
 
-func searchFiles(term string, l list.Writer, t table.Writer) Results {
+func searchFiles(term string, l *util.ListWriter, t *util.TableWriter) Results {
 	var results Results
 
-	repos := installer.Repositories{}
-	for _, repo := range LuetCfg.SystemRepositories {
-		if !repo.Enable {
-			continue
-		}
-		r := installer.NewSystemRepository(repo)
-		repos = append(repos, r)
-	}
-
 	inst := installer.NewLuetInstaller(
 		installer.LuetInstallerOptions{
-			Concurrency:   LuetCfg.GetGeneral().Concurrency,
-			SolverOptions: *LuetCfg.GetSolverOptions(),
+			Concurrency:         util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:       *util.DefaultContext.Config.GetSolverOptions(),
+			PackageRepositories: util.DefaultContext.Config.SystemRepositories,
+			Context:             util.DefaultContext,
 		},
 	)
-	inst.Repositories(repos)
-	synced, err := inst.SyncRepositories(false)
+	synced, err := inst.SyncRepositories()
 	if err != nil {
-		Fatal("Error: " + err.Error())
+		util.DefaultContext.Fatal("Error: " + err.Error())
 	}
 
-	Info("--- Search results (" + term + "): ---")
+	util.DefaultContext.Info("--- Search results (" + term + "): ---")
 
 	matches := []installer.PackageMatch{}
 
@@ -306,55 +309,38 @@ Search can also return results in the terminal in different ways: as terminal ou
 `,
 	Aliases: []string{"s"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var results Results
 		if len(args) > 1 {
-			Fatal("Wrong number of arguments (expected 1)")
+			util.DefaultContext.Fatal("Wrong number of arguments (expected 1)")
 		} else if len(args) == 0 {
 			args = []string{"."}
 		}
 		hidden, _ := cmd.Flags().GetBool("hidden")
 
-		installed := LuetCfg.Viper.GetBool("installed")
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		installed := viper.GetBool("installed")
 		searchWithLabel, _ := cmd.Flags().GetBool("by-label")
 		searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex")
 		revdeps, _ := cmd.Flags().GetBool("revdeps")
 		tableMode, _ := cmd.Flags().GetBool("table")
 		files, _ := cmd.Flags().GetBool("files")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig(util.DefaultContext)
+		util.SetSolverConfig(util.DefaultContext)
+
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
-			LuetCfg.GetLogging().SetLogLevel("error")
+			util.DefaultContext.Config.GetLogging().SetLogLevel("error")
 		}
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-
-		l := list.NewWriter()
-		t := table.NewWriter()
-		t.AppendHeader(rows)
-		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		l := &util.ListWriter{}
+		t := &util.TableWriter{}
+		t.AppendRow(rows)
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
 
 		switch {
 		case files && installed:
@@ -367,14 +353,10 @@ Search can also return results in the terminal in different ways: as terminal ou
 			results = searchLocally(args[0], l, t, searchWithLabel, searchWithLabelMatch, revdeps, hidden)
 		}
 
-		t.AppendFooter(rows)
-		t.SetStyle(table.StyleColoredBright)
-
-		l.SetStyle(list.StyleConnectedRounded)
 		if tableMode {
-			Info(t.Render())
+			t.Render()
 		} else {
-			Info(l.Render())
+			l.Render()
 		}
 
 		y, err := yaml.Marshal(results)
@@ -403,7 +385,7 @@ func init() {
 	searchCmd.Flags().String("system-engine", "", "System DB engine")
 
 	searchCmd.Flags().Bool("installed", false, "Search between system packages")
-	searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
 	searchCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 	searchCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	searchCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")

cmd/serve-repo.go

@@ -18,7 +18,7 @@ import (
 	"net/http"
 	"os"
 
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/cmd/util"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -41,15 +41,15 @@ var serverepoCmd = &cobra.Command{
 
 		http.Handle("/", http.FileServer(http.Dir(dir)))
 
-		Info("Serving ", dir, " on HTTP port: ", port)
-		Fatal(http.ListenAndServe(address+":"+port, nil))
+		util.DefaultContext.Info("Serving ", dir, " on HTTP port: ", port)
+		util.DefaultContext.Fatal(http.ListenAndServe(address+":"+port, nil))
 	},
 }
 
 func init() {
 	path, err := os.Getwd()
 	if err != nil {
-		Fatal(err)
+		util.DefaultContext.Fatal(err)
 	}
 	serverepoCmd.Flags().String("dir", path, "Packages folder (output from build)")
 	serverepoCmd.Flags().String("port", "9090", "Listening port")

cmd/tree/bump.go

@@ -19,7 +19,7 @@ package cmd_tree
 import (
 	"fmt"
 
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/cmd/util"
 	spectooling "github.com/mudler/luet/pkg/spectooling"
 	tree "github.com/mudler/luet/pkg/tree"
 	version "github.com/mudler/luet/pkg/versioner"
@@ -36,7 +36,7 @@ func NewTreeBumpCommand() *cobra.Command {
 		PreRun: func(cmd *cobra.Command, args []string) {
 			df, _ := cmd.Flags().GetString("definition-file")
 			if df == "" {
-				Fatal("Mandatory definition.yaml path missing.")
+				util.DefaultContext.Fatal("Mandatory definition.yaml path missing.")
 			}
 		},
 		Run: func(cmd *cobra.Command, args []string) {
@@ -45,34 +45,34 @@ func NewTreeBumpCommand() *cobra.Command {
 			pkgVersion, _ := cmd.Flags().GetString("pkg-version")
 			pack, err := tree.ReadDefinitionFile(spec)
 			if err != nil {
-				Fatal(err.Error())
+				util.DefaultContext.Fatal(err.Error())
 			}
 
 			if pkgVersion != "" {
 				validator := &version.WrappedVersioner{}
 				err := validator.Validate(pkgVersion)
 				if err != nil {
-					Fatal("Invalid version string: " + err.Error())
+					util.DefaultContext.Fatal("Invalid version string: " + err.Error())
 				}
 				pack.SetVersion(pkgVersion)
 			} else {
 				// Retrieve version build section with Gentoo parser
 				err = pack.BumpBuildVersion()
 				if err != nil {
-					Fatal("Error on increment build version: " + err.Error())
+					util.DefaultContext.Fatal("Error on increment build version: " + err.Error())
 				}
 			}
 			if toStdout {
 				data, err := spectooling.NewDefaultPackageSanitized(&pack).Yaml()
 				if err != nil {
-					Fatal("Error on yaml conversion: " + err.Error())
+					util.DefaultContext.Fatal("Error on yaml conversion: " + err.Error())
 				}
 				fmt.Println(string(data))
 			} else {
 
 				err = tree.WriteDefinitionFile(&pack, spec)
 				if err != nil {
-					Fatal("Error on write definition file: " + err.Error())
+					util.DefaultContext.Fatal("Error on write definition file: " + err.Error())
 				}
 
 				fmt.Printf("Bumped package %s/%s-%s.\n", pack.Category, pack.Name, pack.Version)

cmd/tree/images.go

@@ -20,14 +20,13 @@ import (
 	"fmt"
 	"os"
 
-	//. "github.com/mudler/luet/pkg/config"
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/compiler/types/options"
-	. "github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
+
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
@@ -44,12 +43,13 @@ func NewTreeImageCommand() *cobra.Command {
 		PreRun: func(cmd *cobra.Command, args []string) {
 			t, _ := cmd.Flags().GetStringArray("tree")
 			if len(t) == 0 {
-				Fatal("Mandatory tree param missing.")
+				util.DefaultContext.Fatal("Mandatory tree param missing.")
 			}
 
 			if len(args) != 1 {
-				Fatal("Expects one package as parameter")
+				util.DefaultContext.Fatal("Expects one package as parameter")
 			}
+			util.BindValuesFlags(cmd)
 			viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
 
 		},
@@ -59,10 +59,11 @@ func NewTreeImageCommand() *cobra.Command {
 			treePath, _ := cmd.Flags().GetStringArray("tree")
 			imageRepository := viper.GetString("image-repository")
 			pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
+			values := util.ValuesFlags()
 
 			out, _ := cmd.Flags().GetString("output")
 			if out != "terminal" {
-				LuetCfg.GetLogging().SetLogLevel("error")
+				util.DefaultContext.Config.GetLogging().SetLogLevel("error")
 			}
 
 			reciper := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -70,18 +71,21 @@ func NewTreeImageCommand() *cobra.Command {
 			for _, t := range treePath {
 				err := reciper.Load(t)
 				if err != nil {
-					Fatal("Error on load tree ", err)
+					util.DefaultContext.Fatal("Error on load tree ", err)
 				}
 			}
-			compilerBackend := backend.NewSimpleDockerBackend()
+			compilerBackend := backend.NewSimpleDockerBackend(util.DefaultContext)
 
-			opts := *LuetCfg.GetSolverOptions()
+			opts := *util.DefaultContext.Config.GetSolverOptions()
 			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
 			luetCompiler := compiler.NewLuetCompiler(
 				compilerBackend,
 				reciper.GetDatabase(),
+				options.WithBuildValues(values),
+				options.WithContext(util.DefaultContext),
 				options.WithPushRepository(imageRepository),
 				options.WithPullRepositories(pullRepo),
+				options.WithTemplateFolder(util.TemplateFolders(util.DefaultContext, false, treePath)),
 				options.WithSolverOptions(opts),
 			)
 
@@ -89,16 +93,21 @@ func NewTreeImageCommand() *cobra.Command {
 
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
-				Fatal("Invalid package string ", a, ": ", err.Error())
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
 			}
 
 			spec, err := luetCompiler.FromPackage(pack)
 			if err != nil {
-				Fatal("Error: " + err.Error())
+				util.DefaultContext.Fatal("Error: " + err.Error())
 			}
-			asserts, err := luetCompiler.ComputeDepTree(spec)
 
-			for _, assertion := range asserts { //highly dependent on the order
+			ht := compiler.NewHashTree(reciper.GetDatabase())
+			hashtree, err := ht.Query(luetCompiler, spec)
+			if err != nil {
+				util.DefaultContext.Fatal("Error: " + err.Error())
+			}
+
+			for _, assertion := range hashtree.Solution { //highly dependent on the order
 
 				//buildImageHash := imageRepository + ":" + assertion.Hash.BuildHash
 				currentPackageImageHash := imageRepository + ":" + assertion.Hash.PackageHash
@@ -135,7 +144,7 @@ func NewTreeImageCommand() *cobra.Command {
 	}
 	path, err := os.Getwd()
 	if err != nil {
-		Fatal(err)
+		util.DefaultContext.Fatal(err)
 	}
 	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")

cmd/tree/pkglist.go

@@ -21,11 +21,9 @@ import (
 	"os"
 	"sort"
 
-	//. "github.com/mudler/luet/pkg/config"
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
-	. "github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/cmd/util"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
@@ -75,13 +73,13 @@ func NewTreePkglistCommand() *cobra.Command {
 		PreRun: func(cmd *cobra.Command, args []string) {
 			t, _ := cmd.Flags().GetStringArray("tree")
 			if len(t) == 0 {
-				Fatal("Mandatory tree param missing.")
+				util.DefaultContext.Fatal("Mandatory tree param missing.")
 			}
 
 			revdeps, _ := cmd.Flags().GetBool("revdeps")
 			deps, _ := cmd.Flags().GetBool("deps")
 			if revdeps && deps {
-				Fatal("Both revdeps and deps option used. Choice only one.")
+				util.DefaultContext.Fatal("Both revdeps and deps option used. Choice only one.")
 			}
 
 		},
@@ -98,7 +96,7 @@ func NewTreePkglistCommand() *cobra.Command {
 
 			out, _ := cmd.Flags().GetString("output")
 			if out != "terminal" {
-				LuetCfg.GetLogging().SetLogLevel("error")
+				util.DefaultContext.Config.GetLogging().SetLogLevel("error")
 			}
 
 			var reciper tree.Builder
@@ -111,7 +109,7 @@ func NewTreePkglistCommand() *cobra.Command {
 			for _, t := range treePath {
 				err := reciper.Load(t)
 				if err != nil {
-					Fatal("Error on load tree ", err)
+					util.DefaultContext.Fatal("Error on load tree ", err)
 				}
 			}
 
@@ -126,11 +124,11 @@ func NewTreePkglistCommand() *cobra.Command {
 
 			regExcludes, err := helpers.CreateRegexArray(excludes)
 			if err != nil {
-				Fatal(err.Error())
+				util.DefaultContext.Fatal(err.Error())
 			}
 			regMatches, err := helpers.CreateRegexArray(matches)
 			if err != nil {
-				Fatal(err.Error())
+				util.DefaultContext.Fatal(err.Error())
 			}
 
 			plist := make([]string, 0)
@@ -167,78 +165,79 @@ func NewTreePkglistCommand() *cobra.Command {
 					}
 				}
 
-				if addPkg {
-					if revdeps {
-						packs, _ := reciper.GetDatabase().GetRevdeps(p)
-						for _, revdep := range packs {
+				if !addPkg {
+					continue
+				}
+
+				if revdeps {
+					packs, _ := reciper.GetDatabase().GetRevdeps(p)
+					for i := range packs {
+						revdep := packs[i]
+						if full {
+							pkgstr = pkgDetail(revdep)
+						} else if verbose {
+							pkgstr = revdep.HumanReadableString()
+						} else {
+							pkgstr = fmt.Sprintf("%s/%s", revdep.GetCategory(), revdep.GetName())
+						}
+						plist = append(plist, pkgstr)
+						results.Packages = append(results.Packages, TreePackageResult{
+							Name:     revdep.GetName(),
+							Version:  revdep.GetVersion(),
+							Category: revdep.GetCategory(),
+							Path:     revdep.GetPath(),
+						})
+					}
+				} else if deps {
+
+					solution, err := depSolver.Install(pkg.Packages{p})
+					if err != nil {
+						util.DefaultContext.Fatal(err.Error())
+					}
+					ass := solution.SearchByName(p.GetPackageName())
+					solution, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+					if err != nil {
+						util.DefaultContext.Fatal(err.Error())
+					}
+
+					for _, pa := range solution {
+
+						if pa.Value {
+							// Exclude itself
+							if pa.Package.GetName() == p.GetName() && pa.Package.GetCategory() == p.GetCategory() {
+								continue
+							}
+
 							if full {
-								pkgstr = pkgDetail(revdep)
+								pkgstr = pkgDetail(pa.Package)
 							} else if verbose {
-								pkgstr = revdep.HumanReadableString()
+								pkgstr = pa.Package.HumanReadableString()
 							} else {
-								pkgstr = fmt.Sprintf("%s/%s", revdep.GetCategory(), revdep.GetName())
+								pkgstr = fmt.Sprintf("%s/%s", pa.Package.GetCategory(), pa.Package.GetName())
 							}
 							plist = append(plist, pkgstr)
 							results.Packages = append(results.Packages, TreePackageResult{
-								Name:     revdep.GetName(),
-								Version:  revdep.GetVersion(),
-								Category: revdep.GetCategory(),
-								Path:     revdep.GetPath(),
+								Name:     pa.Package.GetName(),
+								Version:  pa.Package.GetVersion(),
+								Category: pa.Package.GetCategory(),
+								Path:     pa.Package.GetPath(),
 							})
 						}
-					} else if deps {
-
-						Spinner(32)
-						solution, err := depSolver.Install(pkg.Packages{p})
-						if err != nil {
-							Fatal(err.Error())
-						}
-						ass := solution.SearchByName(p.GetPackageName())
-						solution, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
-						if err != nil {
-							Fatal(err.Error())
-						}
-						SpinnerStop()
-
-						for _, pa := range solution {
-
-							if pa.Value {
-								// Exclude itself
-								if pa.Package.GetName() == p.GetName() && pa.Package.GetCategory() == p.GetCategory() {
-									continue
-								}
-
-								if full {
-									pkgstr = pkgDetail(pa.Package)
-								} else if verbose {
-									pkgstr = pa.Package.HumanReadableString()
-								} else {
-									pkgstr = fmt.Sprintf("%s/%s", pa.Package.GetCategory(), pa.Package.GetName())
-								}
-								plist = append(plist, pkgstr)
-								results.Packages = append(results.Packages, TreePackageResult{
-									Name:     pa.Package.GetName(),
-									Version:  pa.Package.GetVersion(),
-									Category: pa.Package.GetCategory(),
-									Path:     pa.Package.GetPath(),
-								})
-							}
-
-						}
-
-					} else {
-
-						plist = append(plist, pkgstr)
-						results.Packages = append(results.Packages, TreePackageResult{
-							Name:     p.GetName(),
-							Version:  p.GetVersion(),
-							Category: p.GetCategory(),
-							Path:     p.GetPath(),
-						})
 
 					}
 
+				} else {
+
+					plist = append(plist, pkgstr)
+					results.Packages = append(results.Packages, TreePackageResult{
+						Name:     p.GetName(),
+						Version:  p.GetVersion(),
+						Category: p.GetCategory(),
+						Path:     p.GetPath(),
+					})
+
 				}
+
 			}
 
 			y, err := yaml.Marshal(results)
@@ -269,7 +268,7 @@ func NewTreePkglistCommand() *cobra.Command {
 	}
 	path, err := os.Getwd()
 	if err != nil {
-		Fatal(err)
+		util.DefaultContext.Fatal(err)
 	}
 	ans.Flags().BoolP("buildtime", "b", false, "Build time match")
 	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")

cmd/tree/validate.go

@@ -26,8 +26,8 @@ import (
 	"sync"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
-	. "github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/cmd/util"
+
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
@@ -104,7 +104,7 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 		} else {
 			errstr = "No packages"
 		}
-		Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+		util.DefaultContext.Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
 			checkType,
 			p.GetCategory(), p.GetName(), p.GetVersion(),
 			errstr,
@@ -134,7 +134,7 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 			p.GetCategory(), p.GetName(), p.GetVersion(),
 			err.Error(),
 		)
-		Error(errstr)
+		util.DefaultContext.Error(errstr)
 
 		return errors.New(errstr)
 	}
@@ -173,7 +173,7 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 		}
 	}
 
-	Info(fmt.Sprintf("[%9s] Checking package ", checkType)+
+	util.DefaultContext.Info(fmt.Sprintf("[%9s] Checking package ", checkType)+
 		fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
 		"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
 
@@ -201,7 +201,7 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 			} else {
 				errstr = "No packages"
 			}
-			Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+			util.DefaultContext.Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
 				checkType,
 				p.GetCategory(), p.GetName(), p.GetVersion(),
 				r.GetCategory(), r.GetName(), r.GetVersion(),
@@ -221,12 +221,12 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 
 		} else {
 
-			Debug(fmt.Sprintf("[%9s] Find packages for dep", checkType),
+			util.DefaultContext.Debug(fmt.Sprintf("[%9s] Find packages for dep", checkType),
 				fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
 
 			if opts.WithSolver {
 
-				Info(fmt.Sprintf("[%9s]  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
+				util.DefaultContext.Info(fmt.Sprintf("[%9s]  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
 					checkType,
 					idx+1, len(all),
 					p.GetCategory(), p.GetName(), p.GetVersion(),
@@ -236,15 +236,15 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 				// Check if the solver is already been done for the deep
 				_, err := cacheDeps.Get(r.HashFingerprint(""))
 				if err == nil {
-					Debug(fmt.Sprintf("[%9s]  :direct_hit: Cache Hit for dep", checkType),
+					util.DefaultContext.Debug(fmt.Sprintf("[%9s]  :direct_hit: Cache Hit for dep", checkType),
 						fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
 					continue
 				}
 
-				Spinner(32)
+				util.DefaultContext.Spinner()
 				solution, err := depSolver.Install(pkg.Packages{r})
 				ass := solution.SearchByName(r.GetPackageName())
-				SpinnerStop()
+				util.DefaultContext.SpinnerStop()
 				if err == nil {
 					if ass == nil {
 
@@ -255,7 +255,7 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 								r.GetCategory(), r.GetName(), r.GetVersion(),
 							))
 
-						if LuetCfg.GetGeneral().Debug {
+						if util.DefaultContext.Config.GetGeneral().Debug {
 							for idx, pa := range solution {
 								fmt.Println(fmt.Sprintf("[%9s] %s/%s-%s: solution %d: %s",
 									checkType,
@@ -264,7 +264,7 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 							}
 						}
 
-						Error(ans.Error())
+						util.DefaultContext.Error(ans.Error())
 						opts.IncrBrokenDeps()
 						validpkg = false
 					} else {
@@ -274,7 +274,7 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 
 				if err != nil {
 
-					Error(fmt.Sprintf("[%9s] %s/%s-%s: solver broken for dep %s/%s-%s - %s",
+					util.DefaultContext.Error(fmt.Sprintf("[%9s] %s/%s-%s: solver broken for dep %s/%s-%s - %s",
 						checkType,
 						p.GetCategory(), p.GetName(), p.GetVersion(),
 						r.GetCategory(), r.GetName(), r.GetVersion(),
@@ -376,28 +376,28 @@ func initOpts(opts *ValidateOpts, onlyRuntime, onlyBuildtime, withSolver bool, t
 	opts.BuildtimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
 
 	for _, treePath := range treePaths {
-		Info(fmt.Sprintf("Loading :deciduous_tree: %s...", treePath))
+		util.DefaultContext.Info(fmt.Sprintf("Loading :deciduous_tree: %s...", treePath))
 		if opts.BuildtimeReciper != nil {
 			err = opts.BuildtimeReciper.Load(treePath)
 			if err != nil {
-				Fatal("Error on load tree ", err)
+				util.DefaultContext.Fatal("Error on load tree ", err)
 			}
 		}
 		if opts.RuntimeReciper != nil {
 			err = opts.RuntimeReciper.Load(treePath)
 			if err != nil {
-				Fatal("Error on load tree ", err)
+				util.DefaultContext.Fatal("Error on load tree ", err)
 			}
 		}
 	}
 
 	opts.RegExcludes, err = helpers.CreateRegexArray(opts.Excludes)
 	if err != nil {
-		Fatal(err.Error())
+		util.DefaultContext.Fatal(err.Error())
 	}
 	opts.RegMatches, err = helpers.CreateRegexArray(opts.Matches)
 	if err != nil {
-		Fatal(err.Error())
+		util.DefaultContext.Fatal(err.Error())
 	}
 
 }
@@ -417,16 +417,16 @@ func NewTreeValidateCommand() *cobra.Command {
 			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
 
 			if len(treePaths) < 1 {
-				Fatal("Mandatory tree param missing.")
+				util.DefaultContext.Fatal("Mandatory tree param missing.")
 			}
 			if onlyRuntime && onlyBuildtime {
-				Fatal("Both --only-runtime and --only-buildtime options are not possibile.")
+				util.DefaultContext.Fatal("Both --only-runtime and --only-buildtime options are not possibile.")
 			}
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			var reciper tree.Builder
 
-			concurrency := LuetCfg.GetGeneral().Concurrency
+			concurrency := util.DefaultContext.Config.GetGeneral().Concurrency
 
 			withSolver, _ := cmd.Flags().GetBool("with-solver")
 			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
@@ -472,18 +472,18 @@ func NewTreeValidateCommand() *cobra.Command {
 
 			// fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
 			if len(stringerrs) != 0 {
-				Error(fmt.Sprintf("Found %d broken packages and %d broken deps.",
+				util.DefaultContext.Error(fmt.Sprintf("Found %d broken packages and %d broken deps.",
 					opts.BrokenPkgs, opts.BrokenDeps))
-				Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
+				util.DefaultContext.Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
 			} else {
-				Info("All good! :white_check_mark:")
+				util.DefaultContext.Info("All good! :white_check_mark:")
 				os.Exit(0)
 			}
 		},
 	}
 	path, err := os.Getwd()
 	if err != nil {
-		Fatal(err)
+		util.DefaultContext.Fatal(err)
 	}
 	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
 	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")

cmd/uninstall.go

@@ -16,13 +16,14 @@ package cmd
 
 import (
 	helpers "github.com/mudler/luet/cmd/helpers"
-	. "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
-	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
 var uninstallCmd = &cobra.Command{
@@ -31,16 +32,11 @@ var uninstallCmd = &cobra.Command{
 	Long:    `Uninstall packages`,
 	Aliases: []string{"rm", "un"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
-		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
-		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
-		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		toRemove := []pkg.Package{}
@@ -48,47 +44,34 @@ var uninstallCmd = &cobra.Command{
 
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
-				Fatal("Invalid package string ", a, ": ", err.Error())
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
 			}
 			toRemove = append(toRemove, pack)
 		}
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
-		force := LuetCfg.Viper.GetBool("force")
+		force := viper.GetBool("force")
 		nodeps, _ := cmd.Flags().GetBool("nodeps")
 		full, _ := cmd.Flags().GetBool("full")
 		checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
 		fullClean, _ := cmd.Flags().GetBool("full-clean")
-		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
-		yes := LuetCfg.Viper.GetBool("yes")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
+		yes := viper.GetBool("yes")
+		keepProtected, _ := cmd.Flags().GetBool("keep-protected-files")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig(util.DefaultContext)
+		util.SetSolverConfig(util.DefaultContext)
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-		if concurrent {
-			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
-		} else {
-			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
-		}
-		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+		util.DefaultContext.Config.ConfigProtectSkip = !keepProtected
+
+		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
 
 		// Load config protect configs
-		installer.LoadConfigProtectConfs(LuetCfg)
+		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
-			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
 			NoDeps:                      nodeps,
 			Force:                       force,
 			FullUninstall:               full,
@@ -96,12 +79,13 @@ var uninstallCmd = &cobra.Command{
 			CheckConflicts:              checkconflicts,
 			Ask:                         !yes,
 			PreserveSystemEssentialData: true,
+			Context:                     util.DefaultContext,
 		})
 
-		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
 
 		if err := inst.Uninstall(system, toRemove...); err != nil {
-			Fatal("Error: " + err.Error())
+			util.DefaultContext.Fatal("Error: " + err.Error())
 		}
 	},
 }
@@ -112,7 +96,7 @@ func init() {
 	uninstallCmd.Flags().String("system-target", "", "System rootpath")
 	uninstallCmd.Flags().String("system-engine", "", "System DB engine")
 
-	uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
 	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	uninstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
@@ -123,6 +107,7 @@ func init() {
 	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
 	uninstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	uninstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	uninstallCmd.Flags().BoolP("keep-protected-files", "k", false, "Keep package protected files around")
 
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -15,12 +15,13 @@
 package cmd
 
 import (
-	. "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
-	. "github.com/mudler/luet/pkg/logger"
 	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
 var upgradeCmd = &cobra.Command{
@@ -28,67 +29,36 @@ var upgradeCmd = &cobra.Command{
 	Short:   "Upgrades the system",
 	Aliases: []string{"u"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", installCmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", installCmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
-		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
-		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
 	Long: `Upgrades packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
 
-		repos := installer.Repositories{}
-		for _, repo := range LuetCfg.SystemRepositories {
-			if !repo.Enable {
-				continue
-			}
-
-			r := installer.NewSystemRepository(repo)
-			repos = append(repos, r)
-		}
-
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
-		force := LuetCfg.Viper.GetBool("force")
+		force := viper.GetBool("force")
 		nodeps, _ := cmd.Flags().GetBool("nodeps")
 		full, _ := cmd.Flags().GetBool("full")
 		universe, _ := cmd.Flags().GetBool("universe")
 		clean, _ := cmd.Flags().GetBool("clean")
 		sync, _ := cmd.Flags().GetBool("sync")
-		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
-		yes := LuetCfg.Viper.GetBool("yes")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
+		yes := viper.GetBool("yes")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-		if concurrent {
-			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
-		} else {
-			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
-		}
+		util.SetSystemConfig(util.DefaultContext)
+		opts := util.SetSolverConfig(util.DefaultContext)
 
-		Debug("Solver", LuetCfg.GetSolverOptions().String())
+		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+		util.DefaultContext.Debug("Solver", opts.CompactString())
 
 		// Load config protect configs
-		installer.LoadConfigProtectConfs(LuetCfg)
+		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
-			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
 			Force:                       force,
 			FullUninstall:               full,
 			NoDeps:                      nodeps,
@@ -98,12 +68,13 @@ var upgradeCmd = &cobra.Command{
 			PreserveSystemEssentialData: true,
 			Ask:                         !yes,
 			DownloadOnly:                downloadOnly,
+			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			Context:                     util.DefaultContext,
 		})
-		inst.Repositories(repos)
 
-		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
 		if err := inst.Upgrade(system); err != nil {
-			Fatal("Error: " + err.Error())
+			util.DefaultContext.Fatal("Error: " + err.Error())
 		}
 	},
 }
@@ -113,7 +84,7 @@ func init() {
 	upgradeCmd.Flags().String("system-target", "", "System rootpath")
 	upgradeCmd.Flags().String("system-engine", "", "System DB engine")
 
-	upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
 	upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")

cmd/util.go

@@ -1,5 +1,4 @@
-// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
-//                  Daniele Rondina <geaaru@sabayonlinux.org>
+// Copyright © 2020-2021 Ettore Di Giacinto <mudler@gentoo.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -17,11 +16,87 @@
 package cmd
 
 import (
-	. "github.com/mudler/luet/cmd/util"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/go-units"
+
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/helpers/docker"
 
 	"github.com/spf13/cobra"
 )
 
+func NewUnpackCommand() *cobra.Command {
+
+	c := &cobra.Command{
+		Use:   "unpack image path",
+		Short: "Unpack a docker image natively",
+		Long: `unpack doesn't need the docker daemon to run, and unpacks a docker image in the specified directory:
+		
+	luet util unpack golang:alpine /alpine
+`,
+		PreRun: func(cmd *cobra.Command, args []string) {
+
+			if len(args) != 2 {
+				util.DefaultContext.Fatal("Expects an image and a path")
+			}
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			image := args[0]
+			destination, err := filepath.Abs(args[1])
+			if err != nil {
+				util.DefaultContext.Error("Invalid path %s", destination)
+				os.Exit(1)
+			}
+
+			verify, _ := cmd.Flags().GetBool("verify")
+			user, _ := cmd.Flags().GetString("auth-username")
+			pass, _ := cmd.Flags().GetString("auth-password")
+			authType, _ := cmd.Flags().GetString("auth-type")
+			server, _ := cmd.Flags().GetString("auth-server-address")
+			identity, _ := cmd.Flags().GetString("auth-identity-token")
+			registryToken, _ := cmd.Flags().GetString("auth-registry-token")
+
+			temp, err := util.DefaultContext.Config.GetSystem().TempDir("contentstore")
+			if err != nil {
+				util.DefaultContext.Fatal("Cannot create a tempdir", err.Error())
+			}
+
+			util.DefaultContext.Info("Downloading", image, "to", destination)
+			auth := &types.AuthConfig{
+				Username:      user,
+				Password:      pass,
+				ServerAddress: server,
+				Auth:          authType,
+				IdentityToken: identity,
+				RegistryToken: registryToken,
+			}
+
+			info, err := docker.DownloadAndExtractDockerImage(temp, image, destination, auth, verify)
+			if err != nil {
+				util.DefaultContext.Error(err.Error())
+				os.Exit(1)
+			}
+			util.DefaultContext.Info(fmt.Sprintf("Pulled: %s %s", info.Target.Digest, info.Name))
+			util.DefaultContext.Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size))))
+		},
+	}
+
+	c.Flags().String("auth-username", "", "Username to authenticate to registry/notary")
+	c.Flags().String("auth-password", "", "Password to authenticate to registry")
+	c.Flags().String("auth-type", "", "Auth type")
+	c.Flags().String("auth-server-address", "", "Authentication server address")
+	c.Flags().String("auth-identity-token", "", "Authentication identity token")
+	c.Flags().String("auth-registry-token", "", "Authentication registry token")
+	c.Flags().Bool("verify", false, "Verify signed images to notary before to pull")
+	return c
+}
+
 var utilGroup = &cobra.Command{
 	Use:   "util [command] [OPTIONS]",
 	Short: "General luet internal utilities exposed",

cmd/util/cli.go

@@ -0,0 +1,166 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/marcsauter/single"
+	"github.com/pterm/pterm"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/installer"
+)
+
+var DefaultContext = types.NewContext()
+
+var lockedCommands = []string{"install", "uninstall", "upgrade"}
+var bannerCommands = []string{"install", "build", "uninstall", "upgrade"}
+
+func BindSystemFlags(cmd *cobra.Command) {
+	viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+	viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+	viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+}
+
+func BindSolverFlags(cmd *cobra.Command) {
+	viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+	viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+	viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+	viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+}
+
+func BindValuesFlags(cmd *cobra.Command) {
+	viper.BindPFlag("values", cmd.Flags().Lookup("values"))
+}
+
+func ValuesFlags() []string {
+	return viper.GetStringSlice("values")
+}
+
+func SetSystemConfig(ctx *types.Context) {
+	dbpath := viper.GetString("system.database_path")
+	rootfs := viper.GetString("system.rootfs")
+	engine := viper.GetString("system.database_engine")
+
+	ctx.Config.System.DatabaseEngine = engine
+	ctx.Config.System.DatabasePath = dbpath
+	ctx.Config.System.SetRootFS(rootfs)
+}
+
+func SetSolverConfig(ctx *types.Context) (c *types.LuetSolverOptions) {
+	stype := viper.GetString("solver.type")
+	discount := viper.GetFloat64("solver.discount")
+	rate := viper.GetFloat64("solver.rate")
+	attempts := viper.GetInt("solver.max_attempts")
+
+	ctx.Config.GetSolverOptions().Type = stype
+	ctx.Config.GetSolverOptions().LearnRate = float32(rate)
+	ctx.Config.GetSolverOptions().Discount = float32(discount)
+	ctx.Config.GetSolverOptions().MaxAttempts = attempts
+
+	return &types.LuetSolverOptions{
+		Type:        stype,
+		LearnRate:   float32(rate),
+		Discount:    float32(discount),
+		MaxAttempts: attempts,
+	}
+}
+
+func SetCliFinalizerEnvs(ctx *types.Context, finalizerEnvs []string) error {
+	if len(finalizerEnvs) > 0 {
+		for _, v := range finalizerEnvs {
+			idx := strings.Index(v, "=")
+			if idx < 0 {
+				return errors.New("Found invalid runtime finalizer environment: " + v)
+			}
+
+			ctx.Config.SetFinalizerEnv(v[0:idx], v[idx+1:])
+		}
+
+	}
+
+	return nil
+}
+
+// TemplateFolders returns the default folders which holds shared template between packages in a given tree path
+func TemplateFolders(ctx *types.Context, fromRepo bool, treePaths []string) []string {
+	templateFolders := []string{}
+	for _, t := range treePaths {
+		templateFolders = append(templateFolders, filepath.Join(t, "templates"))
+	}
+	if fromRepo {
+		for _, s := range installer.SystemRepositories(ctx.Config.SystemRepositories) {
+			templateFolders = append(templateFolders, filepath.Join(s.TreePath, "templates"))
+		}
+	}
+	return templateFolders
+}
+
+func IntroScreen() {
+	luetLogo, _ := pterm.DefaultBigText.WithLetters(
+		pterm.NewLettersFromStringWithStyle("LU", pterm.NewStyle(pterm.FgLightMagenta)),
+		pterm.NewLettersFromStringWithStyle("ET", pterm.NewStyle(pterm.FgLightBlue))).
+		Srender()
+
+	pterm.DefaultCenter.Print(luetLogo)
+
+	pterm.DefaultCenter.Print(pterm.DefaultHeader.WithFullWidth().WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(10).Sprint("Luet - 0-deps container-based package manager"))
+}
+
+func HandleLock(c *types.Context) {
+	if os.Getenv("LUET_NOLOCK") != "true" {
+		if len(os.Args) > 1 {
+			for _, lockedCmd := range lockedCommands {
+				if os.Args[1] == lockedCmd {
+					s := single.New("luet")
+					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
+						c.Fatal("another instance of the app is already running, exiting")
+					} else if err != nil {
+						// Another error occurred, might be worth handling it as well
+						c.Fatal("failed to acquire exclusive app lock:", err.Error())
+					}
+					defer s.TryUnlock()
+					break
+				}
+			}
+		}
+	}
+}
+
+func DisplayVersionBanner(c *types.Context, banner func(), version func() string, license []string) {
+	display := false
+	if len(os.Args) > 1 {
+		for _, c := range bannerCommands {
+			if os.Args[1] == c {
+				display = true
+			}
+		}
+	}
+	if display {
+		banner()
+		pterm.DefaultCenter.Print(version())
+		for _, l := range license {
+			pterm.DefaultCenter.Print(l)
+
+		}
+	}
+}

cmd/util/config.go

@@ -0,0 +1,207 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	extensions "github.com/mudler/cobra-extensions"
+	"github.com/mudler/luet/pkg/api/core/types"
+
+	helpers "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+const (
+	LuetEnvPrefix = "LUET"
+)
+
+var cfgFile string
+
+// initConfig reads in config file and ENV variables if set.
+func initConfig() {
+	setDefaults(viper.GetViper())
+	// Luet support these priorities on read configuration file:
+	// - command line option (if available)
+	// - $PWD/.luet.yaml
+	// - $HOME/.luet.yaml
+	// - /etc/luet/luet.yaml
+	//
+	// Note: currently a single viper instance support only one config name.
+
+	viper.SetEnvPrefix(LuetEnvPrefix)
+	viper.SetConfigType("yaml")
+
+	if cfgFile != "" { // enable ability to specify config file via flag
+		viper.SetConfigFile(cfgFile)
+	} else {
+		// Retrieve pwd directory
+		pwdDir, err := os.Getwd()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		homeDir := helpers.GetHomeDir()
+
+		if fileHelper.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && fileHelper.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
+			viper.AddConfigPath(".")
+			if homeDir != "" {
+				viper.AddConfigPath(homeDir)
+			}
+			viper.SetConfigName(".luet")
+		} else {
+			viper.SetConfigName("luet")
+			viper.AddConfigPath("/etc/luet")
+		}
+	}
+
+	viper.AutomaticEnv() // read in environment variables that match
+
+	// Create EnvKey Replacer for handle complex structure
+	replacer := strings.NewReplacer(".", "__")
+	viper.SetEnvKeyReplacer(replacer)
+	viper.SetTypeByDefaultValue(true)
+	// If a config file is found, read it in.
+	viper.ReadInConfig()
+
+}
+
+// InitContext inits the context by parsing the configurations from viper
+// this is meant to be run before each command to be able to parse any override from
+// the CLI/ENV
+func InitContext(ctx *types.Context) (err error) {
+
+	err = viper.Unmarshal(&ctx.Config)
+	if err != nil {
+		return
+	}
+
+	// Inits the context with the configurations loaded
+	// It reads system repositories, sets logging, and all the
+	// context which is required to perform luet actions
+	err = ctx.Init()
+	if err != nil {
+		return
+	}
+
+	// no_spinner is not mapped in our configs
+	ctx.NoSpinner = viper.GetBool("no_spinner")
+	return
+}
+
+func setDefaults(viper *viper.Viper) {
+	viper.SetDefault("logging.level", "info")
+	viper.SetDefault("logging.enable_logfile", false)
+	viper.SetDefault("logging.path", "/var/log/luet.log")
+	viper.SetDefault("logging.json_format", false)
+	viper.SetDefault("logging.enable_emoji", true)
+	viper.SetDefault("logging.color", true)
+
+	viper.SetDefault("general.concurrency", runtime.NumCPU())
+	viper.SetDefault("general.debug", false)
+	viper.SetDefault("general.show_build_output", false)
+	viper.SetDefault("general.fatal_warnings", false)
+	viper.SetDefault("general.http_timeout", 360)
+
+	u, err := user.Current()
+	// os/user doesn't work in from scratch environments
+	if err != nil || (u != nil && u.Uid == "0") {
+		viper.SetDefault("general.same_owner", true)
+	} else {
+		viper.SetDefault("general.same_owner", false)
+	}
+
+	viper.SetDefault("system.database_engine", "boltdb")
+	viper.SetDefault("system.database_path", "/var/cache/luet")
+	viper.SetDefault("system.rootfs", "/")
+	viper.SetDefault("system.tmpdir_base", filepath.Join(os.TempDir(), "tmpluet"))
+	viper.SetDefault("system.pkgs_cache_path", "packages")
+
+	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
+	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
+	viper.SetDefault("config_protect_skip", false)
+	// TODO: Set default to false when we are ready for migration.
+	viper.SetDefault("config_from_host", true)
+	viper.SetDefault("cache_repositories", []string{})
+	viper.SetDefault("system_repositories", []string{})
+	viper.SetDefault("finalizer_envs", make(map[string]string))
+
+	viper.SetDefault("solver.type", "")
+	viper.SetDefault("solver.rate", 0.7)
+	viper.SetDefault("solver.discount", 1.0)
+	viper.SetDefault("solver.max_attempts", 9000)
+}
+
+// InitViper inits a new viper
+// this is meant to be run just once at beginning to setup the root command
+func InitViper(ctx *types.Context, RootCmd *cobra.Command) {
+	cobra.OnInitialize(initConfig)
+	pflags := RootCmd.PersistentFlags()
+	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
+	pflags.BoolP("debug", "d", false, "verbose output")
+	pflags.Bool("fatal", false, "Enables Warnings to exit")
+	pflags.Bool("enable-logfile", false, "Enable log to file")
+	pflags.Bool("no-spinner", false, "Disable spinner.")
+	pflags.Bool("color", ctx.Config.GetLogging().Color, "Enable/Disable color.")
+	pflags.Bool("emoji", ctx.Config.GetLogging().EnableEmoji, "Enable/Disable emoji.")
+	pflags.Bool("skip-config-protect", ctx.Config.ConfigProtectSkip,
+		"Disable config protect analysis.")
+	pflags.StringP("logfile", "l", ctx.Config.GetLogging().Path,
+		"Logfile path. Empty value disable log to file.")
+	pflags.StringSlice("plugin", []string{}, "A list of runtime plugins to load")
+
+	// os/user doesn't work in from scratch environments.
+	// Check if i can retrieve user informations.
+	_, err := user.Current()
+	if err != nil {
+		ctx.Warning("failed to retrieve user identity:", err.Error())
+	}
+	pflags.Bool("same-owner", ctx.Config.GetGeneral().SameOwner, "Maintain same owner on uncompress.")
+	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
+	pflags.Int("http-timeout", ctx.Config.General.HTTPTimeout, "Default timeout for http(s) requests")
+
+	viper.BindPFlag("logging.color", pflags.Lookup("color"))
+	viper.BindPFlag("logging.enable_emoji", pflags.Lookup("emoji"))
+	viper.BindPFlag("logging.enable_logfile", pflags.Lookup("enable-logfile"))
+	viper.BindPFlag("logging.path", pflags.Lookup("logfile"))
+
+	viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
+	viper.BindPFlag("general.debug", pflags.Lookup("debug"))
+	viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
+	viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
+	viper.BindPFlag("plugin", pflags.Lookup("plugin"))
+	viper.BindPFlag("general.http_timeout", pflags.Lookup("http-timeout"))
+
+	// Currently I maintain this only from cli.
+	viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))
+	viper.BindPFlag("config_protect_skip", pflags.Lookup("skip-config-protect"))
+
+	// Extensions must be binary with the "luet-" prefix to be able to be shown in the help.
+	// we also accept extensions in the relative path where luet is being started, "extensions/"
+	exts := extensions.Discover("luet", "extensions")
+	for _, ex := range exts {
+		cobraCmd := ex.CobraCommand()
+		RootCmd.AddCommand(cobraCmd)
+	}
+}

cmd/util/search.go

@@ -0,0 +1,42 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import "github.com/pterm/pterm"
+
+type TableWriter struct {
+	td pterm.TableData
+}
+
+func (l *TableWriter) AppendRow(item []string) {
+	l.td = append(l.td, item)
+}
+
+func (l *TableWriter) Render() {
+	pterm.DefaultTable.WithHasHeader().WithData(l.td).Render()
+}
+
+type ListWriter struct {
+	bb []pterm.BulletListItem
+}
+
+func (l *ListWriter) AppendItem(item pterm.BulletListItem) {
+	l.bb = append(l.bb, item)
+}
+
+func (l *ListWriter) Render() {
+	pterm.DefaultBulletList.WithItems(l.bb).Render()
+}

cmd/util/unpack.go

@@ -1,98 +0,0 @@
-// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package util
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/go-units"
-	"github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
-	. "github.com/mudler/luet/pkg/logger"
-
-	"github.com/spf13/cobra"
-)
-
-func NewUnpackCommand() *cobra.Command {
-
-	c := &cobra.Command{
-		Use:   "unpack image path",
-		Short: "Unpack a docker image natively",
-		Long: `unpack doesn't need the docker daemon to run, and unpacks a docker image in the specified directory:
-		
-	luet util unpack golang:alpine /alpine
-`,
-		PreRun: func(cmd *cobra.Command, args []string) {
-
-			if len(args) != 2 {
-				Fatal("Expects an image and a path")
-			}
-
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-
-			image := args[0]
-			destination, err := filepath.Abs(args[1])
-			if err != nil {
-				Error("Invalid path %s", destination)
-				os.Exit(1)
-			}
-
-			verify, _ := cmd.Flags().GetBool("verify")
-			user, _ := cmd.Flags().GetString("auth-username")
-			pass, _ := cmd.Flags().GetString("auth-password")
-			authType, _ := cmd.Flags().GetString("auth-type")
-			server, _ := cmd.Flags().GetString("auth-server-address")
-			identity, _ := cmd.Flags().GetString("auth-identity-token")
-			registryToken, _ := cmd.Flags().GetString("auth-registry-token")
-
-			temp, err := config.LuetCfg.GetSystem().TempDir("contentstore")
-			if err != nil {
-				Fatal("Cannot create a tempdir", err.Error())
-			}
-
-			Info("Downloading", image, "to", destination)
-			auth := &types.AuthConfig{
-				Username:      user,
-				Password:      pass,
-				ServerAddress: server,
-				Auth:          authType,
-				IdentityToken: identity,
-				RegistryToken: registryToken,
-			}
-
-			info, err := helpers.DownloadAndExtractDockerImage(temp, image, destination, auth, verify)
-			if err != nil {
-				Error(err.Error())
-				os.Exit(1)
-			}
-			Info(fmt.Sprintf("Pulled: %s %s", info.Target.Digest, info.Name))
-			Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
-		},
-	}
-
-	c.Flags().String("auth-username", "", "Username to authenticate to registry/notary")
-	c.Flags().String("auth-password", "", "Password to authenticate to registry")
-	c.Flags().String("auth-type", "", "Auth type")
-	c.Flags().String("auth-server-address", "", "Authentication server address")
-	c.Flags().String("auth-identity-token", "", "Authentication identity token")
-	c.Flags().String("auth-registry-token", "", "Authentication registry token")
-	c.Flags().Bool("verify", false, "Verify signed images to notary before to pull")
-	return c
-}

contrib/config/get_luet_root.sh

@@ -7,7 +7,7 @@ fi
 set -ex
 export LUET_NOLOCK=true
 
-LUET_VERSION=$(curl -s https://api.github.com/repos/mudler/luet/releases/latest | ( grep -oP '"tag_name": "\K(.*)(?=")' || echo "0.9.24" ))
+LUET_VERSION=$(curl -s https://api.github.com/repos/mudler/luet/releases/latest | grep tag_name | awk '{ print $2 }' | sed -e 's/\"//g' -e 's/,//g' || echo "0.9.24" )
 LUET_ROOTFS=${LUET_ROOTFS:-/}
 LUET_DATABASE_PATH=${LUET_DATABASE_PATH:-/var/luet/db}
 LUET_DATABASE_ENGINE=${LUET_DATABASE_ENGINE:-boltdb}

contrib/config/luet.yaml

@@ -99,6 +99,14 @@
 # If set to false rootfs path is used as prefix.
 # config_from_host: true
 #
+#
+# ------------------------------------------------
+# Finalizer Environment Variables
+# -----------------------------------------------
+# finalizer_envs:
+#  - key: "BUILD_ISO"
+#    value: "1"
+#
 # System repositories
 # ---------------------------------------------
 # In alternative to define repositories files

go.mod

@@ -1,71 +1,85 @@
 module github.com/mudler/luet
 
-go 1.14
+go 1.16
 
 require (
-	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/Sabayon/pkgs-checker v0.8.1
+	github.com/DataDog/zstd v1.4.5 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.1.1 // indirect
+	github.com/Microsoft/go-winio v0.5.0 // indirect
+	github.com/Sabayon/pkgs-checker v0.8.4
+	github.com/apex/log v1.9.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
-	github.com/briandowns/spinner v1.12.1-0.20201220203425-e201aaea0a31
 	github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec
+	github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d // indirect
 	github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc
-	github.com/crillab/gophersat v1.3.2-0.20201023142334-3fc2ac466765
-	github.com/docker/cli v0.0.0-20200227165822-2298e6a3fe24
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/crillab/gophersat v1.3.2-0.20210701121804-72b19f5b6b38
+	github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
-	github.com/genuinetools/img v0.5.11
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/ghodss/yaml v1.0.0
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/go-containerregistry v0.2.1
 	github.com/google/renameio v1.0.0
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gookit/color v1.5.0 // indirect
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/hashicorp/go-version v1.2.1
-	github.com/imdario/mergo v0.3.8
-	github.com/jedib0t/go-pretty v4.3.0+incompatible
-	github.com/jedib0t/go-pretty/v6 v6.0.5
+	github.com/hashicorp/go-version v1.3.0
+	github.com/huandu/xstrings v1.3.2 // indirect
+	github.com/imdario/mergo v0.3.12
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
-	github.com/klauspost/compress v1.8.3
+	github.com/klauspost/compress v1.12.2
 	github.com/klauspost/pgzip v1.2.1
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/kyokomi/emoji v2.1.0+incompatible
-	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
-	github.com/moby/buildkit v0.7.2
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/hashstructure/v2 v2.0.1
+	github.com/mitchellh/mapstructure v1.4.2 // indirect
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/mudler/go-pluggable v0.0.0-20210510180427-ba09243a8c65
+	github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
-	github.com/onsi/ginkgo v1.14.2
-	github.com/onsi/gomega v1.10.3
+	github.com/onsi/ginkgo v1.16.4
+	github.com/onsi/gomega v1.16.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
+	github.com/opencontainers/runc v1.0.0-rc9.0.20200221051241-688cf6d43cc4 // indirect
 	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
+	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.7.1
-	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/cobra v1.1.1
-	github.com/spf13/viper v1.7.1
+	github.com/pterm/pterm v0.12.32-0.20211002183613-ada9ef6790c3
+	github.com/rancher-sandbox/gofilecache v0.0.0-20210330135715-becdeff5df15
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/cobra v1.2.1
+	github.com/spf13/viper v1.8.1
 	github.com/theupdateframework/notary v0.7.0
 	go.etcd.io/bbolt v1.3.5
-	go.uber.org/atomic v1.5.1 // indirect
-	go.uber.org/multierr v1.4.0
-	go.uber.org/zap v1.13.0
-	google.golang.org/grpc v1.29.1
-	gopkg.in/yaml.v2 v2.3.0
+	go.uber.org/multierr v1.6.0
+	go.uber.org/zap v1.17.0
+	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
+	golang.org/x/mod v0.4.2
+	golang.org/x/oauth2 v0.0.0-20210810183815-faf39c7919d5 // indirect
+	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	google.golang.org/genproto v0.0.0-20210811021853-ddbe55d93216 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/ini.v1 v1.63.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0
+	gotest.tools/v3 v3.0.2 // indirect
 	helm.sh/helm/v3 v3.3.4
 
 )
 
 replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200605210607-749178b8f80d+incompatible
-
-replace github.com/containerd/containerd => github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55
-
-replace github.com/hashicorp/go-immutable-radix => github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe
-
-replace github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305
-
-replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc9.0.20200221051241-688cf6d43cc4

pkg/api/client/client_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestClient(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Client API Suite")
+}

pkg/api/client/search.go

@@ -0,0 +1,131 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/mudler/luet/pkg/api/client/utils"
+)
+
+func TreePackages(treedir string) (searchResult SearchResult, err error) {
+	var res []byte
+	res, err = utils.RunSHOUT("tree", fmt.Sprintf("luet tree pkglist --tree %s --output json", treedir))
+	if err != nil {
+		fmt.Println(string(res))
+		return
+	}
+	json.Unmarshal(res, &searchResult)
+	return
+}
+
+func imageAvailable(image string) bool {
+	_, err := crane.Digest(image)
+	return err == nil
+}
+
+type SearchResult struct {
+	Packages []Package
+}
+
+type Package struct {
+	Name, Category, Version, Path string
+}
+
+func (p Package) String() string {
+	return fmt.Sprintf("%s/%s@%s", p.Category, p.Name, p.Version)
+}
+
+func (p Package) Image(repository string) string {
+	return fmt.Sprintf("%s:%s-%s-%s", repository, p.Name, p.Category, strings.ReplaceAll(p.Version, "+", "-"))
+}
+
+func (p Package) ImageTag() string {
+	// ${name}-${category}-${version//+/-}
+	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, strings.ReplaceAll(p.Version, "+", "-"))
+}
+
+func (p Package) ImageMetadata(repository string) string {
+	return fmt.Sprintf("%s.metadata.yaml", p.Image(repository))
+}
+
+func (p Package) ImageAvailable(repository string) bool {
+	return imageAvailable(p.Image(repository))
+}
+
+func (p Package) Equal(pp Package) bool {
+	if p.Name == pp.Name && p.Category == pp.Category && p.Version == pp.Version {
+		return true
+	}
+	return false
+}
+
+func (p Package) EqualS(s string) bool {
+	if s == fmt.Sprintf("%s/%s", p.Category, p.Name) {
+		return true
+	}
+	return false
+}
+
+func (p Package) EqualSV(s string) bool {
+	if s == fmt.Sprintf("%s/%s@%s", p.Category, p.Name, p.Version) {
+		return true
+	}
+	return false
+}
+
+func (p Package) EqualNoV(pp Package) bool {
+	if p.Name == pp.Name && p.Category == pp.Category {
+		return true
+	}
+	return false
+}
+
+func (s SearchResult) FilterByCategory(cat string) SearchResult {
+	new := SearchResult{Packages: []Package{}}
+
+	for _, r := range s.Packages {
+		if r.Category == cat {
+			new.Packages = append(new.Packages, r)
+		}
+	}
+	return new
+}
+
+func (s SearchResult) FilterByName(name string) SearchResult {
+	new := SearchResult{Packages: []Package{}}
+
+	for _, r := range s.Packages {
+		if !strings.Contains(r.Name, name) {
+			new.Packages = append(new.Packages, r)
+		}
+	}
+	return new
+}
+
+type Packages []Package
+
+func (p Packages) Exist(pp Package) bool {
+	for _, pi := range p {
+		if pp.Equal(pi) {
+			return true
+		}
+	}
+	return false
+}

pkg/api/client/search_test.go

@@ -0,0 +1,47 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	. "github.com/mudler/luet/pkg/api/client"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Client CLI API", func() {
+	Context("Reads a package tree from the luet CLI", func() {
+		It("Correctly detect packages", func() {
+			t, err := TreePackages("../../../tests/fixtures/alpine")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(t).ToNot(BeNil())
+			Expect(len(t.Packages)).To(Equal(1))
+			Expect(t.Packages[0].Name).To(Equal("alpine"))
+			Expect(t.Packages[0].Category).To(Equal("seed"))
+			Expect(t.Packages[0].Version).To(Equal("1.0"))
+			Expect(t.Packages[0].ImageAvailable("foo")).To(BeFalse())
+			Expect(t.Packages[0].Equal(t.Packages[0])).To(BeTrue())
+			Expect(t.Packages[0].Equal(Package{})).To(BeFalse())
+			Expect(t.Packages[0].EqualNoV(Package{Name: "alpine", Category: "seed"})).To(BeTrue())
+			Expect(t.Packages[0].EqualS("seed/alpine")).To(BeTrue())
+			Expect(t.Packages[0].EqualS("seed/alpinev")).To(BeFalse())
+			Expect(t.Packages[0].EqualSV("seed/alpine@1.0")).To(BeTrue())
+			Expect(t.Packages[0].Image("foo")).To(Equal("foo:alpine-seed-1.0"))
+			Expect(Packages(t.Packages).Exist(t.Packages[0])).To(BeTrue())
+			Expect(Packages(t.Packages).Exist(Package{})).To(BeFalse())
+
+		})
+	})
+})

pkg/api/client/utils/sh.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+func RunSHOUT(stepName, bashFragment string) ([]byte, error) {
+	cmd := exec.Command("sh", "-s")
+	cmd.Stdin = strings.NewReader(bashWrap(bashFragment))
+
+	cmd.Env = os.Environ()
+	//	log.Printf("Running in background: %v", stepName)
+
+	return cmd.CombinedOutput()
+}
+
+func RunSH(stepName, bashFragment string) error {
+	cmd := exec.Command("sh", "-s")
+	cmd.Stdin = strings.NewReader(bashWrap(bashFragment))
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = os.Environ()
+	log.Printf("Running: %v (%v)", stepName, bashFragment)
+
+	return cmd.Run()
+}
+
+func bashWrap(cmd string) string {
+	return `
+set -o errexit
+set -o nounset
+` + cmd + `
+`
+}

pkg/api/core/config/config_protect.go

@@ -65,10 +65,7 @@ func (c *ConfigProtect) GetAnnotationDir() string {
 	return c.AnnotationDir
 }
 
-func (c *ConfigProtect) Map(files []string) {
-	if LuetCfg.ConfigProtectSkip {
-		return
-	}
+func (c *ConfigProtect) Map(files []string, protected []ConfigProtectConfFile) {
 
 	for _, file := range files {
 
@@ -76,8 +73,8 @@ func (c *ConfigProtect) Map(files []string) {
 			file = "/" + file
 		}
 
-		if len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
-			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
+		if len(protected) > 0 {
+			for _, conf := range protected {
 				for _, dir := range conf.Directories {
 					// Note file is without / at begin (on unpack)
 					if strings.HasPrefix(file, filepath.Clean(dir)) {

pkg/api/core/config/config_protect_test.go

@@ -17,7 +17,8 @@
 package config_test
 
 import (
-	config "github.com/mudler/luet/pkg/config"
+	config "github.com/mudler/luet/pkg/api/core/config"
+	"github.com/mudler/luet/pkg/api/core/types"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -28,7 +29,7 @@ var _ = Describe("Config", func() {
 	Context("Test config protect", func() {
 
 		It("Protect1", func() {
-
+			ctx := types.NewContext()
 			files := []string{
 				"etc/foo/my.conf",
 				"usr/bin/foo",
@@ -36,7 +37,7 @@ var _ = Describe("Config", func() {
 			}
 
 			cp := config.NewConfigProtect("/etc")
-			cp.Map(files)
+			cp.Map(files, ctx.Config.ConfigProtectConfFiles)
 
 			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
 			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
@@ -58,6 +59,7 @@ var _ = Describe("Config", func() {
 		})
 
 		It("Protect2", func() {
+			ctx := types.NewContext()
 
 			files := []string{
 				"etc/foo/my.conf",
@@ -66,7 +68,7 @@ var _ = Describe("Config", func() {
 			}
 
 			cp := config.NewConfigProtect("")
-			cp.Map(files)
+			cp.Map(files, ctx.Config.ConfigProtectConfFiles)
 
 			Expect(cp.Protected("etc/foo/my.conf")).To(BeFalse())
 			Expect(cp.Protected("/etc/foo/my.conf")).To(BeFalse())
@@ -84,6 +86,7 @@ var _ = Describe("Config", func() {
 		})
 
 		It("Protect3: Annotation dir without initial slash", func() {
+			ctx := types.NewContext()
 
 			files := []string{
 				"etc/foo/my.conf",
@@ -92,7 +95,7 @@ var _ = Describe("Config", func() {
 			}
 
 			cp := config.NewConfigProtect("etc")
-			cp.Map(files)
+			cp.Map(files, ctx.Config.ConfigProtectConfFiles)
 
 			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
 			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())

pkg/api/core/config/config_suite_test.go

@@ -19,15 +19,11 @@ package config_test
 import (
 	"testing"
 
-	. "github.com/mudler/luet/cmd"
-	config "github.com/mudler/luet/pkg/config"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
-	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Config Suite")
 }

pkg/api/core/types/artifact/artifact.go

@@ -19,6 +19,8 @@ import (
 	"archive/tar"
 	"bufio"
 	"bytes"
+	"crypto/sha1"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -34,31 +36,35 @@ import (
 	"strings"
 	"sync"
 
+	config "github.com/mudler/luet/pkg/api/core/config"
+	types "github.com/mudler/luet/pkg/api/core/types"
 	bus "github.com/mudler/luet/pkg/bus"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
-	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
-	. "github.com/mudler/luet/pkg/logger"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
+
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v2"
 )
 
-//  When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository
+// When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository
 // which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact.
 // In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages.
 type PackageArtifact struct {
 	Path string `json:"path"`
 
-	Dependencies    []*PackageArtifact                `json:"dependencies"`
-	CompileSpec     *compilerspec.LuetCompilationSpec `json:"compilationspec"`
-	Checksums       Checksums                         `json:"checksums"`
-	SourceAssertion solver.PackagesAssertions         `json:"-"`
-	CompressionType compression.Implementation        `json:"compressiontype"`
-	Files           []string                          `json:"files"`
+	Dependencies      []*PackageArtifact                `json:"dependencies"`
+	CompileSpec       *compilerspec.LuetCompilationSpec `json:"compilationspec"`
+	Checksums         Checksums                         `json:"checksums"`
+	SourceAssertion   solver.PackagesAssertions         `json:"-"`
+	CompressionType   compression.Implementation        `json:"compressiontype"`
+	Files             []string                          `json:"files"`
+	PackageCacheImage string                            `json:"package_cacheimage"`
+	Runtime           *pkg.DefaultPackage               `json:"runtime,omitempty"`
 }
 
 func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
@@ -97,19 +103,22 @@ func (a *PackageArtifact) Verify() error {
 	return nil
 }
 
-func (a *PackageArtifact) WriteYaml(dst string) error {
+func (a *PackageArtifact) WriteYAML(dst string) error {
 	// First compute checksum of artifact. When we write the yaml we want to write up-to-date informations.
 	err := a.Hash()
 	if err != nil {
 		return errors.Wrap(err, "Failed generating checksums for artifact")
 	}
 
-	//p := a.CompileSpec.GetPackage().GetPath()
+	// Update runtime package information
+	if a.CompileSpec != nil && a.CompileSpec.Package != nil {
+		runtime, err := a.CompileSpec.Package.GetRuntimePackage()
+		if err != nil {
+			return errors.Wrapf(err, "getting runtime package for '%s'", a.CompileSpec.Package.HumanReadableString())
+		}
+		a.Runtime = runtime
+	}
 
-	//a.CompileSpec.GetPackage().SetPath("")
-	//	for _, ass := range a.CompileSpec.GetSourceAssertion() {
-	//		ass.Package.SetPath("")
-	//	}
 	data, err := yaml.Marshal(a)
 	if err != nil {
 		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
@@ -154,22 +163,22 @@ COPY . /`
 }
 
 // CreateArtifactForFile creates a new artifact from the given file
-func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
+func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
 	if _, err := os.Stat(s); os.IsNotExist(err) {
 		return nil, errors.Wrap(err, "artifact path doesn't exist")
 	}
 	fileName := path.Base(s)
-	archive, err := LuetCfg.GetSystem().TempDir("archive")
+	archive, err := ctx.Config.GetSystem().TempDir("archive")
 	if err != nil {
 		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
 	}
 	defer os.RemoveAll(archive) // clean up
 	dst := filepath.Join(archive, fileName)
-	if err := helpers.CopyFile(s, dst); err != nil {
+	if err := fileHelper.CopyFile(s, dst); err != nil {
 		return nil, errors.Wrapf(err, "error while copying %s to %s", s, dst)
 	}
 
-	artifact, err := LuetCfg.GetSystem().TempDir("artifact")
+	artifact, err := ctx.Config.GetSystem().TempDir("artifact")
 	if err != nil {
 		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
 	}
@@ -187,9 +196,9 @@ type ImageBuilder interface {
 }
 
 // GenerateFinalImage takes an artifact and builds a Docker image with its content
-func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) {
+func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) {
 	builderOpts := backend.Options{}
-	archive, err := LuetCfg.GetSystem().TempDir("archive")
+	archive, err := ctx.Config.GetSystem().TempDir("archive")
 	if err != nil {
 		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
 	}
@@ -202,11 +211,11 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, k
 		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
 	}
 
-	if err := a.Unpack(uncompressedFiles, keepPerms); err != nil {
+	if err := a.Unpack(ctx, uncompressedFiles, keepPerms); err != nil {
 		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
 	}
 
-	empty, err := helpers.DirectoryIsEmpty(uncompressedFiles)
+	empty, err := fileHelper.DirectoryIsEmpty(uncompressedFiles)
 	if err != nil {
 		return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles)
 	}
@@ -215,7 +224,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, k
 	// We can't generate FROM scratch empty images. Docker will refuse to export them
 	// workaround: Inject a .virtual empty file
 	if empty {
-		helpers.Touch(filepath.Join(uncompressedFiles, ".virtual"))
+		fileHelper.Touch(filepath.Join(uncompressedFiles, ".virtual"))
 	}
 
 	data := a.genDockerfile()
@@ -272,7 +281,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 		}
 
 		os.RemoveAll(a.Path) // Remove original
-		Debug("Removed artifact", a.Path)
+		//	Debug("Removed artifact", a.Path)
 
 		a.Path = zstdFile
 		return nil
@@ -306,7 +315,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 		}
 		w.Close()
 		os.RemoveAll(a.Path) // Remove original
-		Debug("Removed artifact", a.Path)
+		//	Debug("Removed artifact", a.Path)
 		//	a.CompressedPath = gzipfile
 		a.Path = gzipfile
 		return nil
@@ -316,7 +325,6 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	default:
 		return helpers.Tar(src, a.getCompressedName())
 	}
-	return errors.New("Compression type must be supplied")
 }
 
 func (a *PackageArtifact) getCompressedName() string {
@@ -339,58 +347,96 @@ func (a *PackageArtifact) GetUncompressedName() string {
 	return a.Path
 }
 
-func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
-	// If the destination path already exists I rename target file name with postfix.
-	var destPath string
-
-	// Read data. TODO: We need change archive callback to permit to return a Reader
-	buffer := bytes.Buffer{}
-	if content != nil {
-		if _, err := buffer.ReadFrom(content); err != nil {
-			return nil, nil, err
-		}
-	}
-
-	// If file is not present on archive but is defined on mods
-	// I receive the callback. Prevent nil exception.
-	if header != nil {
-		switch header.Typeflag {
-		case tar.TypeReg:
-			destPath = filepath.Join(dst, path)
-		default:
-			// Nothing to do. I return original reader
-			return header, buffer.Bytes(), nil
-		}
-
-		// Check if exists
-		if helpers.Exists(destPath) {
-			for i := 1; i < 1000; i++ {
-				name := filepath.Join(filepath.Join(filepath.Dir(path),
-					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
-
-				if helpers.Exists(name) {
-					continue
-				}
-				Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
-					filepath.Join(dst, name)))
-				return &tar.Header{
-					Mode:       header.Mode,
-					Typeflag:   header.Typeflag,
-					PAXRecords: header.PAXRecords,
-					Name:       name,
-				}, buffer.Bytes(), nil
-			}
-		}
-	}
-
-	return header, buffer.Bytes(), nil
+func hashContent(bv []byte) string {
+	hasher := sha1.New()
+	hasher.Write(bv)
+	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	return sha
 }
 
-func (a *PackageArtifact) GetProtectFiles() []string {
+func hashFileContent(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	h := sha1.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+
+	return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil
+}
+
+func tarModifierWrapperFunc(ctx *types.Context) func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	return func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+		// If the destination path already exists I rename target file name with postfix.
+		var destPath string
+
+		// Read data. TODO: We need change archive callback to permit to return a Reader
+		buffer := bytes.Buffer{}
+		if content != nil {
+			if _, err := buffer.ReadFrom(content); err != nil {
+				return nil, nil, err
+			}
+		}
+		tarHash := hashContent(buffer.Bytes())
+
+		// If file is not present on archive but is defined on mods
+		// I receive the callback. Prevent nil exception.
+		if header != nil {
+			switch header.Typeflag {
+			case tar.TypeReg:
+				destPath = filepath.Join(dst, path)
+			default:
+				// Nothing to do. I return original reader
+				return header, buffer.Bytes(), nil
+			}
+
+			existingHash := ""
+			f, err := os.Lstat(destPath)
+			if err == nil {
+				ctx.Debug("File exists already, computing hash for", destPath)
+				hash, herr := hashFileContent(destPath)
+				if herr == nil {
+					existingHash = hash
+				}
+			}
+
+			ctx.Debug("Existing file hash: ", existingHash, "Tar file hashsum: ", tarHash)
+			// We want to protect file only if the hash of the files are differing OR the file size are
+			differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
+			// Check if exists
+			if fileHelper.Exists(destPath) && differs {
+				for i := 1; i < 1000; i++ {
+					name := filepath.Join(filepath.Join(filepath.Dir(path),
+						fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
+
+					if fileHelper.Exists(name) {
+						continue
+					}
+					ctx.Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
+						filepath.Join(dst, name)))
+					return &tar.Header{
+						Mode:       header.Mode,
+						Typeflag:   header.Typeflag,
+						PAXRecords: header.PAXRecords,
+						Name:       name,
+					}, buffer.Bytes(), nil
+				}
+			}
+		}
+
+		return header, buffer.Bytes(), nil
+	}
+}
+
+func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) []string {
 	ans := []string{}
 	annotationDir := ""
 
-	if !LuetCfg.ConfigProtectSkip {
+	if !ctx.Config.ConfigProtectSkip {
 
 		// a.CompileSpec could be nil when artifact.Unpack is used for tree tarball
 		if a.CompileSpec != nil &&
@@ -402,8 +448,8 @@ func (a *PackageArtifact) GetProtectFiles() []string {
 		}
 		// TODO: check if skip this if we have a.CompileSpec nil
 
-		cp := NewConfigProtect(annotationDir)
-		cp.Map(a.Files)
+		cp := config.NewConfigProtect(annotationDir)
+		cp.Map(a.Files, ctx.Config.GetConfigProtectConfFiles())
 
 		// NOTE: for unpack we need files path without initial /
 		ans = cp.GetProtectFiles(false)
@@ -413,12 +459,15 @@ func (a *PackageArtifact) GetProtectFiles() []string {
 }
 
 // Unpack Untar and decompress (TODO) to the given path
-func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+func (a *PackageArtifact) Unpack(ctx *types.Context, dst string, keepPerms bool) error {
+	if !strings.HasPrefix(dst, "/") {
+		return errors.New("destination must be an absolute path")
+	}
 
 	// Create
-	protectedFiles := a.GetProtectFiles()
+	protectedFiles := a.GetProtectFiles(ctx)
 
-	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
+	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc(ctx))
 
 	switch a.CompressionType {
 	case compression.Zstandard:
@@ -450,7 +499,7 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 		}
 
 		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
-			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
+			ctx.Config.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
 		}
@@ -483,14 +532,14 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 		}
 
 		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
-			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
+			ctx.Config.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
 		}
 		return nil
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.UntarProtect(a.Path, dst, LuetCfg.GetGeneral().SameOwner,
+		return helpers.UntarProtect(a.Path, dst, ctx.Config.GetGeneral().SameOwner,
 			protectedFiles, tarModifier)
 	}
 	return errors.New("Compression type must be supplied")
@@ -583,15 +632,15 @@ type CopyJob struct {
 	Artifact string
 }
 
-func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
+func worker(ctx *types.Context, i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
 	for job := range s {
 		_, err := os.Lstat(job.Dst)
 		if err != nil {
-			Debug("Copying ", job.Src)
-			if err := helpers.DeepCopyFile(job.Src, job.Dst); err != nil {
-				Warning("Error copying", job, err)
+			ctx.Debug("Copying ", job.Src)
+			if err := fileHelper.DeepCopyFile(job.Src, job.Dst); err != nil {
+				ctx.Warning("Error copying", job, err)
 			}
 		}
 	}
@@ -602,7 +651,6 @@ func compileRegexes(regexes []string) []*regexp.Regexp {
 	for _, i := range regexes {
 		r, e := regexp.Compile(i)
 		if e != nil {
-			Warning("Failed compiling regex:", e)
 			continue
 		}
 		result = append(result, r)
@@ -627,16 +675,16 @@ type ArtifactLayer struct {
 }
 
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) {
+func ExtractArtifactFromDelta(ctx *types.Context, src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) {
 
-	archive, err := LuetCfg.GetSystem().TempDir("archive")
+	archive, err := ctx.Config.GetSystem().TempDir("archive")
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for archive")
 	}
 	defer os.RemoveAll(archive) // clean up
 
 	if strings.HasSuffix(src, ".tar") {
-		rootfs, err := LuetCfg.GetSystem().TempDir("rootfs")
+		rootfs, err := ctx.Config.GetSystem().TempDir("rootfs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 		}
@@ -653,7 +701,7 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 	var wg = new(sync.WaitGroup)
 	for i := 0; i < concurrency; i++ {
 		wg.Add(1)
-		go worker(i, wg, toCopy)
+		go worker(ctx, i, wg, toCopy)
 	}
 
 	// Handle includes in spec. If specified they filter what gets in the package
@@ -672,10 +720,10 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 				}
 			}
 			for _, a := range l.Diffs.Changes {
-				Debug("File ", a.Name, " changed")
+				ctx.Debug("File ", a.Name, " changed")
 			}
 			for _, a := range l.Diffs.Deletions {
-				Debug("File ", a.Name, " deleted")
+				ctx.Debug("File ", a.Name, " deleted")
 			}
 		}
 
@@ -693,10 +741,10 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
 			}
 			for _, a := range l.Diffs.Changes {
-				Debug("File ", a.Name, " changed")
+				ctx.Debug("File ", a.Name, " changed")
 			}
 			for _, a := range l.Diffs.Deletions {
-				Debug("File ", a.Name, " deleted")
+				ctx.Debug("File ", a.Name, " deleted")
 			}
 		}
 
@@ -721,10 +769,10 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 				}
 			}
 			for _, a := range l.Diffs.Changes {
-				Debug("File ", a.Name, " changed")
+				ctx.Debug("File ", a.Name, " changed")
 			}
 			for _, a := range l.Diffs.Deletions {
-				Debug("File ", a.Name, " deleted")
+				ctx.Debug("File ", a.Name, " deleted")
 			}
 		}
 
@@ -733,14 +781,14 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 			for _, a := range l.Diffs.Additions {
-				Debug("File ", a.Name, " added")
+				ctx.Debug("File ", a.Name, " added")
 				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
 			}
 			for _, a := range l.Diffs.Changes {
-				Debug("File ", a.Name, " changed")
+				ctx.Debug("File ", a.Name, " changed")
 			}
 			for _, a := range l.Diffs.Deletions {
-				Debug("File ", a.Name, " deleted")
+				ctx.Debug("File ", a.Name, " deleted")
 			}
 		}
 	}

pkg/api/core/types/artifact/artifact_suite_test.go

@@ -1,5 +1,4 @@
 // Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//                  Daniele Rondina <geaaru@sabayonlinux.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -14,20 +13,16 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package repository_test
+package artifact_test
 
 import (
 	"testing"
 
-	. "github.com/mudler/luet/cmd"
-	config "github.com/mudler/luet/pkg/config"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
-func TestSolver(t *testing.T) {
+func TestArtifact(t *testing.T) {
 	RegisterFailHandler(Fail)
-	LoadConfig(config.LuetCfg)
-	RunSpecs(t, "Repository Suite")
+	RunSpecs(t, "Artifact Suite")
 }

pkg/api/core/types/artifact/artifact_test.go

@@ -20,15 +20,18 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
-	. "github.com/mudler/luet/pkg/compiler/types/artifact"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -37,16 +40,17 @@ import (
 
 var _ = Describe("Artifact", func() {
 	Context("Simple package build definition", func() {
+		ctx := types.NewContext()
 		It("Generates a verified delta", func() {
 
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
+			err := generalRecipe.Load("../../../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
 			lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -71,7 +75,7 @@ var _ = Describe("Artifact", func() {
 
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -80,7 +84,7 @@ WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin`))
-			b := NewSimpleDockerBackend()
+			b := NewSimpleDockerBackend(ctx)
 			opts := backend.Options{
 				ImageName:      "luet/base",
 				SourcePath:     tmpdir,
@@ -89,12 +93,12 @@ ENV PACKAGE_CATEGORY=app-admin`))
 			}
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
@@ -113,8 +117,8 @@ RUN echo bar > /test2`))
 			}
 			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
-			diffs, err := compiler.GenerateChanges(b, opts, opts2)
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
+			diffs, err := compiler.GenerateChanges(ctx, b, opts, opts2)
 			Expect(err).ToNot(HaveOccurred())
 
 			artifacts := []ArtifactNode{{
@@ -138,17 +142,17 @@ RUN echo bar > /test2`))
 			err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			a, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
+			a, err := ExtractArtifactFromDelta(ctx, rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
 			err = helpers.Untar(a.Path, unpacked, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
-			content1, err := helpers.Read(filepath.Join(unpacked, "test"))
+			Expect(fileHelper.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
+			content1, err := fileHelper.Read(filepath.Join(unpacked, "test"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("foo\n"))
-			content2, err := helpers.Read(filepath.Join(unpacked, "test2"))
+			content2, err := fileHelper.Read(filepath.Join(unpacked, "test2"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content2).To(Equal("bar\n"))
 
@@ -156,14 +160,14 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			err = a.Verify()
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
+			Expect(fileHelper.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
 
 			err = a.Verify()
 			Expect(err).To(HaveOccurred())
 		})
 
 		It("Generates packages images", func() {
-			b := NewSimpleDockerBackend()
+			b := NewSimpleDockerBackend(ctx)
 			imageprefix := "foo/"
 			testString := []byte(`funky test data`)
 
@@ -189,7 +193,7 @@ RUN echo bar > /test2`))
 			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := a.GenerateFinalImage(resultingImage, b, false)
+			opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(opts.ImageName).To(Equal(resultingImage))
 
@@ -214,7 +218,7 @@ RUN echo bar > /test2`))
 		})
 
 		It("Generates empty packages images", func() {
-			b := NewSimpleDockerBackend()
+			b := NewSimpleDockerBackend(ctx)
 			imageprefix := "foo/"
 
 			tmpdir, err := ioutil.TempDir(os.TempDir(), "artifact")
@@ -231,7 +235,7 @@ RUN echo bar > /test2`))
 			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := a.GenerateFinalImage(resultingImage, b, false)
+			opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(opts.ImageName).To(Equal(resultingImage))
 
@@ -244,7 +248,7 @@ RUN echo bar > /test2`))
 			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.DirectoryIsEmpty(result)).To(BeFalse())
+			Expect(fileHelper.DirectoryIsEmpty(result)).To(BeFalse())
 			content, err := ioutil.ReadFile(filepath.Join(result, ".virtual"))
 			Expect(err).ToNot(HaveOccurred())
 

pkg/api/core/types/artifact/cache.go

@@ -0,0 +1,63 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact
+
+import (
+	"crypto/sha512"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/rancher-sandbox/gofilecache"
+)
+
+type ArtifactCache struct {
+	gofilecache.Cache
+}
+
+func NewCache(dir string) *ArtifactCache {
+	return &ArtifactCache{Cache: *gofilecache.InitCache(dir)}
+}
+
+func (c *ArtifactCache) cacheID(a *PackageArtifact) [64]byte {
+	fingerprint := filepath.Base(a.Path)
+	if a.CompileSpec != nil && a.CompileSpec.Package != nil {
+		fingerprint = a.CompileSpec.Package.GetFingerPrint()
+	}
+	if len(a.Checksums) > 0 {
+		for _, cs := range a.Checksums.List() {
+			t := cs[0]
+			result := cs[1]
+			fingerprint += fmt.Sprintf("+%s:%s", t, result)
+		}
+	}
+	return sha512.Sum512([]byte(fingerprint))
+}
+
+func (c *ArtifactCache) Get(a *PackageArtifact) (string, error) {
+	fileName, _, err := c.Cache.GetFile(c.cacheID(a))
+	return fileName, err
+}
+
+func (c *ArtifactCache) Put(a *PackageArtifact) (gofilecache.OutputID, int64, error) {
+	file, err := os.Open(a.Path)
+	if err != nil {
+		return [64]byte{}, 0, errors.Wrapf(err, "failed opening %s", a.Path)
+	}
+	defer file.Close()
+	return c.Cache.Put(c.cacheID(a), file)
+}

pkg/api/core/types/artifact/cache_test.go

@@ -0,0 +1,90 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	types "github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/mudler/luet/pkg/api/core/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Cache", func() {
+	Context("CacheID", func() {
+		It("Get and retrieve files", func() {
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpdirartifact, err := ioutil.TempDir(os.TempDir(), "testartifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdirartifact) // clean up
+
+			err = ioutil.WriteFile(filepath.Join(tmpdirartifact, "foo"), []byte(string("foo")), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			a := NewPackageArtifact(filepath.Join(tmpdir, "foo.tar.gz"))
+			err = a.Compress(tmpdirartifact, 1)
+			Expect(err).ToNot(HaveOccurred())
+
+			cache := NewCache(tmpdir)
+
+			// Put an artifact in the cache and retrieve it later
+			// the artifact is NOT hashed so it is referenced just by the path in the cache
+			_, _, err = cache.Put(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			path, err := cache.Get(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			b := NewPackageArtifact(path)
+			ctx := types.NewContext()
+			err = b.Unpack(ctx, tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "foo"))).To(BeTrue())
+
+			bb, err := ioutil.ReadFile(filepath.Join(tmpdir, "foo"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(bb)).To(Equal("foo"))
+
+			// After the artifact is hashed, the fingerprint mutates so the cache doesn't see it hitting again
+			// the test we did above fails as we expect to.
+			a.Hash()
+			_, err = cache.Get(a)
+			Expect(err).To(HaveOccurred())
+
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "bar"}}
+			_, _, err = cache.Put(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			c := NewPackageArtifact(filepath.Join(tmpdir, "foo.tar.gz"))
+			c.Hash()
+			c.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "bar"}}
+			_, err = cache.Get(c)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})

pkg/api/core/types/artifact/checksum.go

@@ -24,6 +24,7 @@ import (
 	"hash"
 	"io"
 	"os"
+	"sort"
 
 	//	. "github.com/mudler/luet/pkg/logger"
 	"github.com/pkg/errors"
@@ -42,6 +43,18 @@ type HashOptions struct {
 	Type   HashImplementation
 }
 
+func (c Checksums) List() (res [][]string) {
+	keys := make([]string, 0)
+	for k, _ := range c {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		res = append(res, []string{k, c[k]})
+	}
+	return
+}
+
 // Generate generates all Checksums supported for the artifact
 func (c *Checksums) Generate(a *PackageArtifact) error {
 	return c.generateSHA256(a)

pkg/api/core/types/artifact/checksum_test.go

@@ -19,7 +19,7 @@ import (
 	"io/ioutil"
 	"os"
 
-	. "github.com/mudler/luet/pkg/compiler/types/artifact"
+	. "github.com/mudler/luet/pkg/api/core/types/artifact"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -40,13 +40,13 @@ var _ = Describe("Checksum", func() {
 			Expect(len(definitionsum)).To(Equal(0))
 			Expect(len(definitionsum2)).To(Equal(0))
 
-			err = buildsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/build.yaml"))
+			err = buildsum.Generate(NewPackageArtifact("../../../../../tests/fixtures/layers/alpine/build.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = definitionsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			err = definitionsum.Generate(NewPackageArtifact("../../../../../tests/fixtures/layers/alpine/definition.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = definitionsum2.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			err = definitionsum2.Generate(NewPackageArtifact("../../../../../tests/fixtures/layers/alpine/definition.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(buildsum)).To(Equal(1))

pkg/api/core/types/config.go

@@ -0,0 +1,472 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//             2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/mudler/luet/pkg/api/core/config"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	pkg "github.com/mudler/luet/pkg/package"
+	solver "github.com/mudler/luet/pkg/solver"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+var AvailableResolvers = strings.Join([]string{solver.QLearningResolverType}, " ")
+
+type LuetLoggingConfig struct {
+	// Path of the logfile
+	Path string `mapstructure:"path"`
+	// Enable/Disable logging to file
+	EnableLogFile bool `mapstructure:"enable_logfile"`
+	// Enable JSON format logging in file
+	JsonFormat bool `mapstructure:"json_format"`
+
+	// Log level
+	Level LogLevel `mapstructure:"level"`
+
+	// Enable emoji
+	EnableEmoji bool `mapstructure:"enable_emoji"`
+	// Enable/Disable color in logging
+	Color bool `mapstructure:"color"`
+}
+
+type LuetGeneralConfig struct {
+	SameOwner       bool `yaml:"same_owner,omitempty" mapstructure:"same_owner"`
+	Concurrency     int  `yaml:"concurrency,omitempty" mapstructure:"concurrency"`
+	Debug           bool `yaml:"debug,omitempty" mapstructure:"debug"`
+	ShowBuildOutput bool `yaml:"show_build_output,omitempty" mapstructure:"show_build_output"`
+	FatalWarns      bool `yaml:"fatal_warnings,omitempty" mapstructure:"fatal_warnings"`
+	HTTPTimeout     int  `yaml:"http_timeout,omitempty" mapstructure:"http_timeout"`
+}
+
+type LuetSolverOptions struct {
+	solver.Options `yaml:"options,omitempty"`
+	Type           string            `yaml:"type,omitempty" mapstructure:"type"`
+	LearnRate      float32           `yaml:"rate,omitempty" mapstructure:"rate"`
+	Discount       float32           `yaml:"discount,omitempty" mapstructure:"discount"`
+	MaxAttempts    int               `yaml:"max_attempts,omitempty" mapstructure:"max_attempts"`
+	Implementation solver.SolverType `yaml:"implementation,omitempty" mapstructure:"implementation"`
+}
+
+func (opts LuetSolverOptions) ResolverIsSet() bool {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		return true
+	default:
+		return false
+	}
+}
+
+func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		if opts.LearnRate != 0.0 {
+			return solver.NewQLearningResolver(opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+
+		}
+		return solver.SimpleQLearningSolver()
+	}
+
+	return &solver.Explainer{}
+}
+
+func (opts *LuetSolverOptions) CompactString() string {
+	return fmt.Sprintf("type: %s rate: %f, discount: %f, attempts: %d, initialobserved: %d",
+		opts.Type, opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+}
+
+type LuetSystemConfig struct {
+	DatabaseEngine string `yaml:"database_engine" mapstructure:"database_engine"`
+	DatabasePath   string `yaml:"database_path" mapstructure:"database_path"`
+	Rootfs         string `yaml:"rootfs" mapstructure:"rootfs"`
+	PkgsCachePath  string `yaml:"pkgs_cache_path" mapstructure:"pkgs_cache_path"`
+	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
+}
+
+func (s *LuetSystemConfig) SetRootFS(path string) error {
+	p, err := fileHelper.Rel2Abs(path)
+	if err != nil {
+		return err
+	}
+
+	s.Rootfs = p
+	return nil
+}
+
+func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
+	dbpath = filepath.Join(dbpath, "repos/"+name)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc *LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+	dbpath := filepath.Join(sc.Rootfs,
+		sc.DatabasePath)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+	var cachepath string
+	if sc.PkgsCachePath != "" {
+		cachepath = sc.PkgsCachePath
+	} else {
+		// Create dynamic cache for test suites
+		cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
+	}
+
+	if filepath.IsAbs(cachepath) {
+		ans = cachepath
+	} else {
+		ans = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
+	}
+
+	return
+}
+
+func (sc *LuetSystemConfig) GetRootFsAbs() (string, error) {
+	return filepath.Abs(sc.Rootfs)
+}
+
+type LuetKV struct {
+	Key   string `json:"key" yaml:"key" mapstructure:"key"`
+	Value string `json:"value" yaml:"value" mapstructure:"value"`
+}
+
+type LuetConfig struct {
+	Logging LuetLoggingConfig `yaml:"logging,omitempty" mapstructure:"logging"`
+	General LuetGeneralConfig `yaml:"general,omitempty" mapstructure:"general"`
+	System  LuetSystemConfig  `yaml:"system" mapstructure:"system"`
+	Solver  LuetSolverOptions `yaml:"solver,omitempty" mapstructure:"solver"`
+
+	RepositoriesConfDir  []string         `yaml:"repos_confdir,omitempty" mapstructure:"repos_confdir"`
+	ConfigProtectConfDir []string         `yaml:"config_protect_confdir,omitempty" mapstructure:"config_protect_confdir"`
+	ConfigProtectSkip    bool             `yaml:"config_protect_skip,omitempty" mapstructure:"config_protect_skip"`
+	ConfigFromHost       bool             `yaml:"config_from_host,omitempty" mapstructure:"config_from_host"`
+	SystemRepositories   LuetRepositories `yaml:"repositories,omitempty" mapstructure:"repositories"`
+
+	FinalizerEnvs []LuetKV `json:"finalizer_envs,omitempty" yaml:"finalizer_envs,omitempty" mapstructure:"finalizer_envs,omitempty"`
+
+	ConfigProtectConfFiles []config.ConfigProtectConfFile `yaml:"-" mapstructure:"-"`
+}
+
+func (c *LuetConfig) GetSystemDB() pkg.PackageDatabase {
+	switch c.GetSystem().DatabaseEngine {
+	case "boltdb":
+		return pkg.NewBoltDatabase(
+			filepath.Join(c.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+	default:
+		return pkg.NewInMemoryDatabase(true)
+	}
+}
+
+func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
+	c.SystemRepositories = append(c.SystemRepositories, r)
+}
+
+func (c *LuetConfig) GetFinalizerEnvsMap() map[string]string {
+	ans := make(map[string]string)
+
+	for _, kv := range c.FinalizerEnvs {
+		ans[kv.Key] = kv.Value
+	}
+	return ans
+}
+
+func (c *LuetConfig) SetFinalizerEnv(k, v string) {
+	keyPresent := false
+	envs := []LuetKV{}
+
+	for _, kv := range c.FinalizerEnvs {
+		if kv.Key == k {
+			keyPresent = true
+			envs = append(envs, LuetKV{Key: kv.Key, Value: v})
+		} else {
+			envs = append(envs, kv)
+		}
+	}
+	if !keyPresent {
+		envs = append(envs, LuetKV{Key: k, Value: v})
+	}
+
+	c.FinalizerEnvs = envs
+}
+
+func (c *LuetConfig) GetFinalizerEnvs() []string {
+	ans := []string{}
+	for _, kv := range c.FinalizerEnvs {
+		ans = append(ans, fmt.Sprintf("%s=%s", kv.Key, kv.Value))
+	}
+	return ans
+}
+
+func (c *LuetConfig) GetFinalizerEnv(k string) (string, error) {
+	keyNotPresent := true
+	ans := ""
+	for _, kv := range c.FinalizerEnvs {
+		if kv.Key == k {
+			keyNotPresent = false
+			ans = kv.Value
+		}
+	}
+
+	if keyNotPresent {
+		return "", errors.New("Finalizer key " + k + " not found")
+	}
+	return ans, nil
+}
+
+func (c *LuetConfig) GetLogging() *LuetLoggingConfig {
+	return &c.Logging
+}
+
+func (c *LuetConfig) GetGeneral() *LuetGeneralConfig {
+	return &c.General
+}
+
+func (c *LuetConfig) GetSystem() *LuetSystemConfig {
+	return &c.System
+}
+
+func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
+	return &c.Solver
+}
+
+func (c *LuetConfig) YAML() ([]byte, error) {
+	return yaml.Marshal(c)
+}
+
+func (c *LuetConfig) GetConfigProtectConfFiles() []config.ConfigProtectConfFile {
+	return c.ConfigProtectConfFiles
+}
+
+func (c *LuetConfig) AddConfigProtectConfFile(file *config.ConfigProtectConfFile) {
+	if c.ConfigProtectConfFiles == nil {
+		c.ConfigProtectConfFiles = []config.ConfigProtectConfFile{*file}
+	} else {
+		c.ConfigProtectConfFiles = append(c.ConfigProtectConfFiles, *file)
+	}
+}
+
+func (c *LuetConfig) LoadRepositories(ctx *Context) error {
+	var regexRepo = regexp.MustCompile(`.yml$|.yaml$`)
+	var err error
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, rdir := range c.RepositoriesConfDir {
+
+		rdir = filepath.Join(rootfs, rdir)
+
+		ctx.Debug("Parsing Repository Directory", rdir, "...")
+
+		files, err := ioutil.ReadDir(rdir)
+		if err != nil {
+			ctx.Debug("Skip dir", rdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexRepo.MatchString(file.Name()) {
+				ctx.Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(rdir, file.Name()))
+			if err != nil {
+				ctx.Warning("On read file", file.Name(), ":", err.Error())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := LoadRepository(content)
+			if err != nil {
+				ctx.Warning("On parse file", file.Name(), ":", err.Error())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Urls) == 0 || r.Type == "" {
+				ctx.Warning("Invalid repository ", file.Name())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddSystemRepository(*r)
+		}
+	}
+	return nil
+}
+
+func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
+	var ans *LuetRepository = nil
+
+	for idx, repo := range c.SystemRepositories {
+		if repo.Name == name {
+			ans = &c.SystemRepositories[idx]
+			break
+		}
+	}
+	if ans == nil {
+		return nil, errors.New("Repository " + name + " not found")
+	}
+
+	return ans, nil
+}
+
+func (c *LuetConfig) LoadConfigProtect(ctx *Context) error {
+	var regexConfs = regexp.MustCompile(`.yml$`)
+	var err error
+
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, cdir := range c.ConfigProtectConfDir {
+		cdir = filepath.Join(rootfs, cdir)
+
+		ctx.Debug("Parsing Config Protect Directory", cdir, "...")
+
+		files, err := ioutil.ReadDir(cdir)
+		if err != nil {
+			ctx.Debug("Skip dir", cdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexConfs.MatchString(file.Name()) {
+				ctx.Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(cdir, file.Name()))
+			if err != nil {
+				ctx.Warning("On read file", file.Name(), ":", err.Error())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := loadConfigProtectConFile(file.Name(), content)
+			if err != nil {
+				ctx.Warning("On parse file", file.Name(), ":", err.Error())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Directories) == 0 {
+				ctx.Warning("Invalid config protect file", file.Name())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddConfigProtectConfFile(r)
+		}
+	}
+	return nil
+
+}
+
+func loadConfigProtectConFile(filename string, data []byte) (*config.ConfigProtectConfFile, error) {
+	ans := config.NewConfigProtectConfFile(filename)
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}
+
+func (c *LuetLoggingConfig) SetLogLevel(s LogLevel) {
+	c.Level = s
+}
+
+func (c *LuetSystemConfig) InitTmpDir() error {
+	if !filepath.IsAbs(c.TmpDirBase) {
+		abs, err := fileHelper.Rel2Abs(c.TmpDirBase)
+		if err != nil {
+			return errors.Wrap(err, "while converting relative path to absolute path")
+		}
+		c.TmpDirBase = abs
+	}
+
+	if _, err := os.Stat(c.TmpDirBase); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(c.TmpDirBase, os.ModePerm)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c *LuetSystemConfig) CleanupTmpDir() error {
+	return os.RemoveAll(c.TmpDirBase)
+}
+
+func (c *LuetSystemConfig) TempDir(pattern string) (string, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return "", err
+	}
+	return ioutil.TempDir(c.TmpDirBase, pattern)
+}
+
+func (c *LuetSystemConfig) TempFile(pattern string) (*os.File, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.TempFile(c.TmpDirBase, pattern)
+}

pkg/api/core/types/config_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestAPITypes(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Types Suite")
+}

pkg/api/core/types/config_test.go

@@ -0,0 +1,89 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types_test
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	types "github.com/mudler/luet/pkg/api/core/types"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+	Context("Load Repository1", func() {
+		ctx := types.NewContext()
+
+		ctx.Config.RepositoriesConfDir = []string{
+			"../../../../tests/fixtures/repos.conf.d",
+		}
+		err := ctx.Config.LoadRepositories(ctx)
+
+		It("Check Load Repository 1", func() {
+			Expect(err).Should(BeNil())
+			Expect(len(ctx.Config.SystemRepositories)).Should(Equal(2))
+			Expect(ctx.Config.SystemRepositories[0].Name).Should(Equal("test1"))
+			Expect(ctx.Config.SystemRepositories[0].Priority).Should(Equal(999))
+			Expect(ctx.Config.SystemRepositories[0].Type).Should(Equal("disk"))
+			Expect(len(ctx.Config.SystemRepositories[0].Urls)).Should(Equal(1))
+			Expect(ctx.Config.SystemRepositories[0].Urls[0]).Should(Equal("tests/repos/test1"))
+		})
+
+		It("Chec Load Repository 2", func() {
+			Expect(err).Should(BeNil())
+			Expect(len(ctx.Config.SystemRepositories)).Should(Equal(2))
+			Expect(ctx.Config.SystemRepositories[1].Name).Should(Equal("test2"))
+			Expect(ctx.Config.SystemRepositories[1].Priority).Should(Equal(1000))
+			Expect(ctx.Config.SystemRepositories[1].Type).Should(Equal("disk"))
+			Expect(len(ctx.Config.SystemRepositories[1].Urls)).Should(Equal(1))
+			Expect(ctx.Config.SystemRepositories[1].Urls[0]).Should(Equal("tests/repos/test2"))
+		})
+	})
+	Context("Simple temporary directory creation", func() {
+
+		It("Create Temporary directory", func() {
+			ctx := types.NewContext()
+
+			ctx.Config.GetSystem().TmpDirBase = os.TempDir() + "/tmpluet"
+
+			tmpDir, err := ctx.Config.GetSystem().TempDir("test1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.HasPrefix(tmpDir, filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
+			Expect(fileHelper.Exists(tmpDir)).To(BeTrue())
+
+			defer os.RemoveAll(tmpDir)
+		})
+
+		It("Create Temporary file", func() {
+			ctx := types.NewContext()
+
+			ctx.Config.GetSystem().TmpDirBase = os.TempDir() + "/tmpluet"
+
+			tmpFile, err := ctx.Config.GetSystem().TempFile("testfile1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.HasPrefix(tmpFile.Name(), filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
+			Expect(fileHelper.Exists(tmpFile.Name())).To(BeTrue())
+
+			defer os.Remove(tmpFile.Name())
+		})
+
+	})
+
+})

pkg/api/core/types/context.go

@@ -0,0 +1,416 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/kyokomi/emoji"
+	"github.com/mudler/luet/pkg/helpers/terminal"
+	"github.com/pkg/errors"
+	"github.com/pterm/pterm"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"golang.org/x/term"
+)
+
+const (
+	ErrorLevel   LogLevel = "error"
+	WarningLevel LogLevel = "warning"
+	InfoLevel    LogLevel = "info"
+	SuccessLevel LogLevel = "success"
+	FatalLevel   LogLevel = "fatal"
+)
+
+type Context struct {
+	context.Context
+	Config     *LuetConfig
+	IsTerminal bool
+	NoSpinner  bool
+
+	s           *pterm.SpinnerPrinter
+	spinnerLock *sync.Mutex
+	z           *zap.Logger
+	AreaPrinter *pterm.AreaPrinter
+	ProgressBar *pterm.ProgressbarPrinter
+}
+
+func NewContext() *Context {
+	return &Context{
+		spinnerLock: &sync.Mutex{},
+		IsTerminal:  terminal.IsTerminal(os.Stdout),
+		Config: &LuetConfig{
+			ConfigFromHost: true,
+			Logging:        LuetLoggingConfig{},
+			General:        LuetGeneralConfig{},
+			System: LuetSystemConfig{
+				DatabasePath: filepath.Join("var", "db", "packages"),
+				TmpDirBase:   filepath.Join(os.TempDir(), "tmpluet")},
+			Solver: LuetSolverOptions{},
+		},
+		s: pterm.DefaultSpinner.WithShowTimer(false).WithRemoveWhenDone(true),
+	}
+}
+
+func (c *Context) Copy() *Context {
+
+	configCopy := *c.Config
+	configCopy.System = *c.Config.GetSystem()
+	configCopy.General = *c.Config.GetGeneral()
+	configCopy.Logging = *c.Config.GetLogging()
+
+	ctx := *c
+	ctxCopy := &ctx
+	ctxCopy.Config = &configCopy
+
+	return ctxCopy
+}
+
+// GetTerminalSize returns the width and the height of the active terminal.
+func (c *Context) GetTerminalSize() (width, height int, err error) {
+	w, h, err := term.GetSize(int(os.Stdout.Fd()))
+	if w <= 0 {
+		w = 0
+	}
+	if h <= 0 {
+		h = 0
+	}
+	if err != nil {
+		err = errors.New("size not detectable")
+	}
+	return w, h, err
+}
+
+func (c *Context) Init() (err error) {
+	if c.IsTerminal {
+		if !c.Config.Logging.Color {
+			c.Debug("Disabling colors")
+			c.NoColor()
+		}
+	} else {
+		c.Debug("Not a terminal, disabling colors")
+		c.NoColor()
+	}
+
+	c.Debug("Colors", c.Config.GetLogging().Color)
+	c.Debug("Logging level", c.Config.GetLogging().Level)
+	c.Debug("Debug mode", c.Config.GetGeneral().Debug)
+
+	if c.Config.GetLogging().EnableLogFile && c.Config.GetLogging().Path != "" {
+		// Init zap logger
+		err = c.InitZap()
+		if err != nil {
+			return
+		}
+	}
+
+	// Load repositories
+	err = c.Config.LoadRepositories(c)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (c *Context) NoColor() {
+	pterm.DisableColor()
+}
+
+func (c *Context) Ask() bool {
+	var input string
+
+	c.Info("Do you want to continue with this operation? [y/N]: ")
+	_, err := fmt.Scanln(&input)
+	if err != nil {
+		return false
+	}
+	input = strings.ToLower(input)
+
+	if input == "y" || input == "yes" {
+		return true
+	}
+	return false
+}
+
+func (c *Context) InitZap() error {
+	var err error
+	if c.z == nil {
+		// TODO: test permission for open logfile.
+		cfg := zap.NewProductionConfig()
+		cfg.OutputPaths = []string{c.Config.GetLogging().Path}
+		cfg.Level = c.Config.GetLogging().Level.ZapLevel()
+		cfg.ErrorOutputPaths = []string{}
+		if c.Config.GetLogging().JsonFormat {
+			cfg.Encoding = "json"
+		} else {
+			cfg.Encoding = "console"
+		}
+		cfg.DisableCaller = true
+		cfg.DisableStacktrace = true
+		cfg.EncoderConfig.TimeKey = "time"
+		cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+
+		c.z, err = cfg.Build()
+		if err != nil {
+			fmt.Fprint(os.Stderr, "Error on initialize file logger: "+err.Error()+"\n")
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Spinner starts the spinner
+func (c *Context) Spinner() {
+	if !c.IsTerminal || c.NoSpinner {
+		return
+	}
+
+	c.spinnerLock.Lock()
+	defer c.spinnerLock.Unlock()
+	var confLevel int
+	if c.Config.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = c.Config.GetLogging().Level.ToNumber()
+	}
+	if 2 > confLevel {
+		return
+	}
+
+	if !c.s.IsActive {
+		c.s, _ = c.s.Start()
+	}
+}
+
+func (c *Context) Screen(text string) {
+	pterm.DefaultHeader.WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(2).Println(text)
+	//pterm.DefaultCenter.Print(pterm.DefaultHeader.WithFullWidth().WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(10).Sprint(text))
+}
+
+func (c *Context) SpinnerText(suffix, prefix string) {
+	if !c.IsTerminal || c.NoSpinner {
+		return
+	}
+
+	c.spinnerLock.Lock()
+	defer c.spinnerLock.Unlock()
+	if c.Config.GetGeneral().Debug {
+		fmt.Printf("%s %s\n",
+			suffix, prefix,
+		)
+	} else {
+		c.s.UpdateText(suffix + prefix)
+	}
+}
+
+func (c *Context) SpinnerStop() {
+	if !c.IsTerminal {
+		return
+	}
+
+	c.spinnerLock.Lock()
+	defer c.spinnerLock.Unlock()
+	var confLevel int
+	if c.Config.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = c.Config.GetLogging().Level.ToNumber()
+	}
+	if 2 > confLevel {
+		return
+	}
+	if c.s != nil {
+		c.s.Success()
+	}
+}
+
+func (c *Context) log2File(level LogLevel, msg string) {
+	switch level {
+	case FatalLevel:
+		c.z.Fatal(msg)
+	case ErrorLevel:
+		c.z.Error(msg)
+	case WarningLevel:
+		c.z.Warn(msg)
+	case InfoLevel, SuccessLevel:
+		c.z.Info(msg)
+	default:
+		c.z.Debug(msg)
+	}
+}
+
+func (c *Context) Msg(level LogLevel, ln bool, msg ...interface{}) {
+	var message string
+	var confLevel, msgLevel int
+
+	if c.Config.GetGeneral().Debug {
+		confLevel = 3
+		pterm.EnableDebugMessages()
+	} else {
+		confLevel = c.Config.GetLogging().Level.ToNumber()
+	}
+	msgLevel = level.ToNumber()
+
+	if msgLevel > confLevel {
+		return
+	}
+
+	for _, m := range msg {
+		message += " " + fmt.Sprintf("%v", m)
+	}
+
+	// Color message
+	levelMsg := message
+
+	if c.Config.GetLogging().Color {
+		switch level {
+		case WarningLevel:
+			levelMsg = pterm.LightYellow(":construction: warning" + message)
+		case InfoLevel:
+			levelMsg = message
+		case SuccessLevel:
+			levelMsg = pterm.LightGreen(message)
+		case ErrorLevel:
+			levelMsg = pterm.Red(message)
+		default:
+			levelMsg = pterm.Blue(message)
+		}
+	}
+
+	// Strip emoji if needed
+	if c.Config.GetLogging().EnableEmoji && c.IsTerminal {
+		levelMsg = emoji.Sprint(levelMsg)
+	} else {
+		re := regexp.MustCompile(`[:][\w]+[:]`)
+		levelMsg = re.ReplaceAllString(levelMsg, "")
+	}
+
+	if c.z != nil {
+		c.log2File(level, message)
+	}
+
+	// Print the message based on the level
+	switch level {
+	case SuccessLevel:
+		if ln {
+			pterm.Success.Println(levelMsg)
+		} else {
+			pterm.Success.Print(levelMsg)
+		}
+	case InfoLevel:
+		if ln {
+			pterm.Info.Println(levelMsg)
+		} else {
+			pterm.Info.Print(levelMsg)
+		}
+	case WarningLevel:
+		if ln {
+			pterm.Warning.Println(levelMsg)
+		} else {
+			pterm.Warning.Print(levelMsg)
+		}
+	case ErrorLevel:
+		if ln {
+			pterm.Error.Println(levelMsg)
+		} else {
+			pterm.Error.Print(levelMsg)
+		}
+	case FatalLevel:
+		if ln {
+			pterm.Fatal.Println(levelMsg)
+		} else {
+			pterm.Fatal.Print(levelMsg)
+		}
+	default:
+		if ln {
+			pterm.Debug.Println(levelMsg)
+		} else {
+			pterm.Debug.Print(levelMsg)
+		}
+	}
+}
+
+func (c *Context) Warning(mess ...interface{}) {
+	c.Msg("warning", true, mess...)
+	if c.Config.GetGeneral().FatalWarns {
+		os.Exit(2)
+	}
+}
+
+func (c *Context) Debug(mess ...interface{}) {
+	pc, file, line, ok := runtime.Caller(1)
+	if ok {
+		mess = append([]interface{}{fmt.Sprintf("(%s:#%d:%v)",
+			path.Base(file), line, runtime.FuncForPC(pc).Name())}, mess...)
+	}
+	c.Msg("debug", true, mess...)
+}
+
+func (c *Context) Info(mess ...interface{}) {
+	c.Msg("info", true, mess...)
+}
+
+func (c *Context) Success(mess ...interface{}) {
+	c.Msg("success", true, mess...)
+}
+
+func (c *Context) Error(mess ...interface{}) {
+	c.Msg("error", true, mess...)
+}
+
+func (c *Context) Fatal(mess ...interface{}) {
+	c.Error(mess...)
+	os.Exit(1)
+}
+
+type LogLevel string
+
+func (level LogLevel) ToNumber() int {
+	switch level {
+	case ErrorLevel, FatalLevel:
+		return 0
+	case WarningLevel:
+		return 1
+	case InfoLevel, SuccessLevel:
+		return 2
+	default: // debug
+		return 3
+	}
+}
+
+func (level LogLevel) ZapLevel() zap.AtomicLevel {
+	switch level {
+	case FatalLevel:
+		return zap.NewAtomicLevelAt(zap.FatalLevel)
+	case ErrorLevel:
+		return zap.NewAtomicLevelAt(zap.ErrorLevel)
+	case WarningLevel:
+		return zap.NewAtomicLevelAt(zap.WarnLevel)
+	case InfoLevel, SuccessLevel:
+		return zap.NewAtomicLevelAt(zap.InfoLevel)
+	default:
+		return zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+}

pkg/api/core/types/repository.go

@@ -0,0 +1,96 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types
+
+import (
+	"fmt"
+	"runtime"
+
+	"gopkg.in/yaml.v2"
+)
+
+type LuetRepository struct {
+	Name           string            `json:"name" yaml:"name" mapstructure:"name"`
+	Description    string            `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
+	Urls           []string          `json:"urls" yaml:"urls" mapstructure:"urls"`
+	Type           string            `json:"type" yaml:"type" mapstructure:"type"`
+	Mode           string            `json:"mode,omitempty" yaml:"mode,omitempty" mapstructure:"mode,omitempty"`
+	Priority       int               `json:"priority,omitempty" yaml:"priority,omitempty" mapstructure:"priority"`
+	Enable         bool              `json:"enable" yaml:"enable" mapstructure:"enable"`
+	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
+	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
+	TreePath       string            `json:"treepath,omitempty" yaml:"treepath,omitempty" mapstructure:"treepath"`
+	MetaPath       string            `json:"metapath,omitempty" yaml:"metapath,omitempty" mapstructure:"metapath"`
+	Verify         bool              `json:"verify,omitempty" yaml:"verify,omitempty" mapstructure:"verify"`
+	Arch           string            `json:"arch,omitempty" yaml:"arch,omitempty" mapstructure:"arch"`
+
+	ReferenceID string `json:"reference,omitempty" yaml:"reference,omitempty" mapstructure:"reference"`
+
+	// Incremented value that identify revision of the repository in a user-friendly way.
+	Revision int `json:"revision,omitempty" yaml:"-" mapstructure:"-"`
+	// Epoch time in seconds
+	LastUpdate string `json:"last_update,omitempty" yaml:"-" mapstructure:"-"`
+}
+
+func (r *LuetRepository) String() string {
+	return fmt.Sprintf("[%s] prio: %d, type: %s, enable: %t, cached: %t",
+		r.Name, r.Priority, r.Type, r.Enable, r.Cached)
+}
+
+// Enabled returns a boolean indicating if the repository should be considered enabled or not
+func (r *LuetRepository) Enabled() bool {
+	return r.Arch != "" && r.Arch == runtime.GOARCH && !r.Enable || r.Enable
+}
+
+type LuetRepositories []LuetRepository
+
+func (l LuetRepositories) Enabled() (res LuetRepositories) {
+	for _, r := range l {
+		if r.Enabled() {
+			res = append(res, r)
+		}
+	}
+	return
+}
+
+func NewLuetRepository(name, t, descr string, urls []string, priority int, enable, cached bool) *LuetRepository {
+	return &LuetRepository{
+		Name:           name,
+		Description:    descr,
+		Urls:           urls,
+		Type:           t,
+		Priority:       priority,
+		Enable:         enable,
+		Cached:         cached,
+		Authentication: make(map[string]string),
+	}
+}
+
+func NewEmptyLuetRepository() *LuetRepository {
+	return &LuetRepository{
+		Priority:       9999,
+		Authentication: make(map[string]string),
+	}
+}
+
+func LoadRepository(data []byte) (*LuetRepository, error) {
+	ans := NewEmptyLuetRepository()
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}

pkg/api/core/types/repository_test.go

@@ -0,0 +1,49 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types_test
+
+import (
+	"runtime"
+
+	types "github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Types", func() {
+	Context("Repository detects underlying arch", func() {
+		It("is enabled if arch is matching", func() {
+			r := types.LuetRepository{Arch: runtime.GOARCH}
+			Expect(r.Enabled()).To(BeTrue())
+		})
+		It("is disabled if arch is NOT matching", func() {
+			r := types.LuetRepository{Arch: "foo"}
+			Expect(r.Enabled()).To(BeFalse())
+		})
+		It("is enabled if arch is NOT matching and enabled is true", func() {
+			r := types.LuetRepository{Arch: "foo", Enable: true}
+			Expect(r.Enabled()).To(BeTrue())
+		})
+		It("enabled is true", func() {
+			r := types.LuetRepository{Enable: true}
+			Expect(r.Enabled()).To(BeTrue())
+		})
+		It("enabled is false", func() {
+			r := types.LuetRepository{Enable: false}
+			Expect(r.Enabled()).To(BeFalse())
+		})
+	})
+})

pkg/box/exec.go

@@ -23,9 +23,9 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/pkg/errors"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
-	helpers "github.com/mudler/luet/pkg/helpers"
+	"github.com/pkg/errors"
 )
 
 type Box interface {
@@ -107,7 +107,7 @@ func (b *DefaultBox) Exec() error {
 
 func (b *DefaultBox) Run() error {
 
-	if !helpers.Exists(b.Root) {
+	if !fileHelper.Exists(b.Root) {
 		return errors.New(b.Root + " does not exist")
 	}
 

pkg/bus/events.go

@@ -2,6 +2,7 @@ package bus
 
 import (
 	"github.com/mudler/go-pluggable"
+	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -44,24 +45,59 @@ var (
 	EventRepositoryPreBuild pluggable.EventType = "repository.pre.build"
 	// EventRepositoryPostBuild is the event fired after a repository was built
 	EventRepositoryPostBuild pluggable.EventType = "repository.post.build"
+
+	// Image unpack
+
+	// EventImagePreUnPack is the event fired before unpacking an image to a local dir
+	EventImagePreUnPack pluggable.EventType = "image.pre.unpack"
+	// EventImagePostUnPack is the event fired after unpacking an image to a local dir
+	EventImagePostUnPack pluggable.EventType = "image.post.unpack"
 )
 
 // Manager is the bus instance manager, which subscribes plugins to events emitted by Luet
-var Manager *pluggable.Manager = pluggable.NewManager(
-	[]pluggable.EventType{
-		EventPackageInstall,
-		EventPackageUnInstall,
-		EventPackagePreBuild,
-		EventPackagePreBuildArtifact,
-		EventPackagePostBuildArtifact,
-		EventPackagePostBuild,
-		EventRepositoryPreBuild,
-		EventRepositoryPostBuild,
-		EventImagePreBuild,
-		EventImagePrePull,
-		EventImagePrePush,
-		EventImagePostBuild,
-		EventImagePostPull,
-		EventImagePostPush,
-	},
-)
+var Manager *Bus = &Bus{
+	Manager: pluggable.NewManager(
+		[]pluggable.EventType{
+			EventPackageInstall,
+			EventPackageUnInstall,
+			EventPackagePreBuild,
+			EventPackagePreBuildArtifact,
+			EventPackagePostBuildArtifact,
+			EventPackagePostBuild,
+			EventRepositoryPreBuild,
+			EventRepositoryPostBuild,
+			EventImagePreBuild,
+			EventImagePrePull,
+			EventImagePrePush,
+			EventImagePostBuild,
+			EventImagePostPull,
+			EventImagePostPush,
+			EventImagePreUnPack,
+			EventImagePostUnPack,
+		},
+	),
+}
+
+type Bus struct {
+	*pluggable.Manager
+}
+
+func (b *Bus) Initialize(plugin ...string) {
+	b.Manager.Load(plugin...).Register()
+
+	for _, e := range b.Manager.Events {
+		b.Manager.Response(e, func(p *pluggable.Plugin, r *pluggable.EventResponse) {
+			if r.Errored() {
+				logrus.Fatal("Plugin", p.Name, "at", p.Executable, "Error", r.Error)
+			}
+			logrus.Debug(
+				"plugin_event",
+				"received from",
+				p.Name,
+				"at",
+				p.Executable,
+				r,
+			)
+		})
+	}
+}

pkg/compiler/backend.go

@@ -7,23 +7,21 @@ import (
 	"path/filepath"
 	"strings"
 
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/api/core/types"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 
 	"github.com/mudler/luet/pkg/compiler/backend"
-	"github.com/mudler/luet/pkg/config"
 	"github.com/pkg/errors"
-
-	. "github.com/mudler/luet/pkg/logger"
 )
 
-func NewBackend(s string) (CompilerBackend, error) {
+func NewBackend(ctx *types.Context, s string) (CompilerBackend, error) {
 	var compilerBackend CompilerBackend
 
 	switch s {
 	case backend.ImgBackend:
-		compilerBackend = backend.NewSimpleImgBackend()
+		compilerBackend = backend.NewSimpleImgBackend(ctx)
 	case backend.DockerBackend:
-		compilerBackend = backend.NewSimpleDockerBackend()
+		compilerBackend = backend.NewSimpleDockerBackend(ctx)
 	default:
 		return nil, errors.New("invalid backend. Unsupported")
 	}
@@ -73,11 +71,11 @@ type CompilerBackend interface {
 //     }
 //   }
 // ]
-func GenerateChanges(b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) {
+func GenerateChanges(ctx *types.Context, b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) {
 
 	res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
 
-	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
+	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
 	if err != nil {
 		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
@@ -99,7 +97,7 @@ func GenerateChanges(b CompilerBackend, fromImage, toImage backend.Options) ([]a
 		ImageName:   fromImage.ImageName,
 		Destination: srcRootFS,
 	}
-	Debug("Extracting source image", fromImage.ImageName)
+	ctx.Debug("Extracting source image", fromImage.ImageName)
 	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
 	if err != nil {
 		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
@@ -109,7 +107,7 @@ func GenerateChanges(b CompilerBackend, fromImage, toImage backend.Options) ([]a
 		ImageName:   toImage.ImageName,
 		Destination: dstRootFS,
 	}
-	Debug("Extracting destination image", toImage.ImageName)
+	ctx.Debug("Extracting destination image", toImage.ImageName)
 	err = b.ExtractRootfs(dstImageExtract, false)
 	if err != nil {
 		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
@@ -182,10 +180,10 @@ func GenerateChanges(b CompilerBackend, fromImage, toImage backend.Options) ([]a
 
 	diffs := []artifact.ArtifactLayer{res}
 
-	if config.LuetCfg.GetGeneral().Debug {
+	if ctx.Config.GetGeneral().Debug {
 		summary := ComputeArtifactLayerSummary(diffs)
 		for _, l := range summary.Layers {
-			Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
+			ctx.Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
 				l.FromImage, l.ToImage,
 				l.AddFiles, l.AddSizes,
 				l.DelFiles, l.DelSizes,

pkg/compiler/backend/backend_suite_test.go

@@ -18,15 +18,11 @@ package backend_test
 import (
 	"testing"
 
-	. "github.com/mudler/luet/cmd"
-	config "github.com/mudler/luet/pkg/config"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
-	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Backend Suite")
 }

pkg/compiler/backend/common.go

@@ -18,8 +18,7 @@ package backend
 import (
 	"os/exec"
 
-	"github.com/mudler/luet/pkg/config"
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/pkg/api/core/types"
 
 	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/pkg/errors"
@@ -44,17 +43,17 @@ type Options struct {
 	BackendArgs    []string
 }
 
-func runCommand(cmd *exec.Cmd) error {
+func runCommand(ctx *types.Context, cmd *exec.Cmd) error {
 	output := ""
-	buffered := !config.LuetCfg.GetGeneral().ShowBuildOutput
-	writer := NewBackendWriter(buffered)
+	buffered := !ctx.Config.GetGeneral().ShowBuildOutput
+	writer := NewBackendWriter(buffered, ctx)
 
 	cmd.Stdout = writer
 	cmd.Stderr = writer
 
 	if buffered {
-		Spinner(22)
-		defer SpinnerStop()
+		ctx.Spinner()
+		defer ctx.SpinnerStop()
 	}
 
 	err := cmd.Start()

pkg/compiler/backend/simpledocker.go

@@ -23,63 +23,66 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/mudler/luet/pkg/api/core/types"
 	bus "github.com/mudler/luet/pkg/bus"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	capi "github.com/mudler/docker-companion/api"
 
 	"github.com/mudler/luet/pkg/helpers"
-	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/pkg/errors"
 )
 
-type SimpleDocker struct{}
+type SimpleDocker struct {
+	ctx *types.Context
+}
 
-func NewSimpleDockerBackend() *SimpleDocker {
-	return &SimpleDocker{}
+func NewSimpleDockerBackend(ctx *types.Context) *SimpleDocker {
+	return &SimpleDocker{ctx: ctx}
 }
 
 // TODO: Missing still: labels, and build args expansion
-func (*SimpleDocker) BuildImage(opts Options) error {
+func (s *SimpleDocker) BuildImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePreBuild, opts)
 
 	buildarg := genBuildCommand(opts)
-	Info(":whale2: Building image " + name)
+	s.ctx.Info(":whale2: Building image " + name)
 	cmd := exec.Command("docker", buildarg...)
 	cmd.Dir = opts.SourcePath
-	err := runCommand(cmd)
+	err := runCommand(s.ctx, cmd)
 	if err != nil {
 		return err
 	}
 
-	Info(":whale: Building image " + name + " done")
+	s.ctx.Success(":whale: Building image " + name + " done")
 
 	bus.Manager.Publish(bus.EventImagePostBuild, opts)
 
 	return nil
 }
 
-func (*SimpleDocker) CopyImage(src, dst string) error {
-	Debug(":whale: Tagging image:", src, "->", dst)
+func (s *SimpleDocker) CopyImage(src, dst string) error {
+	s.ctx.Debug(":whale: Tagging image:", src, "->", dst)
 	cmd := exec.Command("docker", "tag", src, dst)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed tagging image: "+string(out))
 	}
-	Info(":whale: Tagged image:", src, "->", dst)
+	s.ctx.Success(":whale: Tagged image:", src, "->", dst)
 	return nil
 }
 
-func (*SimpleDocker) DownloadImage(opts Options) error {
+func (s *SimpleDocker) DownloadImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePull, opts)
 
 	buildarg := []string{"pull", name}
-	Debug(":whale: Downloading image " + name)
+	s.ctx.Debug(":whale: Downloading image " + name)
 
-	Spinner(22)
-	defer SpinnerStop()
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 
 	cmd := exec.Command("docker", buildarg...)
 	out, err := cmd.CombinedOutput()
@@ -87,20 +90,20 @@ func (*SimpleDocker) DownloadImage(opts Options) error {
 		return errors.Wrap(err, "Failed pulling image: "+string(out))
 	}
 
-	Info(":whale: Downloaded image:", name)
+	s.ctx.Success(":whale: Downloaded image:", name)
 	bus.Manager.Publish(bus.EventImagePostPull, opts)
 
 	return nil
 }
 
-func (*SimpleDocker) ImageExists(imagename string) bool {
+func (s *SimpleDocker) ImageExists(imagename string) bool {
 	buildarg := []string{"inspect", "--type=image", imagename}
-	Debug(":whale: Checking existance of docker image: " + imagename)
+	s.ctx.Debug(":whale: Checking existance of docker image: " + imagename)
 	cmd := exec.Command("docker", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		Debug("Image not present")
-		Debug(string(out))
+		s.ctx.Debug("Image not present")
+		s.ctx.Debug(string(out))
 		return false
 	}
 	return true
@@ -110,31 +113,31 @@ func (*SimpleDocker) ImageAvailable(imagename string) bool {
 	return imageAvailable(imagename)
 }
 
-func (*SimpleDocker) RemoveImage(opts Options) error {
+func (s *SimpleDocker) RemoveImage(opts Options) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
 	out, err := exec.Command("docker", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed removing image: "+string(out))
 	}
-	Info(":whale: Removed image:", name)
+	s.ctx.Success(":whale: Removed image:", name)
 	//Info(string(out))
 	return nil
 }
 
-func (*SimpleDocker) Push(opts Options) error {
+func (s *SimpleDocker) Push(opts Options) error {
 	name := opts.ImageName
 	pusharg := []string{"push", name}
 	bus.Manager.Publish(bus.EventImagePrePush, opts)
 
-	Spinner(22)
-	defer SpinnerStop()
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 
 	out, err := exec.Command("docker", pusharg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed pushing image: "+string(out))
 	}
-	Info(":whale: Pushed image:", name)
+	s.ctx.Success(":whale: Pushed image:", name)
 	bus.Manager.Publish(bus.EventImagePostPush, opts)
 
 	//Info(string(out))
@@ -154,22 +157,22 @@ func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error {
 	return nil
 }
 
-func (*SimpleDocker) ExportImage(opts Options) error {
+func (s *SimpleDocker) ExportImage(opts Options) error {
 	name := opts.ImageName
 	path := opts.Destination
 
 	buildarg := []string{"save", name, "-o", path}
-	Debug(":whale: Saving image " + name)
+	s.ctx.Debug(":whale: Saving image " + name)
 
-	Spinner(22)
-	defer SpinnerStop()
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 
 	out, err := exec.Command("docker", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed exporting image: "+string(out))
 	}
 
-	Debug(":whale: Exported image:", name)
+	s.ctx.Debug(":whale: Exported image:", name)
 	return nil
 }
 
@@ -181,6 +184,12 @@ func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
 	name := opts.ImageName
 	dst := opts.Destination
 
+	if !b.ImageExists(name) {
+		if err := b.DownloadImage(opts); err != nil {
+			return errors.Wrap(err, "failed pulling image "+name+" during extraction")
+		}
+	}
+
 	tempexport, err := ioutil.TempDir(dst, "tmprootfs")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
@@ -189,15 +198,13 @@ func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
 
 	imageExport := filepath.Join(tempexport, "image.tar")
 
-	Spinner(22)
-	defer SpinnerStop()
+	b.ctx.Spinner()
+	defer b.ctx.SpinnerStop()
 
 	if err := b.ExportImage(Options{ImageName: name, Destination: imageExport}); err != nil {
 		return errors.Wrap(err, "failed while extracting rootfs for "+name)
 	}
 
-	SpinnerStop()
-
 	src := imageExport
 
 	if src == "" && opts.ImageName != "" {
@@ -231,7 +238,7 @@ func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
 		return errors.Wrap(err, "Error met while unpacking rootfs")
 	}
 
-	manifest, err := helpers.Read(filepath.Join(rootfs, "manifest.json"))
+	manifest, err := fileHelper.Read(filepath.Join(rootfs, "manifest.json"))
 	if err != nil {
 		return errors.Wrap(err, "Error met while reading image manifest")
 	}

pkg/compiler/backend/simpledocker_test.go

@@ -16,17 +16,18 @@
 package backend_test
 
 import (
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
-	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
-	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -35,6 +36,7 @@ import (
 
 var _ = Describe("Docker backend", func() {
 	Context("Simple Docker backend satisfies main interface functionalities", func() {
+		ctx := types.NewContext()
 		It("Builds and generate tars", func() {
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
@@ -60,7 +62,7 @@ var _ = Describe("Docker backend", func() {
 
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -69,7 +71,7 @@ WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin`))
-			b := NewSimpleDockerBackend()
+			b := NewSimpleDockerBackend(ctx)
 			opts := backend.Options{
 				ImageName:      "luet/base",
 				SourcePath:     tmpdir,
@@ -79,11 +81,11 @@ ENV PACKAGE_CATEGORY=app-admin`))
 
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
@@ -103,7 +105,7 @@ RUN echo bar > /test2`))
 
 			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
 			artifacts := []artifact.ArtifactNode{{
 				Name: "/luetbuild/LuetDockerfile",
@@ -115,7 +117,7 @@ RUN echo bar > /test2`))
 			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4})
 			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4})
 
-			Expect(compiler.GenerateChanges(b, opts, opts2)).To(Equal(
+			Expect(compiler.GenerateChanges(ctx, b, opts, opts2)).To(Equal(
 				[]artifact.ArtifactLayer{{
 					FromImage: "luet/base",
 					ToImage:   "test",
@@ -132,12 +134,12 @@ RUN echo bar > /test2`))
 			}
 
 			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
 			Expect(b.ImageExists(opts2.ImageName)).To(BeFalse())
 		})
 
 		It("Detects available images", func() {
-			b := NewSimpleDockerBackend()
+			b := NewSimpleDockerBackend(ctx)
 			Expect(b.ImageAvailable("quay.io/mocaccino/extra")).To(BeTrue())
 			Expect(b.ImageAvailable("ubuntu:20.10")).To(BeTrue())
 			Expect(b.ImageAvailable("igjo5ijgo25nho52")).To(BeFalse())

pkg/compiler/backend/simpleimg.go

@@ -20,64 +20,65 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/mudler/luet/pkg/api/core/types"
 	bus "github.com/mudler/luet/pkg/bus"
 
-	. "github.com/mudler/luet/pkg/logger"
-
 	"github.com/pkg/errors"
 )
 
-type SimpleImg struct{}
+type SimpleImg struct {
+	ctx *types.Context
+}
 
-func NewSimpleImgBackend() *SimpleImg {
-	return &SimpleImg{}
+func NewSimpleImgBackend(ctx *types.Context) *SimpleImg {
+	return &SimpleImg{ctx: ctx}
 }
 
 // TODO: Missing still: labels, and build args expansion
-func (*SimpleImg) BuildImage(opts Options) error {
+func (s *SimpleImg) BuildImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePreBuild, opts)
 
 	buildarg := genBuildCommand(opts)
 
-	Info(":tea: Building image " + name)
+	s.ctx.Info(":tea: Building image " + name)
 
 	cmd := exec.Command("img", buildarg...)
 	cmd.Dir = opts.SourcePath
-	err := runCommand(cmd)
+	err := runCommand(s.ctx, cmd)
 	if err != nil {
 		return err
 	}
 	bus.Manager.Publish(bus.EventImagePostBuild, opts)
 
-	Info(":tea: Building image " + name + " done")
+	s.ctx.Info(":tea: Building image " + name + " done")
 
 	return nil
 }
 
-func (*SimpleImg) RemoveImage(opts Options) error {
+func (s *SimpleImg) RemoveImage(opts Options) error {
 	name := opts.ImageName
 	buildarg := []string{"rm", name}
-	Spinner(22)
-	defer SpinnerStop()
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed removing image: "+string(out))
 	}
 
-	Info(":tea: Image " + name + " removed")
+	s.ctx.Info(":tea: Image " + name + " removed")
 	return nil
 }
 
-func (*SimpleImg) DownloadImage(opts Options) error {
+func (s *SimpleImg) DownloadImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePull, opts)
 
 	buildarg := []string{"pull", name}
-	Debug(":tea: Downloading image " + name)
+	s.ctx.Debug(":tea: Downloading image " + name)
 
-	Spinner(22)
-	defer SpinnerStop()
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 
 	cmd := exec.Command("img", buildarg...)
 	out, err := cmd.CombinedOutput()
@@ -85,27 +86,27 @@ func (*SimpleImg) DownloadImage(opts Options) error {
 		return errors.Wrap(err, "Failed downloading image: "+string(out))
 	}
 
-	Info(":tea: Image " + name + " downloaded")
+	s.ctx.Info(":tea: Image " + name + " downloaded")
 	bus.Manager.Publish(bus.EventImagePostPull, opts)
 
 	return nil
 }
-func (*SimpleImg) CopyImage(src, dst string) error {
-	Spinner(22)
-	defer SpinnerStop()
+func (s *SimpleImg) CopyImage(src, dst string) error {
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 
-	Debug(":tea: Tagging image", src, dst)
+	s.ctx.Debug(":tea: Tagging image", src, dst)
 	cmd := exec.Command("img", "tag", src, dst)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed tagging image: "+string(out))
 	}
-	Info(":tea: Image " + dst + " tagged")
+	s.ctx.Info(":tea: Image " + dst + " tagged")
 
 	return nil
 }
 
-func (*SimpleImg) ImageAvailable(imagename string) bool {
+func (s *SimpleImg) ImageAvailable(imagename string) bool {
 	return imageAvailable(imagename)
 }
 
@@ -135,20 +136,20 @@ func (s *SimpleImg) ImageDefinitionToTar(opts Options) error {
 	return nil
 }
 
-func (*SimpleImg) ExportImage(opts Options) error {
+func (s *SimpleImg) ExportImage(opts Options) error {
 	name := opts.ImageName
 	path := opts.Destination
 	buildarg := []string{"save", "-o", path, name}
-	Debug(":tea: Saving image " + name)
+	s.ctx.Debug(":tea: Saving image " + name)
 
-	Spinner(22)
-	defer SpinnerStop()
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed exporting image: "+string(out))
 	}
-	Info(":tea: Image " + name + " saved")
+	s.ctx.Info(":tea: Image " + name + " saved")
 	return nil
 }
 
@@ -166,20 +167,20 @@ func (s *SimpleImg) ExtractRootfs(opts Options, keepPerms bool) error {
 	os.RemoveAll(path)
 
 	buildarg := []string{"unpack", "-o", path, name}
-	Debug(":tea: Extracting image " + name)
+	s.ctx.Debug(":tea: Extracting image " + name)
 
-	Spinner(22)
-	defer SpinnerStop()
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed extracting image: "+string(out))
 	}
-	Debug(":tea: Image " + name + " extracted")
+	s.ctx.Debug(":tea: Image " + name + " extracted")
 	return nil
 }
 
-func (*SimpleImg) Push(opts Options) error {
+func (s *SimpleImg) Push(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePush, opts)
 
@@ -188,9 +189,9 @@ func (*SimpleImg) Push(opts Options) error {
 	if err != nil {
 		return errors.Wrap(err, "Failed pushing image: "+string(out))
 	}
-	Info(":tea: Pushed image:", name)
+	s.ctx.Info(":tea: Pushed image:", name)
 	bus.Manager.Publish(bus.EventImagePostPush, opts)
 
-	//Info(string(out))
+	//s.ctx.Info(string(out))
 	return nil
 }

pkg/compiler/backend/writer.go

@@ -19,18 +19,20 @@ package backend
 import (
 	"bytes"
 
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/pkg/api/core/types"
 )
 
 type BackendWriter struct {
 	BufferedOutput bool
 	Buffer         *bytes.Buffer
+	ctx            *types.Context
 }
 
-func NewBackendWriter(buffered bool) *BackendWriter {
+func NewBackendWriter(buffered bool, ctx *types.Context) *BackendWriter {
 	return &BackendWriter{
 		BufferedOutput: buffered,
 		Buffer:         &bytes.Buffer{},
+		ctx:            ctx,
 	}
 }
 
@@ -39,7 +41,7 @@ func (b *BackendWriter) Write(p []byte) (int, error) {
 		return b.Buffer.Write(p)
 	}
 
-	Msg("info", false, false, (string(p)))
+	b.ctx.Msg("info", false, (string(p)))
 
 	return len(p), nil
 }

pkg/compiler/backend_test.go

@@ -16,6 +16,7 @@
 package compiler_test
 
 import (
+	"github.com/mudler/luet/pkg/api/core/types"
 	. "github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
 
@@ -25,9 +26,9 @@ import (
 
 var _ = Describe("Docker image diffs", func() {
 	var b CompilerBackend
-
+	ctx := types.NewContext()
 	BeforeEach(func() {
-		b = NewSimpleDockerBackend()
+		b = NewSimpleDockerBackend(ctx)
 	})
 
 	Context("Generate diffs from docker images", func() {
@@ -38,7 +39,7 @@ var _ = Describe("Docker image diffs", func() {
 			err := b.DownloadImage(opts)
 			Expect(err).ToNot(HaveOccurred())
 
-			layers, err := GenerateChanges(b, opts, opts)
+			layers, err := GenerateChanges(ctx, b, opts, opts)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(layers)).To(Equal(1))
 			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
@@ -56,7 +57,7 @@ var _ = Describe("Docker image diffs", func() {
 			})
 			Expect(err).ToNot(HaveOccurred())
 
-			layers, err := GenerateChanges(b, Options{
+			layers, err := GenerateChanges(ctx, b, Options{
 				ImageName: "quay.io/mocaccino/micro",
 			}, Options{
 				ImageName: "quay.io/mocaccino/extra",

pkg/compiler/buildtree.go

@@ -0,0 +1,140 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"encoding/json"
+	"sort"
+)
+
+func rankMapStringInt(values map[string]int) []string {
+	type kv struct {
+		Key   string
+		Value int
+	}
+	var ss []kv
+	for k, v := range values {
+		ss = append(ss, kv{k, v})
+	}
+	sort.Slice(ss, func(i, j int) bool {
+		return ss[i].Value > ss[j].Value
+	})
+	ranked := make([]string, len(values))
+	for i, kv := range ss {
+		ranked[i] = kv.Key
+	}
+	return ranked
+}
+
+type BuildTree struct {
+	order map[string]int
+}
+
+func (bt *BuildTree) Increase(s string) {
+	if bt.order == nil {
+		bt.order = make(map[string]int)
+	}
+	if _, ok := bt.order[s]; !ok {
+		bt.order[s] = 0
+	}
+
+	bt.order[s]++
+}
+
+func (bt *BuildTree) Reset(s string) {
+	if bt.order == nil {
+		bt.order = make(map[string]int)
+	}
+	bt.order[s] = 0
+}
+
+func (bt *BuildTree) Level(s string) int {
+	return bt.order[s]
+}
+
+func ints(input []int) []int {
+	u := make([]int, 0, len(input))
+	m := make(map[int]bool)
+
+	for _, val := range input {
+		if _, ok := m[val]; !ok {
+			m[val] = true
+			u = append(u, val)
+		}
+	}
+
+	return u
+}
+
+func (bt *BuildTree) AllInLevel(l int) []string {
+	var all []string
+	for k, v := range bt.order {
+		if v == l {
+			all = append(all, k)
+		}
+	}
+	return all
+}
+
+func (bt *BuildTree) Order(compilationTree map[string]map[string]interface{}) {
+	sentinel := false
+	for !sentinel {
+		sentinel = true
+
+	LEVEL:
+		for _, l := range bt.AllLevels() {
+
+			for _, j := range bt.AllInLevel(l) {
+				for _, k := range bt.AllInLevel(l) {
+					if j == k {
+						continue
+					}
+					if _, ok := compilationTree[j][k]; ok {
+						if bt.Level(j) == bt.Level(k) {
+							bt.Increase(j)
+							sentinel = false
+							break LEVEL
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+func (bt *BuildTree) AllLevels() []int {
+	var all []int
+	for _, v := range bt.order {
+		all = append(all, v)
+	}
+
+	sort.Sort(sort.IntSlice(all))
+
+	return ints(all)
+}
+
+func (bt *BuildTree) JSON() (string, error) {
+	type buildjob struct {
+		Jobs []string `json:"packages"`
+	}
+
+	result := []buildjob{}
+	for _, l := range bt.AllLevels() {
+		result = append(result, buildjob{Jobs: bt.AllInLevel(l)})
+	}
+	dat, err := json.Marshal(&result)
+	return string(dat), err
+}

pkg/compiler/compiler.go

@@ -16,7 +16,9 @@
 package compiler
 
 import (
+	"crypto/md5"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -27,18 +29,21 @@ import (
 	"sync"
 	"time"
 
-	"github.com/imdario/mergo"
+	"github.com/mudler/luet/pkg/api/core/types"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	bus "github.com/mudler/luet/pkg/bus"
 	"github.com/mudler/luet/pkg/compiler/backend"
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/helpers"
-	. "github.com/mudler/luet/pkg/logger"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/imdario/mergo"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
+	"helm.sh/helm/v3/pkg/chart"
 )
 
 const BuildFile = "build.yaml"
@@ -75,12 +80,12 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, compilerOp
 	// The CompilerRecipe will gives us a tree with only build deps listed.
 
 	c := NewCompiler(compilerOpts...)
+	if c.Options.Context == nil {
+		c.Options.Context = types.NewContext()
+	}
 	//	c.Options.BackendType
 	c.Backend = backend
 	c.Database = db
-	// c.CompilerRecipe = &tree.CompilerRecipe{
-	// 	Recipe: tree.Recipe{Database: db},
-	// }
 
 	return c
 }
@@ -89,7 +94,7 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *c
 	defer wg.Done()
 
 	for s := range cspecs {
-		ar, err := cs.compile(concurrency, keepPermissions, s)
+		ar, err := cs.compile(concurrency, keepPermissions, nil, nil, s)
 		if err != nil {
 			errors <- err
 		}
@@ -107,7 +112,7 @@ func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps *compile
 		return artifacts, err
 	}
 
-	Info(":ant: Resolving reverse dependencies")
+	cs.Options.Context.Info(":ant: Resolving reverse dependencies")
 	toCompile := compilerspec.NewLuetCompilationspecs()
 	for _, a := range artifacts {
 
@@ -125,7 +130,7 @@ func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps *compile
 
 	uniques := toCompile.Unique().Remove(ps)
 	for _, u := range uniques.All() {
-		Info(" :arrow_right_hook:", u.GetPackage().GetName(), ":leaves:", u.GetPackage().GetVersion(), "(", u.GetPackage().GetCategory(), ")")
+		cs.Options.Context.Info(" :arrow_right_hook:", u.GetPackage().GetName(), ":leaves:", u.GetPackage().GetVersion(), "(", u.GetPackage().GetCategory(), ")")
 	}
 
 	artifacts2, err := cs.CompileParallel(keepPermissions, uniques)
@@ -146,11 +151,6 @@ func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps *compilerspec.L
 	}
 
 	for _, p := range ps.All() {
-		asserts, err := cs.ComputeDepTree(p)
-		if err != nil {
-			panic(err)
-		}
-		p.SetSourceAssertion(asserts)
 		all <- p
 	}
 
@@ -196,11 +196,15 @@ func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, includ
 		for _, i := range includeRegexp {
 			if i.MatchString(abspath) {
 				match = true
+				break
 			}
 		}
 
 		if include && !match || !include && match {
 			toRemove = append(toRemove, currentpath)
+			cs.Options.Context.Debug(":scissors: Removing file", currentpath)
+		} else {
+			cs.Options.Context.Debug(":sun: Matched file", currentpath)
 		}
 
 		return nil
@@ -214,7 +218,7 @@ func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, includ
 	for _, s := range toRemove {
 		e := os.RemoveAll(s)
 		if e != nil {
-			Warning("Failed removing", s, e.Error())
+			cs.Options.Context.Warning("Failed removing", s, e.Error())
 			return e
 		}
 	}
@@ -236,7 +240,7 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
 	}
 
 	if p.GetPackageDir() != "" {
-		Info(":tophat: Packing from output dir", p.GetPackageDir())
+		cs.Options.Context.Info(":tophat: Packing from output dir", p.GetPackageDir())
 		rootfs = filepath.Join(rootfs, p.GetPackageDir())
 	}
 
@@ -275,18 +279,18 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co
 		}
 	}
 
-	Info(pkgTag, ":hammer: Generating delta")
-	diffs, err := GenerateChanges(cs.Backend, builderOpts, runnerOpts)
+	cs.Options.Context.Info(pkgTag, ":hammer: Generating delta")
+	diffs, err := GenerateChanges(cs.Options.Context, cs.Backend, builderOpts, runnerOpts)
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not generate changes from layers")
 	}
 
-	Debug("Extracting image to grab files from delta")
+	cs.Options.Context.Debug("Extracting image to grab files from delta")
 	if err := cs.Backend.ExtractRootfs(backend.Options{
 		ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions); err != nil {
 		return nil, errors.Wrap(err, "Could not extract rootfs")
 	}
-	artifact, err := artifact.ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.Options.CompressionType)
+	artifact, err := artifact.ExtractArtifactFromDelta(cs.Options.Context, rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.Options.CompressionType)
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not generate deltas")
 	}
@@ -295,17 +299,6 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co
 	return artifact, nil
 }
 
-func (cs *LuetCompiler) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
-	// Use packageImage as salt into the fp being used
-	// so the hash is unique also in cases where
-	// some package deps does have completely different
-	// depgraphs
-	// TODO: We should use the image tag, or pass by the package assertion hash which is unique
-	// and identifies the deptree of the package.
-	return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(helpers.StripRegistryFromImage(packageImage)))
-
-}
-
 func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
 	concurrency int, keepPermissions bool,
 	p *compilerspec.LuetCompilationSpec) (backend.Options, backend.Options, error) {
@@ -333,7 +326,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	defer os.RemoveAll(buildDir) // clean up
 
 	// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
-	err = helpers.CopyDir(p.GetPackage().GetPath(), buildDir)
+	err = fileHelper.CopyDir(p.GetPackage().GetPath(), buildDir)
 	if err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not copy package sources")
 	}
@@ -342,7 +335,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	if len(p.GetRetrieve()) > 0 {
 		err := p.CopyRetrieves(buildDir)
 		if err != nil {
-			Warning("Failed copying retrieves", err.Error())
+			cs.Options.Context.Warning("Failed copying retrieves", err.Error())
 		}
 	}
 
@@ -388,8 +381,8 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 			if err == nil {
 				buildImage = false
 			} else {
-				Warning("Failed to download '" + opts.ImageName + "'. Will keep going and build the image unless you use --fatal")
-				Warning(err.Error())
+				cs.Options.Context.Warning("Failed to download '" + opts.ImageName + "'. Will keep going and build the image unless you use --fatal")
+				cs.Options.Context.Warning(err.Error())
 			}
 		}
 		if buildImage {
@@ -406,7 +399,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	}
 	// SKIPBUILD
 	//	if len(p.GetPreBuildSteps()) != 0 {
-	Info(pkgTag, ":whale: Generating 'builder' image from", image, "as", buildertaggedImage, "with prelude steps")
+	cs.Options.Context.Info(pkgTag, ":whale: Generating 'builder' image from", image, "as", buildertaggedImage, "with prelude steps")
 	if err := buildAndPush(builderOpts); err != nil {
 		return builderOpts, runnerOpts, errors.Wrapf(err, "Could not push image: %s %s", image, builderOpts.DockerFileName)
 	}
@@ -414,7 +407,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 
 	// Even if we might not have any steps to build, we do that so we can tag the image used in this moment and use that to cache it in a registry, or in the system.
 	// acting as a docker tag.
-	Info(pkgTag, ":whale: Generating 'package' image from", buildertaggedImage, "as", packageImage, "with build steps")
+	cs.Options.Context.Info(pkgTag, ":whale: Generating 'package' image from", buildertaggedImage, "as", packageImage, "with build steps")
 	if err := buildAndPush(runnerOpts); err != nil {
 		return builderOpts, runnerOpts, errors.Wrapf(err, "Could not push image: %s %s", image, runnerOpts.DockerFileName)
 	}
@@ -429,7 +422,7 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	var rootfs string
 	var err error
 	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
-	Debug(pkgTag, "Generating artifact")
+	cs.Options.Context.Debug(pkgTag, "Generating artifact")
 	// We can't generate delta in this case. It implies the package is a virtual, and nothing has to be done really
 	if p.EmptyPackage() {
 		fakePackage := p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")
@@ -450,11 +443,11 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 		a.CompileSpec = p
 		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
 
-		err = a.WriteYaml(p.GetOutputPath())
+		err = a.WriteYAML(p.GetOutputPath())
 		if err != nil {
 			return a, errors.Wrap(err, "Failed while writing metadata file")
 		}
-		Info(pkgTag, "   :white_check_mark: done (empty virtual package)")
+		cs.Options.Context.Success(pkgTag, "   :white_check_mark: done (empty virtual package)")
 		return a, nil
 	}
 
@@ -480,11 +473,11 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	a.Files = filelist
 	a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
 
-	err = a.WriteYaml(p.GetOutputPath())
+	err = a.WriteYAML(p.GetOutputPath())
 	if err != nil {
 		return a, errors.Wrap(err, "Failed while writing metadata file")
 	}
-	Info(pkgTag, "   :white_check_mark: Done")
+	cs.Options.Context.Success(pkgTag, "   :white_check_mark: Done")
 
 	return a, nil
 }
@@ -493,12 +486,12 @@ func (cs *LuetCompiler) waitForImages(images []string) {
 	if cs.Options.PullFirst && cs.Options.Wait {
 		available, _ := oneOfImagesAvailable(images, cs.Backend)
 		if !available {
-			Info(fmt.Sprintf("Waiting for image %s to be available... :zzz:", images))
-			Spinner(22)
-			defer SpinnerStop()
+			cs.Options.Context.Info(fmt.Sprintf("Waiting for image %s to be available... :zzz:", images))
+			cs.Options.Context.Spinner()
+			defer cs.Options.Context.SpinnerStop()
 			for !available {
 				available, _ = oneOfImagesAvailable(images, cs.Backend)
-				Info(fmt.Sprintf("Image %s not available yet, sleeping", images))
+				cs.Options.Context.Info(fmt.Sprintf("Image %s not available yet, sleeping", images))
 				time.Sleep(5 * time.Second)
 			}
 		}
@@ -522,9 +515,9 @@ func oneOfImagesAvailable(images []string, b CompilerBackend) (bool, string) {
 	return false, ""
 }
 
-func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
+func (cs *LuetCompiler) findImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
 	var resolvedImage string
-	Debug("Resolving image hash for", p.Package.HumanReadableString(), "hash", imageHash, "Pull repositories", p.BuildOptions.PullImageRepository)
+	cs.Options.Context.Debug("Resolving image hash for", p.Package.HumanReadableString(), "hash", imageHash, "Pull repositories", p.BuildOptions.PullImageRepository)
 	toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)},
 		genImageList(p.BuildOptions.PullImageRepository, imageHash)...)
 	if exists, which := oneOfImagesExists(toChecklist, cs.Backend); exists {
@@ -535,6 +528,11 @@ func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilersp
 			resolvedImage = which
 		}
 	}
+	return resolvedImage
+}
+
+func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
+	resolvedImage := cs.findImageHash(imageHash, p)
 
 	if resolvedImage == "" {
 		resolvedImage = fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)
@@ -560,20 +558,20 @@ func LoadArtifactFromYaml(spec *compilerspec.LuetCompilationSpec) (*artifact.Pac
 func (cs *LuetCompiler) getImageArtifact(hash string, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
 	// we check if there is an available image with the given hash and
 	// we return a full artifact if can be loaded locally.
-	Debug("Get image artifact for", p.Package.HumanReadableString(), "hash", hash, "Pull repositories", p.BuildOptions.PullImageRepository)
+	cs.Options.Context.Debug("Get image artifact for", p.Package.HumanReadableString(), "hash", hash, "Pull repositories", p.BuildOptions.PullImageRepository)
 
 	toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, hash)},
 		genImageList(p.BuildOptions.PullImageRepository, hash)...)
 
 	exists, _ := oneOfImagesExists(toChecklist, cs.Backend)
 	if art, err := LoadArtifactFromYaml(p); err == nil && exists { // If YAML is correctly loaded, and both images exists, no reason to rebuild.
-		Debug("Package reloaded from YAML. Skipping build")
+		cs.Options.Context.Debug("Package reloaded from YAML. Skipping build")
 		return art, nil
 	}
 	cs.waitForImages(toChecklist)
 	available, _ := oneOfImagesAvailable(toChecklist, cs.Backend)
 	if exists || (cs.Options.PullFirst && available) {
-		Debug("Image available, returning empty artifact")
+		cs.Options.Context.Debug("Image available, returning empty artifact")
 		return &artifact.PackageArtifact{}, nil
 	}
 
@@ -612,20 +610,32 @@ func (cs *LuetCompiler) compileWithImage(image, builderHash string, packageTagHa
 	// if buildertaggedImage == "" {
 	// 	buildertaggedImage = fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, buildertaggedImage)
 	// 	generated = true
-	// 	//	Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image)
+	// 	//	cs.Options.Context.Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image)
 	// }
 
 	if cs.Options.PullFirst && !cs.Options.Rebuild {
-		Debug("Checking if an image is already available")
+		cs.Options.Context.Debug("Checking if an image is already available")
 		// FIXUP here. If packageimage hash exists and pull is true, generate package
 		resolved := cs.resolveExistingImageHash(packageTagHash, p)
+		cs.Options.Context.Debug("Resolved: " + resolved)
+		cs.Options.Context.Debug("Expected remote: " + resolved)
+		cs.Options.Context.Debug("Package image: " + packageImage)
+		cs.Options.Context.Debug("Resolved builder image: " + builderResolved)
 
-		//
-		if resolved != packageImage && remoteBuildertaggedImage != builderResolved { // an image is there already
-			Debug("Images available for", p.Package.HumanReadableString(), "generating artifact from remote images:", resolved)
+		// a remote image is there already
+		remoteImageAvailable := resolved != packageImage && remoteBuildertaggedImage != builderResolved
+		// or a local one is available
+		localImageAvailable := cs.Backend.ImageExists(remoteBuildertaggedImage) && cs.Backend.ImageExists(packageImage)
+
+		switch {
+		case remoteImageAvailable:
+			cs.Options.Context.Debug("Images available remotely for", p.Package.HumanReadableString(), "generating artifact from remote images:", resolved)
 			return cs.genArtifact(p, backend.Options{ImageName: builderResolved}, backend.Options{ImageName: resolved}, concurrency, keepPermissions)
-		} else {
-			Debug("Images not available for", p.Package.HumanReadableString())
+		case localImageAvailable:
+			cs.Options.Context.Debug("Images locally available for", p.Package.HumanReadableString(), "generating artifact from image:", resolved)
+			return cs.genArtifact(p, backend.Options{ImageName: remoteBuildertaggedImage}, backend.Options{ImageName: packageImage}, concurrency, keepPermissions)
+		default:
+			cs.Options.Context.Debug("Images not available for", p.Package.HumanReadableString())
 		}
 	}
 
@@ -639,10 +649,10 @@ func (cs *LuetCompiler) compileWithImage(image, builderHash string, packageTagHa
 		defer func() {
 			// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
 			if err := cs.Backend.RemoveImage(builderOpts); err != nil {
-				Warning("Could not remove image ", builderOpts.ImageName)
+				cs.Options.Context.Warning("Could not remove image ", builderOpts.ImageName)
 			}
 			if err := cs.Backend.RemoveImage(runnerOpts); err != nil {
-				Warning("Could not remove image ", runnerOpts.ImageName)
+				cs.Options.Context.Warning("Could not remove image ", runnerOpts.ImageName)
 			}
 		}()
 	}
@@ -680,33 +690,7 @@ func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst s
 	}
 }
 
-// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
-func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
-	// Generate a set with all the deps of the provided specs
-	// we will use that set to remove the deps from the list of provided compilation specs
-	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
-	result := []*compilerspec.LuetCompilationSpec{}
-	for _, spec := range p {
-		ass, err := cs.ComputeDepTree(spec)
-		if err != nil {
-			return result, errors.Wrap(err, "computin specs deptree")
-		}
-
-		allDependencies = append(allDependencies, ass.Drop(spec.GetPackage())...)
-	}
-
-	for _, spec := range p {
-		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
-			result = append(result, spec)
-		}
-	}
-	return result, nil
-}
-
-// ComputeDepTree computes the dependency tree of a compilation spec and returns solver assertions
-// in order to be able to compile the spec.
 func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
-
 	s := solver.NewResolver(cs.Options.SolverOptions.Options, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
 
 	solution, err := s.Install(pkg.Packages{p.GetPackage()})
@@ -718,32 +702,80 @@ func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (sol
 	if err != nil {
 		return nil, errors.Wrap(err, "While order a solution for "+p.GetPackage().HumanReadableString())
 	}
+	return dependencies, nil
+}
 
-	assertions := solver.PackagesAssertions{}
-	for _, assertion := range dependencies { //highly dependent on the order
-		if assertion.Value {
-			nthsolution := dependencies.Cut(assertion.Package)
-			assertion.Hash = solver.PackageHash{
-				BuildHash:   nthsolution.HashFrom(assertion.Package),
-				PackageHash: nthsolution.AssertionHash(),
+// BuildTree returns a BuildTree which represent the order in which specs should be compiled.
+// It places specs into levels, and each level can be built in parallel. The root nodes starting from the top.
+// A BuildTree can be marshaled into JSON or walked like:
+// for _, l := range bt.AllLevels() {
+//	fmt.Println(strings.Join(bt.AllInLevel(l), " "))
+// }
+func (cs *LuetCompiler) BuildTree(compilerSpecs compilerspec.LuetCompilationspecs) (*BuildTree, error) {
+	compilationTree := map[string]map[string]interface{}{}
+	bt := &BuildTree{}
+
+	for _, sp := range compilerSpecs.All() {
+		ass, err := cs.ComputeDepTree(sp)
+		if err != nil {
+			return nil, err
+		}
+		bt.Reset(fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName()))
+		for _, p := range ass {
+			bt.Reset(fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName()))
+
+			spec, err := cs.FromPackage(p.Package)
+			if err != nil {
+				return nil, err
 			}
-			assertion.Package.SetTreeDir(p.Package.GetTreeDir())
-			assertions = append(assertions, assertion)
+			ass, err := cs.ComputeDepTree(spec)
+			if err != nil {
+				return nil, err
+			}
+			for _, r := range ass {
+				if compilationTree[fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())] == nil {
+					compilationTree[fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())] = make(map[string]interface{})
+				}
+				compilationTree[fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())][fmt.Sprintf("%s/%s", r.Package.GetCategory(), r.Package.GetName())] = nil
+			}
+			if compilationTree[fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName())] == nil {
+				compilationTree[fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName())] = make(map[string]interface{})
+			}
+			compilationTree[fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName())][fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())] = nil
 		}
 	}
-	p.SetSourceAssertion(assertions)
-	return assertions, nil
+
+	bt.Order(compilationTree)
+
+	return bt, nil
+}
+
+// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
+func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
+	// Generate a set with all the deps of the provided specs
+	// we will use that set to remove the deps from the list of provided compilation specs
+	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
+	result := []*compilerspec.LuetCompilationSpec{}
+	for _, spec := range p {
+		sol, err := cs.ComputeDepTree(spec)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed querying hashtree")
+		}
+		allDependencies = append(allDependencies, sol.Drop(spec.GetPackage())...)
+	}
+
+	for _, spec := range p {
+		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
+			result = append(result, spec)
+		}
+	}
+	return result, nil
 }
 
 // Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
 // an artifact
 func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	asserts, err := cs.ComputeDepTree(p)
-	if err != nil {
-		return nil, err
-	}
-	p.SetSourceAssertion(asserts)
-	return cs.compile(cs.Options.Concurrency, keepPermissions, p)
+	return cs.compile(cs.Options.Concurrency, keepPermissions, nil, nil, p)
 }
 
 func genImageList(refs []string, hash string) []string {
@@ -755,32 +787,207 @@ func genImageList(refs []string, hash string) []string {
 }
 
 func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationSpec) {
-	Debug(p.GetPackage().HumanReadableString(), "Build options before inherit", p.BuildOptions)
+	cs.Options.Context.Debug(p.GetPackage().HumanReadableString(), "Build options before inherit", p.BuildOptions)
 
 	// Append push repositories from buildpsec buildoptions as pull if found.
 	// This allows to resolve the hash automatically if we pulled the metadata from
 	// repositories that are advertizing their cache.
 	if len(p.BuildOptions.PushImageRepository) != 0 {
 		p.BuildOptions.PullImageRepository = append(p.BuildOptions.PullImageRepository, p.BuildOptions.PushImageRepository)
-		Debug("Inheriting pull repository from PushImageRepository buildoptions", p.BuildOptions.PullImageRepository)
+		cs.Options.Context.Debug("Inheriting pull repository from PushImageRepository buildoptions", p.BuildOptions.PullImageRepository)
 	}
 
 	if len(cs.Options.PullImageRepository) != 0 {
 		p.BuildOptions.PullImageRepository = append(p.BuildOptions.PullImageRepository, cs.Options.PullImageRepository...)
-		Debug("Inheriting pull repository from PullImageRepository buildoptions", p.BuildOptions.PullImageRepository)
+		cs.Options.Context.Debug("Inheriting pull repository from PullImageRepository buildoptions", p.BuildOptions.PullImageRepository)
 	}
-	Debug(p.GetPackage().HumanReadableString(), "Build options after inherit", p.BuildOptions)
+
+	cs.Options.Context.Debug(p.GetPackage().HumanReadableString(), "Build options after inherit", p.BuildOptions)
 }
 
-func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	// TODO: Racy, remove it
-	// Inherit build options from compilation specs metadata
-	// orig := cs.Options.PullImageRepository
-	// defer func() { cs.Options.PullImageRepository = orig }()
+func (cs *LuetCompiler) getSpecHash(pkgs pkg.DefaultPackages, salt string) (string, error) {
+	ht := NewHashTree(cs.Database)
+	overallFp := ""
+	for _, p := range pkgs {
+		compileSpec, err := cs.FromPackage(p)
+		if err != nil {
+			return "", errors.Wrap(err, "Error while generating compilespec for "+p.GetName())
+		}
+		packageHashTree, err := ht.Query(cs, compileSpec)
+		if err != nil {
+			return "nil", errors.Wrap(err, "failed querying hashtree")
+		}
+		overallFp = overallFp + packageHashTree.Target.Hash.PackageHash + p.GetFingerPrint()
+	}
 
-	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
+	h := md5.New()
+	io.WriteString(h, fmt.Sprintf("%s-%s", overallFp, salt))
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
 
-	Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
+func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+
+	joinTag := ">:loop: final images<"
+	var fromPackages pkg.DefaultPackages
+
+	if p.RequiresFinalImages {
+		cs.Options.Context.Info(joinTag, "Generating a parent image from final packages")
+		fromPackages = p.Package.GetRequires()
+	} else {
+		// No source image to resolve
+		return nil
+	}
+
+	// First compute a hash and check if image is available. if it is, then directly consume that
+	overallFp, err := cs.getSpecHash(fromPackages, "join")
+	if err != nil {
+		return errors.Wrap(err, "could not generate image hash")
+	}
+
+	cs.Options.Context.Info(joinTag, "Searching existing image with hash", overallFp)
+
+	image := cs.findImageHash(overallFp, p)
+	if image != "" {
+		cs.Options.Context.Info("Image already found", image)
+		p.SetImage(image)
+		return nil
+	}
+	cs.Options.Context.Info(joinTag, "Image not found. Generating image join with hash ", overallFp)
+
+	// Make sure there is an output path
+	if err := os.MkdirAll(p.GetOutputPath(), os.ModePerm); err != nil {
+		return errors.Wrap(err, "while creating output path")
+	}
+
+	// otherwise, generate it and push it aside
+	joinDir, err := ioutil.TempDir(p.GetOutputPath(), "join")
+	if err != nil {
+		return errors.Wrap(err, "could not create tempdir for joining images")
+	}
+	defer os.RemoveAll(joinDir) // clean up
+
+	for _, p := range fromPackages {
+		cs.Options.Context.Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
+	}
+
+	current := 0
+	for _, c := range fromPackages {
+		current++
+		if c != nil && c.Name != "" && c.Version != "" {
+			joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(p.Package.GetRequires()), c.HumanReadableString())
+
+			cs.Options.Context.Info(joinTag2, "compilation starts")
+			spec, err := cs.FromPackage(c)
+			if err != nil {
+				return errors.Wrap(err, "while generating images to join from")
+			}
+			wantsArtifact := true
+			genDepsArtifact := !cs.Options.PackageTargetOnly
+
+			spec.SetOutputPath(p.GetOutputPath())
+
+			artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, &genDepsArtifact, spec)
+			if err != nil {
+				return errors.Wrap(err, "failed building join image")
+			}
+
+			err = artifact.Unpack(cs.Options.Context, joinDir, keepPermissions)
+			if err != nil {
+				return errors.Wrap(err, "failed building join image")
+			}
+			cs.Options.Context.Info(joinTag2, ":white_check_mark: Done")
+		}
+	}
+
+	artifactDir, err := ioutil.TempDir(p.GetOutputPath(), "artifact")
+	if err != nil {
+		return errors.Wrap(err, "could not create tempdir for final artifact")
+	}
+	defer os.RemoveAll(joinDir) // clean up
+
+	cs.Options.Context.Info(joinTag, ":droplet: generating artifact for source image of", p.GetPackage().HumanReadableString())
+
+	// After unpack, create a new artifact and a new final image from it.
+	// no need to compress, as we are going to toss it away.
+	a := artifact.NewPackageArtifact(filepath.Join(artifactDir, p.GetPackage().GetFingerPrint()+".join.tar"))
+	if err := a.Compress(joinDir, concurrency); err != nil {
+		return errors.Wrap(err, "error met while creating package archive")
+	}
+
+	joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp)
+	cs.Options.Context.Info(joinTag, ":droplet: generating image from artifact", joinImageName)
+	opts, err := a.GenerateFinalImage(cs.Options.Context, joinImageName, cs.Backend, keepPermissions)
+	if err != nil {
+		return errors.Wrap(err, "could not create final image")
+	}
+	if cs.Options.Push {
+		cs.Options.Context.Info(joinTag, ":droplet: pushing image from artifact", joinImageName)
+		if err = cs.Backend.Push(opts); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
+		}
+	}
+	cs.Options.Context.Info(joinTag, ":droplet: Consuming image", joinImageName)
+	p.SetImage(joinImageName)
+	return nil
+}
+
+func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+	resolvedCopyFields := []compilerspec.CopyField{}
+	copyTag := ">:droplet: copy<"
+
+	if len(p.Copy) != 0 {
+		cs.Options.Context.Info(copyTag, "Package has multi-stage copy, generating required images")
+	}
+
+	current := 0
+	// TODO: we should run this only if we are going to build the image
+	for _, c := range p.Copy {
+		current++
+		if c.Package != nil && c.Package.Name != "" && c.Package.Version != "" {
+			copyTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", copyTag, current, len(p.Copy), c.Package.HumanReadableString())
+
+			cs.Options.Context.Info(copyTag2, "generating multi-stage images for", c.Package.HumanReadableString())
+			spec, err := cs.FromPackage(c.Package)
+			if err != nil {
+				return errors.Wrap(err, "while generating images to copy from")
+			}
+
+			// If we specify --only-target package, we don't want any artifact, otherwise we do
+			genArtifact := !cs.Options.PackageTargetOnly
+			spec.SetOutputPath(p.GetOutputPath())
+			artifact, err := cs.compile(concurrency, keepPermissions, &genArtifact, &genArtifact, spec)
+			if err != nil {
+				return errors.Wrap(err, "failed building multi-stage image")
+			}
+
+			resolvedCopyFields = append(resolvedCopyFields, compilerspec.CopyField{
+				Image:       cs.resolveExistingImageHash(artifact.PackageCacheImage, spec),
+				Source:      c.Source,
+				Destination: c.Destination,
+			})
+			cs.Options.Context.Success(copyTag2, ":white_check_mark: Done")
+		} else {
+			resolvedCopyFields = append(resolvedCopyFields, c)
+		}
+	}
+	p.Copy = resolvedCopyFields
+	return nil
+}
+
+func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateFinalArtifact *bool, generateDependenciesFinalArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
+	cs.Options.Context.Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
+
+	//Before multistage : join - same as multistage, but keep artifacts, join them, create a new one and generate a final image.
+	// When the image is there, use it as a source here, in place of GetImage().
+	if err := cs.resolveFinalImages(concurrency, keepPermissions, p); err != nil {
+		return nil, errors.Wrap(err, "while resolving join images")
+	}
+
+	if err := cs.resolveMultiStageImages(concurrency, keepPermissions, p); err != nil {
+		return nil, errors.Wrap(err, "while resolving multi-stage images")
+	}
+
+	cs.Options.Context.Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
 	if !p.HasImageSource() && !p.EmptyPackage() {
 		return nil,
 			fmt.Errorf(
@@ -789,64 +996,84 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 			)
 	}
 
-	targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
+	ht := NewHashTree(cs.Database)
+
+	packageHashTree, err := ht.Query(cs, p)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed querying hashtree")
+	}
+
+	// This is in order to have the metadata in the yaml
+	p.SetSourceAssertion(packageHashTree.Solution)
+	targetAssertion := packageHashTree.Target
 
 	bus.Manager.Publish(bus.EventPackagePreBuild, struct {
-		CompileSpec *compilerspec.LuetCompilationSpec
-		Assert      solver.PackageAssert
+		CompileSpec     *compilerspec.LuetCompilationSpec
+		Assert          solver.PackageAssert
+		PackageHashTree *PackageImageHashTree
 	}{
-		CompileSpec: p,
-		Assert:      *targetAssertion,
+		CompileSpec:     p,
+		Assert:          *targetAssertion,
+		PackageHashTree: packageHashTree,
 	})
 
 	// Update compilespec build options - it will be then serialized into the compilation metadata file
-	//p.SetBuildOptions(cs.Options)
 	p.BuildOptions.PushImageRepository = cs.Options.PushImageRepository
-	//p.BuildOptions.BuildValues = cs.Options.BuildValues
-	//p.BuildOptions.BuildValuesFile = cs.Options.BuildValuesFile
 
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		return cs.compileWithImage(p.GetImage(), cs.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
+		localGenerateArtifact := true
+		if generateFinalArtifact != nil {
+			localGenerateArtifact = *generateFinalArtifact
+		}
+
+		a, err := cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
+		if err != nil {
+			return nil, errors.Wrap(err, "building direct image")
+		}
+		a.SourceAssertion = p.GetSourceAssertion()
+
+		a.PackageCacheImage = targetAssertion.Hash.PackageHash
+		return a, nil
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
 	// on how we compute the resolvable tree.
 	// This means to recursively build all the build-images needed to reach that tree part.
 	// - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image.
-
-	dependencies := p.GetSourceAssertion().Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
-	departifacts := []*artifact.PackageArtifact{}               // TODO: Return this somehow
-	var lastHash string
+	dependencies := packageHashTree.Dependencies  // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
+	departifacts := []*artifact.PackageArtifact{} // TODO: Return this somehow
 	depsN := 0
 	currentN := 0
 
 	packageDeps := !cs.Options.PackageTargetOnly
+	if generateDependenciesFinalArtifact != nil {
+		packageDeps = *generateDependenciesFinalArtifact
+	}
+
 	buildDeps := !cs.Options.NoDeps
 	buildTarget := !cs.Options.OnlyDeps
 
 	if buildDeps {
-		Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
+		cs.Options.Context.Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
 		for _, assertion := range dependencies { //highly dependent on the order
 			depsN++
-			Info(" :arrow_right_hook:", assertion.Package.HumanReadableString(), ":leaves:")
+			cs.Options.Context.Info(" :arrow_right_hook:", assertion.Package.HumanReadableString(), ":leaves:")
 		}
 
 		for _, assertion := range dependencies { //highly dependent on the order
 			currentN++
 			pkgTag := fmt.Sprintf(":package: %d/%d %s ⤑ :hammer: build %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
-			Info(pkgTag, " starts")
+			cs.Options.Context.Info(pkgTag, " starts")
 			compileSpec, err := cs.FromPackage(assertion.Package)
 			if err != nil {
 				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
 			}
 			compileSpec.BuildOptions.PullImageRepository = append(compileSpec.BuildOptions.PullImageRepository, p.BuildOptions.PullImageRepository...)
-			Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
+			cs.Options.Context.Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
 
 			compileSpec.SetOutputPath(p.GetOutputPath())
-			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash)
-			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
 
 			bus.Manager.Publish(bus.EventPackagePreBuild, struct {
 				CompileSpec *compilerspec.LuetCompilationSpec
@@ -856,29 +1083,53 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 				Assert:      assertion,
 			})
 
-			lastHash = assertion.Hash.PackageHash
-			// for the source instead, pick an image and a buildertaggedImage from hashes if they exists.
-			// otherways fallback to the pushed repo
-			// Resolve images from the hashtree
-			resolvedBuildImage := cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec)
-			if compileSpec.GetImage() != "" {
-				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
-
-				a, err := cs.compileWithImage(compileSpec.GetImage(), assertion.Hash.BuildHash, assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps)
-				if err != nil {
-					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
-				}
-				departifacts = append(departifacts, a)
-				Info(pkgTag, ":white_check_mark: Done")
-				continue
+			if err := cs.resolveFinalImages(concurrency, keepPermissions, compileSpec); err != nil {
+				return nil, errors.Wrap(err, "while resolving join images")
 			}
 
-			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
-			a, err := cs.compileWithImage(resolvedBuildImage, cs.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash), assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps)
+			if err := cs.resolveMultiStageImages(concurrency, keepPermissions, compileSpec); err != nil {
+				return nil, errors.Wrap(err, "while resolving multi-stage images")
+			}
+
+			buildHash, err := packageHashTree.DependencyBuildImage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed looking for dependency in hashtree")
+			}
+
+			cs.Options.Context.Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash)
+			cs.Options.Context.Debug(pkgTag, "    :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
+
+			var sourceImage string
+
+			if compileSpec.GetImage() != "" {
+				cs.Options.Context.Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
+				sourceImage = compileSpec.GetImage()
+			} else {
+				// for the source instead, pick an image and a buildertaggedImage from hashes if they exists.
+				// otherways fallback to the pushed repo
+				// Resolve images from the hashtree
+				sourceImage = cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec)
+				cs.Options.Context.Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
+			}
+
+			a, err := cs.compileWithImage(
+				sourceImage,
+				buildHash,
+				assertion.Hash.PackageHash,
+				concurrency,
+				keepPermissions,
+				cs.Options.KeepImg,
+				compileSpec,
+				packageDeps,
+			)
 			if err != nil {
 				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 			}
 
+			a.PackageCacheImage = assertion.Hash.PackageHash
+
+			cs.Options.Context.Success(pkgTag, ":white_check_mark: Done")
+
 			bus.Manager.Publish(bus.EventPackagePostBuild, struct {
 				CompileSpec *compilerspec.LuetCompilationSpec
 				Artifact    *artifact.PackageArtifact
@@ -888,24 +1139,24 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 			})
 
 			departifacts = append(departifacts, a)
-			Info(pkgTag, ":white_check_mark: Done")
 		}
-
-	} else if len(dependencies) > 0 {
-		lastHash = dependencies[len(dependencies)-1].Hash.PackageHash
 	}
 
 	if buildTarget {
-		resolvedBuildImage := cs.resolveExistingImageHash(lastHash, p)
-		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
-		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedBuildImage)
-		a, err := cs.compileWithImage(resolvedBuildImage, cs.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
+		localGenerateArtifact := true
+		if generateFinalArtifact != nil {
+			localGenerateArtifact = *generateFinalArtifact
+		}
+		resolvedSourceImage := cs.resolveExistingImageHash(packageHashTree.SourceHash, p)
+		cs.Options.Context.Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
+		cs.Options.Context.Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedSourceImage)
+		a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
 		if err != nil {
 			return a, err
 		}
 		a.Dependencies = departifacts
 		a.SourceAssertion = p.GetSourceAssertion()
-
+		a.PackageCacheImage = targetAssertion.Hash.PackageHash
 		bus.Manager.Publish(bus.EventPackagePostBuild, struct {
 			CompileSpec *compilerspec.LuetCompilationSpec
 			Artifact    *artifact.PackageArtifact
@@ -923,6 +1174,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 type templatedata map[string]interface{}
 
 func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.Package, dst templatedata) ([]byte, error) {
+	// Grab shared templates first
+	var chartFiles []*chart.File
+	if len(cs.Options.TemplatesFolder) != 0 {
+		c, err := helpers.ChartFiles(cs.Options.TemplatesFolder)
+		if err == nil {
+			chartFiles = c
+		}
+	}
 
 	var dataresult []byte
 	val := pack.Rel(DefinitionFile)
@@ -960,7 +1219,7 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
 			return nil, errors.Wrap(err, "merging values maps")
 		}
 
-		dat, err := helpers.RenderHelm(string(dataBuild), td, dst)
+		dat, err := helpers.RenderHelm(append(chartFiles, helpers.ChartFileB(dataBuild)...), td, dst)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}
@@ -978,14 +1237,20 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
 				if err != nil {
 					return nil, errors.Wrap(err, "while marshalling values file")
 				}
-				f := filepath.Join(valuesdir, helpers.RandStringRunes(20))
+				f := filepath.Join(valuesdir, fileHelper.RandStringRunes(20))
 				if err := ioutil.WriteFile(f, out, os.ModePerm); err != nil {
 					return nil, errors.Wrap(err, "while writing temporary values file")
 				}
 				bv = append([]string{f}, bv...)
 			}
 		}
-		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, bv...)
+
+		raw, err := ioutil.ReadFile(pack.Rel(BuildFile))
+		if err != nil {
+			return nil, err
+		}
+
+		out, err := helpers.RenderFiles(append(chartFiles, helpers.ChartFileB(raw)...), val, bv...)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}
@@ -1006,7 +1271,7 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio
 	opts := options.Compiler{}
 
 	artifactMetadataFile := filepath.Join(pack.GetTreeDir(), "..", pack.GetMetadataFilePath())
-	Debug("Checking if metadata file is present", artifactMetadataFile)
+	cs.Options.Context.Debug("Checking if metadata file is present", artifactMetadataFile)
 	if _, err := os.Stat(artifactMetadataFile); err == nil {
 		f, err := os.Open(artifactMetadataFile)
 		if err != nil {
@@ -1021,14 +1286,14 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio
 			return nil, errors.Wrap(err, "could not decode package from yaml")
 		}
 
-		Debug("Read build options:", art.CompileSpec.BuildOptions, "from", artifactMetadataFile)
+		cs.Options.Context.Debug("Read build options:", art.CompileSpec.BuildOptions, "from", artifactMetadataFile)
 		if art.CompileSpec.BuildOptions != nil {
 			opts = *art.CompileSpec.BuildOptions
 		}
 	} else if !os.IsNotExist(err) {
-		Debug("error reading artifact metadata file: ", err.Error())
+		cs.Options.Context.Debug("error reading artifact metadata file: ", err.Error())
 	} else if os.IsNotExist(err) {
-		Debug("metadata file not present, skipping", artifactMetadataFile)
+		cs.Options.Context.Debug("metadata file not present, skipping", artifactMetadataFile)
 	}
 
 	// Update processed build values
@@ -1051,6 +1316,11 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio
 
 	cs.inheritSpecBuildOptions(newSpec)
 
+	// Update the package in the compiler database to catch updates from NewLuetCompilationSpec
+	if err := cs.Database.UpdatePackage(newSpec.Package); err != nil {
+		return nil, errors.Wrap(err, "failed updating new package entry in compiler database")
+	}
+
 	return newSpec, err
 }
 

pkg/compiler/compiler_suite_test.go

@@ -18,15 +18,11 @@ package compiler_test
 import (
 	"testing"
 
-	. "github.com/mudler/luet/cmd"
-	config "github.com/mudler/luet/pkg/config"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
-	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Compiler Suite")
 }

pkg/compiler/compiler_test.go

@@ -19,12 +19,14 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/mudler/luet/pkg/api/core/types"
 	. "github.com/mudler/luet/pkg/compiler"
 	sd "github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/compiler/types/compression"
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	helpers "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -32,6 +34,8 @@ import (
 )
 
 var _ = Describe("Compiler", func() {
+	ctx := types.NewContext()
+
 	Context("Simple package build definition", func() {
 		It("Compiles it correctly", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -41,7 +45,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(types.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -59,15 +63,15 @@ var _ = Describe("Compiler", func() {
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
@@ -75,6 +79,68 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Copy and Join", func() {
+		It("Compiles it correctly with Copy", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/copy")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(types.NewContext()))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(spec.Rel("result"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("bina/busybox"))).To(BeTrue())
+		})
+
+		It("Compiles it correctly with Join", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/join")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(types.NewContext()))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(spec.Rel("newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+		})
+	})
+
 	Context("Simple package build definition", func() {
 		It("Compiles it in parallel", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -84,7 +150,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(1), options.WithContext(types.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -102,7 +168,7 @@ var _ = Describe("Compiler", func() {
 			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
@@ -118,7 +184,7 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/templates")
 			Expect(err).ToNot(HaveOccurred())
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 			pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -142,7 +208,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(types.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -165,23 +231,23 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(3))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
-			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("c"))
+			content1, err := fileHelper.Read(spec.Rel("c"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("cd"))
+			content2, err := fileHelper.Read(spec.Rel("cd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("c\n"))
 			Expect(content2).To(Equal("c\n"))
 
-			content1, err = helpers.Read(spec.Rel("d"))
+			content1, err = fileHelper.Read(spec.Rel("d"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err = helpers.Read(spec.Rel("dd"))
+			content2, err = fileHelper.Read(spec.Rel("dd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("s\n"))
 			Expect(content2).To(Equal("dd\n"))
@@ -198,7 +264,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(1), options.WithContext(types.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -215,17 +281,17 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts2)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			for _, artifact := range artifacts2 {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
-			Expect(helpers.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test1"))).To(BeTrue())
 		})
 
 		It("Compiles and includes ony wanted files", func() {
@@ -239,7 +305,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -254,12 +320,12 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
 		It("Compiles and excludes files", func() {
@@ -273,7 +339,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -288,13 +354,13 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 		})
 
 		It("Compiles includes and excludes files", func() {
@@ -308,7 +374,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -323,13 +389,13 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
 		It("Compiles and excludes ony wanted files also from unpacked packages", func() {
@@ -343,7 +409,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -357,12 +423,12 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 		})
 
 		It("Compiles includes and excludes ony wanted files also from unpacked packages", func() {
@@ -376,7 +442,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -390,12 +456,12 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 		})
 
 		It("Compiles and includes ony wanted files also from unpacked packages", func() {
@@ -409,7 +475,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -423,16 +489,16 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test2"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("lib/firmware"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test2"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("lib/firmware"))).ToNot(BeTrue())
 		})
 
 		It("Compiles a more complex tree", func() {
@@ -446,7 +512,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
@@ -461,18 +527,18 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("extra-layer"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("base-layer-0.1.metadata.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.metadata.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
 		It("Compiles with provides support", func() {
@@ -486,7 +552,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -502,19 +568,19 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("dd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("cd"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
 		})
 
 		It("Compiles with provides and selectors support", func() {
@@ -529,7 +595,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -545,19 +611,19 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("dd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("cd"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
 		})
 		It("Compiles revdeps", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -570,7 +636,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -585,16 +651,40 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(2))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("extra-layer"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
+		})
+
+		It("Generates a correct buildtree", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/complex/selection")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
+			Expect(err).ToNot(HaveOccurred())
+
+			bt, err := compiler.BuildTree(compilerspec.LuetCompilationspecs{*spec})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(bt.AllLevels()).To(Equal([]int{0, 1, 2, 3, 4, 5}))
+			Expect(bt.AllInLevel(0)).To(Equal([]string{"layer/build"}))
+			Expect(bt.AllInLevel(1)).To(Equal([]string{"layer/sabayon-build-portage"}))
+			Expect(bt.AllInLevel(2)).To(Equal([]string{"layer/build-sabayon-overlay"}))
+			Expect(bt.AllInLevel(3)).To(Equal([]string{"layer/build-sabayon-overlays"}))
+			Expect(bt.AllInLevel(4)).To(ContainElements("sys-kernel/linux-sabayon", "sys-kernel/sabayon-sources"))
+			Expect(bt.AllInLevel(5)).To(Equal([]string{"sys-fs-5.4.2/vhba"}))
 		})
 
 		It("Compiles complex dependencies trees with best matches", func() {
@@ -608,7 +698,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
 			Expect(err).ToNot(HaveOccurred())
@@ -623,17 +713,17 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].Dependencies)).To(Equal(6))
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("build-sabayon-overlays-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("linux-sabayon-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("sabayon-sources-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("vhba"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("build-sabayon-overlays-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("linux-sabayon-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("sabayon-sources-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("vhba"))).To(BeTrue())
 		})
 
 		It("Compiles revdeps with seeds", func() {
@@ -647,7 +737,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
@@ -658,31 +748,31 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(4))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			// A deps on B, so A artifacts are here:
-			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
 
 			// B
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("artifact42"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("artifact42"))).To(BeTrue())
 
 			// C depends on B, so B is here
-			content1, err := helpers.Read(spec.Rel("c"))
+			content1, err := fileHelper.Read(spec.Rel("c"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("cd"))
+			content2, err := fileHelper.Read(spec.Rel("cd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("c\n"))
 			Expect(content2).To(Equal("c\n"))
 
 			// D is here as it requires C, and C was recompiled
-			content1, err = helpers.Read(spec.Rel("d"))
+			content1, err = fileHelper.Read(spec.Rel("d"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err = helpers.Read(spec.Rel("dd"))
+			content2, err = fileHelper.Read(spec.Rel("dd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("s\n"))
 			Expect(content2).To(Equal("dd\n"))
@@ -699,7 +789,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(types.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -715,19 +805,19 @@ var _ = Describe("Compiler", func() {
 			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 				for _, d := range artifact.Dependencies {
-					Expect(helpers.Exists(d.Path)).To(BeTrue())
+					Expect(fileHelper.Exists(d.Path)).To(BeTrue())
 					Expect(helpers.Untar(d.Path, tmpdir, false)).ToNot(HaveOccurred())
 				}
 			}
 
-			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
 		})
 	})
@@ -741,7 +831,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -759,8 +849,8 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 			Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
 	})
 
@@ -773,7 +863,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase())
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{
 				Name:     "dironly",
@@ -807,13 +897,13 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts[0].Dependencies)).To(Equal(0))
 
 			Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test2"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test1"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test2"))).To(BeTrue())
 
 			Expect(helpers.Untar(spec2.Rel("dironly_filter-test-1.0.package.tar"), tmpdir2, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec2.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
 		})
 	})
 
@@ -826,7 +916,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -843,12 +933,12 @@ var _ = Describe("Compiler", func() {
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
-			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
-			Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
+			Expect(artifacts[0].Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			//	Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
 	})
 
@@ -861,7 +951,7 @@ var _ = Describe("Compiler", func() {
 			err := generalRecipe.Load("../../tests/fixtures/includeimage")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
 
 			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
 			Expect(err).ToNot(HaveOccurred())
@@ -880,7 +970,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -899,7 +989,7 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 			Expect(artifacts[0].Files).To(ContainElement("bin/busybox"))
 
-			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
 
 			art, err := LoadArtifactFromYaml(spec)
 			Expect(err).ToNot(HaveOccurred())

pkg/compiler/imagehashtree.go

@@ -0,0 +1,161 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"fmt"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	"github.com/pkg/errors"
+)
+
+// ImageHashTree is holding the Database
+// and the options to resolve PackageImageHashTrees
+// for a given specfile
+// It is responsible of returning a concrete result
+// which identifies a Package in a HashTree
+type ImageHashTree struct {
+	Database      pkg.PackageDatabase
+	SolverOptions types.LuetSolverOptions
+}
+
+// PackageImageHashTree represent the Package into a given image hash tree
+// The hash tree is constructed by a set of images representing
+// the package during its build stage. A Hash is assigned to each image
+// from the package fingerprint, plus the SAT solver assertion result (which is hashed as well)
+// and the specfile signatures. This guarantees that each image of the build stage
+// is unique and can be identified later on.
+type PackageImageHashTree struct {
+	Target                       *solver.PackageAssert
+	Dependencies                 solver.PackagesAssertions
+	Solution                     solver.PackagesAssertions
+	dependencyBuilderImageHashes map[string]string
+	SourceHash                   string
+	BuilderImageHash             string
+}
+
+func NewHashTree(db pkg.PackageDatabase) *ImageHashTree {
+	return &ImageHashTree{
+		Database: db,
+	}
+}
+
+func (ht *PackageImageHashTree) DependencyBuildImage(p pkg.Package) (string, error) {
+	found, ok := ht.dependencyBuilderImageHashes[p.GetFingerPrint()]
+	if !ok {
+		return "", errors.New("package hash not found")
+	}
+	return found, nil
+}
+
+func (ht *PackageImageHashTree) String() string {
+	return fmt.Sprintf(
+		"Target buildhash: %s\nTarget packagehash: %s\nBuilder Imagehash: %s\nSource Imagehash: %s\n",
+		ht.Target.Hash.BuildHash,
+		ht.Target.Hash.PackageHash,
+		ht.BuilderImageHash,
+		ht.SourceHash,
+	)
+}
+
+// Query takes a compiler and a compilation spec and returns a PackageImageHashTree tied to it.
+// PackageImageHashTree contains all the informations to resolve the spec build images in order to
+// reproducibly re-build images from packages
+func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) {
+	assertions, err := ht.resolve(cs, p)
+	if err != nil {
+		return nil, err
+	}
+	targetAssertion := assertions.Search(p.GetPackage().GetFingerPrint())
+
+	dependencies := assertions.Drop(p.GetPackage())
+	var sourceHash string
+	imageHashes := map[string]string{}
+	for _, assertion := range dependencies {
+		var depbuildImageTag string
+		compileSpec, err := cs.FromPackage(assertion.Package)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+		}
+		if compileSpec.GetImage() != "" {
+			depbuildImageTag = assertion.Hash.BuildHash
+		} else {
+			depbuildImageTag = ht.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash)
+		}
+		imageHashes[assertion.Package.GetFingerPrint()] = depbuildImageTag
+		sourceHash = assertion.Hash.PackageHash
+	}
+
+	return &PackageImageHashTree{
+		Dependencies:                 dependencies,
+		Target:                       targetAssertion,
+		SourceHash:                   sourceHash,
+		BuilderImageHash:             ht.genBuilderImageTag(p, targetAssertion.Hash.PackageHash),
+		dependencyBuilderImageHashes: imageHashes,
+		Solution:                     assertions,
+	}, nil
+}
+
+func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(packageImage))
+}
+
+// resolve computes the dependency tree of a compilation spec and returns solver assertions
+// in order to be able to compile the spec.
+func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
+	dependencies, err := cs.ComputeDepTree(p)
+	if err != nil {
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
+	}
+
+	// Get hash from buildpsecs
+	salts := map[string]string{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			spec, err := cs.FromPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "while computing hash buildspecs")
+			}
+			hash, err := spec.Hash()
+			if err != nil {
+				return nil, errors.Wrap(err, "failed computing hash")
+			}
+			salts[assertion.Package.GetFingerPrint()] = hash
+		}
+	}
+
+	assertions := solver.PackagesAssertions{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			nthsolution := dependencies.Cut(assertion.Package)
+			assertion.Hash = solver.PackageHash{
+				BuildHash:   nthsolution.SaltedHashFrom(assertion.Package, salts),
+				PackageHash: nthsolution.SaltedAssertionHash(salts),
+			}
+			assertion.Package.SetTreeDir(p.Package.GetTreeDir())
+			assertions = append(assertions, assertion)
+		}
+	}
+
+	return assertions, nil
+}

pkg/compiler/imagehashtree_test.go

@@ -0,0 +1,151 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler_test
+
+import (
+	"github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/mudler/luet/pkg/compiler"
+	sd "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ImageHashTree", func() {
+	ctx := types.NewContext()
+	generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+	compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+	hashtree := NewHashTree(generalRecipe.GetDatabase())
+	Context("Simple package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			err := generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+
+		It("Calculates the hash correctly", func() {
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(packageHash.Target.Hash.BuildHash).To(Equal("895697a8bb51b219b78ed081fa1b778801e81505bb03f56acafcf3c476620fc1"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("2a6c3dc0dd7af2902fd8823a24402d89b2030cfbea6e63fe81afb34af8b1a005"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-3a28d240f505d69123735a567beaf80e"))
+		})
+	})
+
+	expectedPackageHash := "f3f42a7435293225e92a51da8416f90b7c0ccd5958cd5c72276c39ece408c01f"
+
+	Context("complex package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(expectedPackageHash))
+			Expect(packageHash.SourceHash).To(Equal(expectedPackageHash))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-977129605c0d7e974cc8a431a563cec1"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("9112e2c97bf8ca998c1df303a9ebc4957b685930c882e9aa556eab4507220079"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal("b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+			Expect(assertionA.Hash.PackageHash).To(Equal(expectedPackageHash))
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+
+			Expect(assertionB.Hash.PackageHash).To(Equal("b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hashB).To(Equal("fc6fdd4bd62d51fc06c2c22e8bc56543727a2340220972594e28c623ea3a9c6c"))
+		})
+	})
+
+	Context("complex package definition, with small change in build.yaml", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			//Definition of A here is slightly changed in the steps build.yaml file (1 character only)
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision_content_changed")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).ToNot(Equal(expectedPackageHash))
+			sourceHash := "f0d96e79a0021b065674c58203158a02ed702ff29cc2ee6605e1f16f1fae9b89"
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(sourceHash))
+			Expect(packageHash.SourceHash).To(Equal(sourceHash))
+
+			Expect(packageHash.SourceHash).ToNot(Equal(expectedPackageHash))
+
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-05506adb3777ed74825254e8b46da5e9"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("489ece9825e18eba2de91e93293d5c61578d492d42bbfe9b6ae2e2c3c11bc842"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal("b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+
+			Expect(assertionA.Hash.PackageHash).To(Equal("f0d96e79a0021b065674c58203158a02ed702ff29cc2ee6605e1f16f1fae9b89"))
+			Expect(assertionA.Hash.PackageHash).ToNot(Equal(expectedPackageHash))
+
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+
+			Expect(assertionB.Hash.PackageHash).To(Equal("b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hashB).To(Equal("fc6fdd4bd62d51fc06c2c22e8bc56543727a2340220972594e28c623ea3a9c6c"))
+		})
+	})
+
+})

pkg/compiler/types/options/compiler_options.go

@@ -18,8 +18,8 @@ package options
 import (
 	"runtime"
 
+	"github.com/mudler/luet/pkg/api/core/types"
 	"github.com/mudler/luet/pkg/compiler/types/compression"
-	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/solver"
 )
 
@@ -33,7 +33,7 @@ type Compiler struct {
 	Wait            bool
 	OnlyDeps        bool
 	NoDeps          bool
-	SolverOptions   config.LuetSolverOptions
+	SolverOptions   types.LuetSolverOptions
 	BuildValuesFile []string
 	BuildValues     []map[string]interface{}
 
@@ -43,6 +43,11 @@ type Compiler struct {
 	BackendArgs []string
 
 	BackendType string
+
+	// TemplatesFolder. should default to tree/templates
+	TemplatesFolder []string
+
+	Context *types.Context
 }
 
 func NewDefaultCompiler() *Compiler {
@@ -55,7 +60,7 @@ func NewDefaultCompiler() *Compiler {
 		Concurrency:         runtime.NumCPU(),
 		OnlyDeps:            false,
 		NoDeps:              false,
-		SolverOptions:       config.LuetSolverOptions{Options: solver.Options{Concurrency: 1, Type: solver.SingleCoreSimple}},
+		SolverOptions:       types.LuetSolverOptions{Options: solver.Options{Concurrency: 1, Type: solver.SingleCoreSimple}},
 	}
 }
 
@@ -87,6 +92,13 @@ func WithBackendType(r string) func(cfg *Compiler) error {
 	}
 }
 
+func WithTemplateFolder(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.TemplatesFolder = r
+		return nil
+	}
+}
+
 func WithBuildValues(r []string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		cfg.BuildValuesFile = r
@@ -191,9 +203,16 @@ func WithCompressionType(t compression.Implementation) func(cfg *Compiler) error
 	}
 }
 
-func WithSolverOptions(c config.LuetSolverOptions) func(cfg *Compiler) error {
+func WithSolverOptions(c types.LuetSolverOptions) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		cfg.SolverOptions = c
 		return nil
 	}
 }
+
+func WithContext(c *types.Context) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Context = c
+		return nil
+	}
+}

pkg/compiler/types/spec/spec.go

@@ -16,15 +16,18 @@
 package compilerspec
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
 
+	"github.com/mitchellh/hashstructure/v2"
 	options "github.com/mudler/luet/pkg/compiler/types/options"
 
+	"github.com/ghodss/yaml"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/otiai10/copy"
-	yaml "gopkg.in/yaml.v2"
+	dirhash "golang.org/x/mod/sumdb/dirhash"
 )
 
 type LuetCompilationspecs []LuetCompilationSpec
@@ -85,6 +88,13 @@ func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
 	return &newSpecs
 }
 
+type CopyField struct {
+	Package     *pkg.DefaultPackage `json:"package"`
+	Image       string              `json:"image"`
+	Source      string              `json:"source"`
+	Destination string              `json:"destination"`
+}
+
 type LuetCompilationSpec struct {
 	Steps           []string                  `json:"steps"` // Are run inside a container and the result layer diff is saved
 	Env             []string                  `json:"env"`
@@ -103,15 +113,74 @@ type LuetCompilationSpec struct {
 	Excludes   []string `json:"excludes"`
 
 	BuildOptions *options.Compiler `json:"build_options"`
+
+	Copy []CopyField `json:"copy"`
+
+	RequiresFinalImages bool `json:"requires_final_images" yaml:"requires_final_images"`
+}
+
+// Signature is a portion of the spec that yields a signature for the hash
+type Signature struct {
+	Image               string
+	Steps               []string
+	PackageDir          string
+	Prelude             []string
+	Seed                string
+	Env                 []string
+	Retrieve            []string
+	Unpack              bool
+	Includes            []string
+	Excludes            []string
+	Copy                []CopyField
+	Requires            pkg.DefaultPackages
+	RequiresFinalImages bool
+}
+
+func (cs *LuetCompilationSpec) signature() Signature {
+	return Signature{
+		Image:               cs.Image,
+		Steps:               cs.Steps,
+		PackageDir:          cs.PackageDir,
+		Prelude:             cs.Prelude,
+		Seed:                cs.Seed,
+		Env:                 cs.Env,
+		Retrieve:            cs.Retrieve,
+		Unpack:              cs.Unpack,
+		Includes:            cs.Includes,
+		Excludes:            cs.Excludes,
+		Copy:                cs.Copy,
+		Requires:            cs.Package.GetRequires(),
+		RequiresFinalImages: cs.RequiresFinalImages,
+	}
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
 	var spec LuetCompilationSpec
+	var packageDefinition pkg.DefaultPackage
 	err := yaml.Unmarshal(b, &spec)
 	if err != nil {
 		return &spec, err
 	}
-	spec.Package = p.(*pkg.DefaultPackage)
+	err = yaml.Unmarshal(b, &packageDefinition)
+	if err != nil {
+		return &spec, err
+	}
+
+	// Update requires/conflict/provides
+	// When we have been passed a bytes slice, parse it as a package
+	// and updates requires/conflicts/provides.
+	// This is required in order to allow manipulation of such fields with templating
+	copy := *p.(*pkg.DefaultPackage)
+	spec.Package = &copy
+	if len(packageDefinition.GetRequires()) != 0 {
+		spec.Package.Requires(packageDefinition.GetRequires())
+	}
+	if len(packageDefinition.GetConflicts()) != 0 {
+		spec.Package.Conflicts(packageDefinition.GetConflicts())
+	}
+	if len(packageDefinition.GetProvides()) != 0 {
+		spec.Package.SetProvides(packageDefinition.GetProvides())
+	}
 	return &spec, nil
 }
 func (cs *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
@@ -196,7 +265,7 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 }
 
 func (cs *LuetCompilationSpec) EmptyPackage() bool {
-	return len(cs.BuildSteps()) == 0 && len(cs.GetPreBuildSteps()) == 0 && !cs.UnpackedPackage()
+	return len(cs.BuildSteps()) == 0 && !cs.UnpackedPackage()
 }
 
 func (cs *LuetCompilationSpec) UnpackedPackage() bool {
@@ -213,7 +282,21 @@ func (cs *LuetCompilationSpec) UnpackedPackage() bool {
 // a compilation spec has an image source when it depends on other packages or have a source image
 // explictly supplied
 func (cs *LuetCompilationSpec) HasImageSource() bool {
-	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != ""
+	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != "" || (cs.RequiresFinalImages && len(cs.Package.GetRequires()) != 0)
+}
+
+func (cs *LuetCompilationSpec) Hash() (string, error) {
+	// build a signature, we want to be part of the hash only the fields that are relevant for build purposes
+	signature := cs.signature()
+	h, err := hashstructure.Hash(signature, hashstructure.FormatV2, nil)
+	if err != nil {
+		return "", err
+	}
+	sum, err := dirhash.HashDir(cs.Package.Path, "", dirhash.DefaultHash)
+	if err != nil {
+		return fmt.Sprint(h), err
+	}
+	return fmt.Sprint(h, sum), err
 }
 
 func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
@@ -256,6 +339,13 @@ ADD ` + s + ` /luetbuild/`
 		}
 	}
 
+	for _, c := range cs.Copy {
+		if c.Image != "" {
+			copyLine := fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
+			spec = spec + copyLine
+		}
+	}
+
 	for _, s := range cs.Env {
 		spec = spec + `
 ENV ` + s

pkg/compiler/types/spec/spec_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compilerspec_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSpec(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Spec Suite")
+}

pkg/compiler/types/spec/spec_test.go

@@ -20,10 +20,11 @@ import (
 	"os"
 	"path/filepath"
 
+	options "github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	. "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -74,11 +75,67 @@ var _ = Describe("Spec", func() {
 		})
 	})
 
+	Context("Image hashing", func() {
+		It("is stable", func() {
+			spec1 := &compilerspec.LuetCompilationSpec{
+				Image:        "foo",
+				BuildOptions: &options.Compiler{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
+
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			spec2 := &compilerspec.LuetCompilationSpec{
+				Image:        "foo",
+				BuildOptions: &options.Compiler{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			spec3 := &compilerspec.LuetCompilationSpec{
+				Image: "foo",
+				Steps: []string{"foo"},
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			hash, err := spec1.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			hash2, err := spec2.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			hash3, err := spec3.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal(hash2))
+			hashagain, err := spec2.Hash()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hash).ToNot(Equal(hash3))
+			Expect(hash).To(Equal(hashagain))
+		})
+	})
+
 	Context("Simple package build definition", func() {
 		It("Loads it correctly", func() {
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
+			err := generalRecipe.Load("../../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
@@ -97,7 +154,7 @@ var _ = Describe("Spec", func() {
 			lspec.Env = []string{"test=1"}
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -110,7 +167,7 @@ ENV test=1`))
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
@@ -130,7 +187,7 @@ RUN echo bar > /test2`))
 	It("Renders retrieve and env fields", func() {
 		generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-		err := generalRecipe.Load("../../tests/fixtures/retrieve")
+		err := generalRecipe.Load("../../../../tests/fixtures/retrieve")
 		Expect(err).ToNot(HaveOccurred())
 
 		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
@@ -148,7 +205,7 @@ RUN echo bar > /test2`))
 
 		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
-		dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
 		Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -165,7 +222,7 @@ ENV test=1`))
 
 		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
-		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
 		Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -180,7 +237,7 @@ ENV test=1`))
 
 		err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
-		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
 
 		Expect(dockerfile).To(Equal(`

pkg/config/config.go

@@ -1,433 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//                  Daniele Rondina <geaaru@sabayonlinux.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package config
-
-import (
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/user"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"time"
-
-	"github.com/mudler/luet/pkg/helpers"
-	pkg "github.com/mudler/luet/pkg/package"
-	solver "github.com/mudler/luet/pkg/solver"
-
-	v "github.com/spf13/viper"
-)
-
-var LuetCfg = NewLuetConfig(v.GetViper())
-var AvailableResolvers = strings.Join([]string{solver.QLearningResolverType}, " ")
-
-type LuetLoggingConfig struct {
-	// Path of the logfile
-	Path string `mapstructure:"path"`
-	// Enable/Disable logging to file
-	EnableLogFile bool `mapstructure:"enable_logfile"`
-	// Enable JSON format logging in file
-	JsonFormat bool `mapstructure:"json_format"`
-
-	// Log level
-	Level string `mapstructure:"level"`
-
-	// Enable emoji
-	EnableEmoji bool `mapstructure:"enable_emoji"`
-	// Enable/Disable color in logging
-	Color bool `mapstructure:"color"`
-}
-
-type LuetGeneralConfig struct {
-	SameOwner       bool `mapstructure:"same_owner"`
-	Concurrency     int  `mapstructure:"concurrency"`
-	Debug           bool `mapstructure:"debug"`
-	ShowBuildOutput bool `mapstructure:"show_build_output"`
-	SpinnerMs       int  `mapstructure:"spinner_ms"`
-	SpinnerCharset  int  `mapstructure:"spinner_charset"`
-	FatalWarns      bool `mapstructure:"fatal_warnings"`
-}
-
-type LuetSolverOptions struct {
-	solver.Options
-	Type           string            `mapstructure:"type"`
-	LearnRate      float32           `mapstructure:"rate"`
-	Discount       float32           `mapstructure:"discount"`
-	MaxAttempts    int               `mapstructure:"max_attempts"`
-	Implementation solver.SolverType `mapstructure:"implementation"`
-}
-
-func (opts LuetSolverOptions) ResolverIsSet() bool {
-	switch opts.Type {
-	case solver.QLearningResolverType:
-		return true
-	default:
-		return false
-	}
-}
-
-func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
-	switch opts.Type {
-	case solver.QLearningResolverType:
-		if opts.LearnRate != 0.0 {
-			return solver.NewQLearningResolver(opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
-
-		}
-		return solver.SimpleQLearningSolver()
-	}
-
-	return &solver.DummyPackageResolver{}
-}
-
-func (opts *LuetSolverOptions) CompactString() string {
-	return fmt.Sprintf("type: %s rate: %f, discount: %f, attempts: %d, initialobserved: %d",
-		opts.Type, opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
-}
-
-type LuetSystemConfig struct {
-	DatabaseEngine string `yaml:"database_engine" mapstructure:"database_engine"`
-	DatabasePath   string `yaml:"database_path" mapstructure:"database_path"`
-	Rootfs         string `yaml:"rootfs" mapstructure:"rootfs"`
-	PkgsCachePath  string `yaml:"pkgs_cache_path" mapstructure:"pkgs_cache_path"`
-	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
-}
-
-func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
-	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
-	dbpath = filepath.Join(dbpath, "repos/"+name)
-	err := os.MkdirAll(dbpath, os.ModePerm)
-	if err != nil {
-		panic(err)
-	}
-	return dbpath
-}
-
-func (sc *LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
-	dbpath := filepath.Join(sc.Rootfs,
-		sc.DatabasePath)
-	err := os.MkdirAll(dbpath, os.ModePerm)
-	if err != nil {
-		panic(err)
-	}
-	return dbpath
-}
-
-func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
-	var cachepath string
-	if sc.PkgsCachePath != "" {
-		cachepath = sc.PkgsCachePath
-	} else {
-		// Create dynamic cache for test suites
-		cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
-	}
-
-	if filepath.IsAbs(cachepath) {
-		ans = cachepath
-	} else {
-		ans = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
-	}
-
-	return
-}
-
-func (sc *LuetSystemConfig) GetRootFsAbs() (string, error) {
-	return filepath.Abs(sc.Rootfs)
-}
-
-type LuetRepository struct {
-	Name           string            `json:"name" yaml:"name" mapstructure:"name"`
-	Description    string            `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
-	Urls           []string          `json:"urls" yaml:"urls" mapstructure:"urls"`
-	Type           string            `json:"type" yaml:"type" mapstructure:"type"`
-	Mode           string            `json:"mode,omitempty" yaml:"mode,omitempty" mapstructure:"mode,omitempty"`
-	Priority       int               `json:"priority,omitempty" yaml:"priority,omitempty" mapstructure:"priority"`
-	Enable         bool              `json:"enable" yaml:"enable" mapstructure:"enable"`
-	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
-	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
-	TreePath       string            `json:"treepath,omitempty" yaml:"treepath,omitempty" mapstructure:"treepath"`
-	MetaPath       string            `json:"metapath,omitempty" yaml:"metapath,omitempty" mapstructure:"metapath"`
-	Verify         bool              `json:"verify,omitempty" yaml:"verify,omitempty" mapstructure:"verify"`
-
-	// Serialized options not used in repository configuration
-
-	// Incremented value that identify revision of the repository in a user-friendly way.
-	Revision int `json:"revision,omitempty" yaml:"-,omitempty" mapstructure:"-,omitempty"`
-	// Epoch time in seconds
-	LastUpdate string `json:"last_update,omitempty" yaml:"-,omitempty" mapstructure:"-,omitempty"`
-}
-
-func NewLuetRepository(name, t, descr string, urls []string, priority int, enable, cached bool) *LuetRepository {
-	return &LuetRepository{
-		Name:        name,
-		Description: descr,
-		Urls:        urls,
-		Type:        t,
-		// Used in cached repositories
-		Mode:           "",
-		Priority:       priority,
-		Enable:         enable,
-		Cached:         cached,
-		Authentication: make(map[string]string, 0),
-		TreePath:       "",
-		MetaPath:       "",
-	}
-}
-
-func NewEmptyLuetRepository() *LuetRepository {
-	return &LuetRepository{
-		Name:           "",
-		Description:    "",
-		Urls:           []string{},
-		Type:           "",
-		Priority:       9999,
-		TreePath:       "",
-		MetaPath:       "",
-		Enable:         false,
-		Cached:         false,
-		Authentication: make(map[string]string, 0),
-	}
-}
-
-func (r *LuetRepository) String() string {
-	return fmt.Sprintf("[%s] prio: %d, type: %s, enable: %t, cached: %t",
-		r.Name, r.Priority, r.Type, r.Enable, r.Cached)
-}
-
-type LuetConfig struct {
-	Viper *v.Viper
-
-	Logging LuetLoggingConfig `mapstructure:"logging"`
-	General LuetGeneralConfig `mapstructure:"general"`
-	System  LuetSystemConfig  `mapstructure:"system"`
-	Solver  LuetSolverOptions `mapstructure:"solver"`
-
-	RepositoriesConfDir  []string         `mapstructure:"repos_confdir"`
-	ConfigProtectConfDir []string         `mapstructure:"config_protect_confdir"`
-	ConfigProtectSkip    bool             `mapstructure:"config_protect_skip"`
-	ConfigFromHost       bool             `mapstructure:"config_from_host"`
-	CacheRepositories    []LuetRepository `mapstructure:"repetitors"`
-	SystemRepositories   []LuetRepository `mapstructure:"repositories"`
-
-	ConfigProtectConfFiles []ConfigProtectConfFile
-}
-
-func NewLuetConfig(viper *v.Viper) *LuetConfig {
-	if viper == nil {
-		viper = v.New()
-	}
-
-	GenDefault(viper)
-	return &LuetConfig{Viper: viper, ConfigProtectConfFiles: nil}
-}
-
-func GenDefault(viper *v.Viper) {
-	viper.SetDefault("logging.level", "info")
-	viper.SetDefault("logging.enable_logfile", false)
-	viper.SetDefault("logging.path", "/var/log/luet.log")
-	viper.SetDefault("logging.json_format", false)
-	viper.SetDefault("logging.enable_emoji", true)
-	viper.SetDefault("logging.color", true)
-
-	viper.SetDefault("general.concurrency", runtime.NumCPU())
-	viper.SetDefault("general.debug", false)
-	viper.SetDefault("general.show_build_output", false)
-	viper.SetDefault("general.spinner_ms", 100)
-	viper.SetDefault("general.spinner_charset", 22)
-	viper.SetDefault("general.fatal_warnings", false)
-
-	u, err := user.Current()
-	// os/user doesn't work in from scratch environments
-	if err != nil || (u != nil && u.Uid == "0") {
-		viper.SetDefault("general.same_owner", true)
-	} else {
-		viper.SetDefault("general.same_owner", false)
-	}
-
-	viper.SetDefault("system.database_engine", "boltdb")
-	viper.SetDefault("system.database_path", "/var/cache/luet")
-	viper.SetDefault("system.rootfs", "/")
-	viper.SetDefault("system.tmpdir_base", filepath.Join(os.TempDir(), "tmpluet"))
-	viper.SetDefault("system.pkgs_cache_path", "packages")
-
-	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
-	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
-	viper.SetDefault("config_protect_skip", false)
-	// TODO: Set default to false when we are ready for migration.
-	viper.SetDefault("config_from_host", true)
-	viper.SetDefault("cache_repositories", []string{})
-	viper.SetDefault("system_repositories", []string{})
-
-	viper.SetDefault("solver.type", "")
-	viper.SetDefault("solver.rate", 0.7)
-	viper.SetDefault("solver.discount", 1.0)
-	viper.SetDefault("solver.max_attempts", 9000)
-}
-
-func (c *LuetConfig) GetSystemDB() pkg.PackageDatabase {
-	switch LuetCfg.GetSystem().DatabaseEngine {
-	case "boltdb":
-		return pkg.NewBoltDatabase(
-			filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
-	default:
-		return pkg.NewInMemoryDatabase(true)
-	}
-}
-
-func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
-	c.SystemRepositories = append(c.SystemRepositories, r)
-}
-
-func (c *LuetConfig) GetLogging() *LuetLoggingConfig {
-	return &c.Logging
-}
-
-func (c *LuetConfig) GetGeneral() *LuetGeneralConfig {
-	return &c.General
-}
-
-func (c *LuetConfig) GetSystem() *LuetSystemConfig {
-	return &c.System
-}
-
-func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
-	return &c.Solver
-}
-
-func (c *LuetConfig) GetConfigProtectConfFiles() []ConfigProtectConfFile {
-	return c.ConfigProtectConfFiles
-}
-
-func (c *LuetConfig) AddConfigProtectConfFile(file *ConfigProtectConfFile) {
-	if c.ConfigProtectConfFiles == nil {
-		c.ConfigProtectConfFiles = []ConfigProtectConfFile{*file}
-	} else {
-		c.ConfigProtectConfFiles = append(c.ConfigProtectConfFiles, *file)
-	}
-}
-
-func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
-	var ans *LuetRepository = nil
-
-	for idx, repo := range c.SystemRepositories {
-		if repo.Name == name {
-			ans = &c.SystemRepositories[idx]
-			break
-		}
-	}
-	if ans == nil {
-		return nil, errors.New("Repository " + name + " not found")
-	}
-
-	return ans, nil
-}
-
-func (c *LuetSolverOptions) String() string {
-	ans := fmt.Sprintf(`
-solver:
-  type: %s
-  rate: %f
-  discount: %f
-  max_attempts: %d`, c.Type, c.LearnRate, c.Discount,
-		c.MaxAttempts)
-
-	return ans
-}
-
-func (c *LuetGeneralConfig) String() string {
-	ans := fmt.Sprintf(`
-general:
-  concurrency: %d
-  same_owner: %t
-  debug: %t
-  fatal_warnings: %t
-  show_build_output: %t
-  spinner_ms: %d
-  spinner_charset: %d`, c.Concurrency, c.SameOwner, c.Debug,
-		c.FatalWarns, c.ShowBuildOutput,
-		c.SpinnerMs, c.SpinnerCharset)
-
-	return ans
-}
-
-func (c *LuetGeneralConfig) GetSpinnerMs() time.Duration {
-	duration, err := time.ParseDuration(fmt.Sprintf("%dms", c.SpinnerMs))
-	if err != nil {
-		return 100 * time.Millisecond
-	}
-	return duration
-}
-
-func (c *LuetLoggingConfig) SetLogLevel(s string) {
-	c.Level = s
-}
-
-func (c *LuetLoggingConfig) String() string {
-	ans := fmt.Sprintf(`
-logging:
-  enable_logfile: %t
-  path: %s
-  json_format: %t
-  color: %t
-  enable_emoji: %t
-  level: %s`, c.EnableLogFile, c.Path, c.JsonFormat,
-		c.Color, c.EnableEmoji, c.Level)
-
-	return ans
-}
-
-func (c *LuetSystemConfig) String() string {
-	ans := fmt.Sprintf(`
-system:
-  database_engine: %s
-  database_path: %s
-  pkgs_cache_path: %s
-  tmpdir_base: %s
-  rootfs: %s`,
-		c.DatabaseEngine, c.DatabasePath, c.PkgsCachePath,
-		c.TmpDirBase, c.Rootfs)
-
-	return ans
-}
-
-func (c *LuetSystemConfig) InitTmpDir() error {
-	if !helpers.Exists(c.TmpDirBase) {
-		return os.MkdirAll(c.TmpDirBase, os.ModePerm)
-	}
-	return nil
-}
-
-func (c *LuetSystemConfig) CleanupTmpDir() error {
-	return os.RemoveAll(c.TmpDirBase)
-}
-
-func (c *LuetSystemConfig) TempDir(pattern string) (string, error) {
-	err := c.InitTmpDir()
-	if err != nil {
-		return "", err
-	}
-	return ioutil.TempDir(c.TmpDirBase, pattern)
-}
-
-func (c *LuetSystemConfig) TempFile(pattern string) (*os.File, error) {
-	err := c.InitTmpDir()
-	if err != nil {
-		return nil, err
-	}
-	return ioutil.TempFile(c.TmpDirBase, pattern)
-}

pkg/config/config_test.go

@@ -1,66 +0,0 @@
-// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
-//                       Daniele Rondina <geaaru@sabayonlinux.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package config_test
-
-import (
-	"os"
-	"path/filepath"
-	"strings"
-
-	config "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Config", func() {
-
-	Context("Simple temporary directory creation", func() {
-
-		It("Create Temporary directory", func() {
-			// PRE: tmpdir_base contains default value.
-
-			tmpDir, err := config.LuetCfg.GetSystem().TempDir("test1")
-			Expect(err).ToNot(HaveOccurred())
-			Expect(strings.HasPrefix(tmpDir, filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
-			Expect(helpers.Exists(tmpDir)).To(BeTrue())
-
-			defer os.RemoveAll(tmpDir)
-		})
-
-		It("Create Temporary file", func() {
-			// PRE: tmpdir_base contains default value.
-
-			tmpFile, err := config.LuetCfg.GetSystem().TempFile("testfile1")
-			Expect(err).ToNot(HaveOccurred())
-			Expect(strings.HasPrefix(tmpFile.Name(), filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
-			Expect(helpers.Exists(tmpFile.Name())).To(BeTrue())
-
-			defer os.Remove(tmpFile.Name())
-		})
-
-		It("Config1", func() {
-			cfg := config.LuetCfg
-
-			cfg.GetLogging().Color = false
-			Expect(cfg.GetLogging().Color).To(BeFalse())
-		})
-
-	})
-
-})

pkg/helpers/archive.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 
 	"github.com/docker/docker/pkg/archive"
+	"github.com/pkg/errors"
 )
 
 func Tar(src, dest string) error {
@@ -90,8 +91,7 @@ func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modi
 	}
 
 	if sameOwner {
-		// PRE: i have root privileged.
-
+		// we do have root permissions, so we can extract keeping the same permissions.
 		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
 
 		opts := &archive.TarOptions{
@@ -132,7 +132,7 @@ func unTarIgnoreOwner(dest string, in io.ReadCloser, mods map[string]archive.Tar
 			if ok {
 				header, data, err = modifier(header.Name, header, tr)
 				if err != nil {
-					return err
+					return errors.Wrap(err, "running modifier wrapper")
 				}
 
 				// Override target path
@@ -158,7 +158,7 @@ func unTarIgnoreOwner(dest string, in io.ReadCloser, mods map[string]archive.Tar
 
 			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
 			if err != nil {
-				return err
+				return errors.Wrap(err, "creating destination")
 			}
 
 			// copy over contents
@@ -189,12 +189,11 @@ tarEof:
 }
 
 // Untar just a wrapper around the docker functions
-func Untar(src, dest string, sameOwner bool) error {
-	var ans error
+func Untar(src, dest string, sameOwner bool) (err error) {
 
 	in, err := os.Open(src)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "while opening "+src+" for untar ")
 	}
 	defer in.Close()
 
@@ -205,10 +204,14 @@ func Untar(src, dest string, sameOwner bool) error {
 			ContinueOnError: true,
 		}
 
-		ans = archive.Untar(in, dest, opts)
+		err = archive.Untar(in, dest, opts)
 	} else {
-		ans = unTarIgnoreOwner(dest, in, nil)
+		err = unTarIgnoreOwner(dest, in, nil)
 	}
 
-	return ans
+	if err != nil {
+		err = errors.Wrap(err, "while untarring "+src+" into "+dest)
+	}
+
+	return
 }

pkg/helpers/archive_test.go

@@ -25,6 +25,8 @@ import (
 	"os"
 	"path/filepath"
 
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	"github.com/docker/docker/pkg/archive"
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
@@ -128,7 +130,7 @@ var _ = Describe("Helpers Archive", func() {
 			err = archive.Untar(replacerArchive, targetDir, opts)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
+			Expect(fileHelper.Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
 		})
 	})
 })

pkg/helpers/docker/docker.go

@@ -13,20 +13,32 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package docker
 
 import (
 	"context"
 	"encoding/hex"
+	"fmt"
 	"os"
 	"strings"
 
+	"github.com/containerd/containerd/images"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
+	continerdarchive "github.com/containerd/containerd/archive"
 	"github.com/docker/cli/cli/trust"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/registry"
-	"github.com/mudler/luet/pkg/helpers/imgworker"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/mudler/luet/pkg/bus"
 	"github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/theupdateframework/notary/tuf/data"
 )
@@ -94,10 +106,67 @@ func trustedResolveDigest(ctx context.Context, ref reference.NamedTagged, authCo
 	return reference.WithDigest(ref, dgst)
 }
 
-// DownloadAndExtractDockerImage is a re-adaption
-// from genuinetools/img https://github.com/genuinetools/img/blob/54d0ca981c1260546d43961a538550eef55c87cf/pull.go
-func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) {
+type staticAuth struct {
+	auth *types.AuthConfig
+}
 
+func (s staticAuth) Authorization() (*authn.AuthConfig, error) {
+	if s.auth == nil {
+		return nil, nil
+	}
+	return &authn.AuthConfig{
+		Username:      s.auth.Username,
+		Password:      s.auth.Password,
+		Auth:          s.auth.Auth,
+		IdentityToken: s.auth.IdentityToken,
+		RegistryToken: s.auth.RegistryToken,
+	}, nil
+}
+
+// UnpackEventData is the data structure to pass for the bus events
+type UnpackEventData struct {
+	Image string
+	Dest  string
+}
+
+// UnarchiveLayers extract layers with archive.Untar from docker instead of containerd
+func UnarchiveLayers(temp string, img v1.Image, image, dest string, auth *types.AuthConfig, verify bool) (int64, error) {
+	layers, err := img.Layers()
+	if err != nil {
+		return 0, fmt.Errorf("reading layers from '%s' image failed: %v", image, err)
+	}
+	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	var size int64
+	for _, l := range layers {
+		s, err := l.Size()
+		if err != nil {
+			return 0, fmt.Errorf("reading layer size from '%s' image failed: %v", image, err)
+		}
+		size += s
+
+		layerReader, err := l.Uncompressed()
+		if err != nil {
+			return 0, fmt.Errorf("reading uncompressed layer from '%s' image failed: %v", image, err)
+		}
+		defer layerReader.Close()
+
+		// Unpack the tarfile to the rootfs path.
+		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
+		if err := archive.Untar(layerReader, dest, &archive.TarOptions{
+			NoLchown:        false,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+		}); err != nil {
+			return 0, fmt.Errorf("extracting '%s' image to directory %s failed: %v", image, dest, err)
+		}
+	}
+	bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	return size, nil
+}
+
+// DownloadAndExtractDockerImage extracts a container image natively. It supports privileged/unprivileged mode
+func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*images.Image, error) {
 	if verify {
 		img, err := verifyImage(image, auth)
 		if err != nil {
@@ -106,22 +175,59 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
 		image = img
 	}
 
+	if !fileHelper.Exists(dest) {
+		if err := os.MkdirAll(dest, os.ModePerm); err != nil {
+			return nil, errors.Wrapf(err, "cannot create destination directory")
+		}
+	}
+
+	ref, err := name.ParseReference(image)
+	if err != nil {
+		return nil, err
+	}
+
+	img, err := remote.Image(ref, remote.WithAuth(staticAuth{auth}))
+	if err != nil {
+		return nil, err
+	}
+
+	m, err := img.Manifest()
+	if err != nil {
+		return nil, err
+	}
+
+	mt, err := img.MediaType()
+	if err != nil {
+		return nil, err
+	}
+
+	d, err := img.Digest()
+	if err != nil {
+		return nil, err
+	}
+
+	reader := mutate.Extract(img)
+	defer reader.Close()
 	defer os.RemoveAll(temp)
-	c, err := imgworker.New(temp, auth)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed creating client")
-	}
-	defer c.Close()
 
-	listedImage, err := c.Pull(image)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed listing images")
+	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
 
+	c, err := continerdarchive.Apply(context.TODO(), dest, reader)
+	if err != nil {
+		return nil, err
 	}
 
-	os.RemoveAll(dest)
-	err = c.Unpack(image, dest)
-	return listedImage, err
+	bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	return &images.Image{
+		Name:   image,
+		Labels: m.Annotations,
+		Target: specs.Descriptor{
+			MediaType: string(mt),
+			Digest:    digest.Digest(d.String()),
+			Size:      c,
+		},
+	}, nil
 }
 
 func StripInvalidStringsFromImage(s string) string {

pkg/helpers/docker/docker_test.go

@@ -13,10 +13,10 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers_test
+package docker_test
 
 import (
-	. "github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/helpers/docker"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -24,7 +24,7 @@ import (
 var _ = Describe("StripInvalidStringsFromImage", func() {
 	Context("Image names", func() {
 		It("strips invalid chars", func() {
-			Expect(StripInvalidStringsFromImage("foo+bar")).To(Equal("foo-bar"))
+			Expect(docker.StripInvalidStringsFromImage("foo+bar")).To(Equal("foo-bar"))
 		})
 	})
 })

pkg/helpers/file/file.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package file
 
 import (
 	"fmt"
@@ -287,3 +287,15 @@ func CopyDir(src string, dst string) (err error) {
 		Sync:      true,
 		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }
+
+func Rel2Abs(s string) (string, error) {
+	pathToSet := s
+	if !filepath.IsAbs(s) {
+		abs, err := filepath.Abs(s)
+		if err != nil {
+			return "", err
+		}
+		pathToSet = abs
+	}
+	return pathToSet, nil
+}

pkg/helpers/file/file_test.go

@@ -13,14 +13,15 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers_test
+package file_test
 
 import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
-	. "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -28,8 +29,8 @@ import (
 var _ = Describe("Helpers", func() {
 	Context("Exists", func() {
 		It("Detect existing and not-existing files", func() {
-			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml")).To(BeTrue())
-			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
+			Expect(fileHelper.Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml")).To(BeTrue())
+			Expect(fileHelper.Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
 		})
 	})
 
@@ -38,15 +39,15 @@ var _ = Describe("Helpers", func() {
 			testDir, err := ioutil.TempDir(os.TempDir(), "test")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(testDir)
-			Expect(DirectoryIsEmpty(testDir)).To(BeTrue())
+			Expect(fileHelper.DirectoryIsEmpty(testDir)).To(BeTrue())
 		})
 		It("Detects directory with files", func() {
 			testDir, err := ioutil.TempDir(os.TempDir(), "test")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(testDir)
-			err = Touch(filepath.Join(testDir, "foo"))
+			err = fileHelper.Touch(filepath.Join(testDir, "foo"))
 			Expect(err).ToNot(HaveOccurred())
-			Expect(DirectoryIsEmpty(testDir)).To(BeFalse())
+			Expect(fileHelper.DirectoryIsEmpty(testDir)).To(BeFalse())
 		})
 	})
 
@@ -72,7 +73,7 @@ var _ = Describe("Helpers", func() {
 			err = ioutil.WriteFile(filepath.Join(testDir, "baz2", "foo"), []byte("test\n"), 0644)
 			Expect(err).ToNot(HaveOccurred())
 
-			ordered, notExisting := OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
+			ordered, notExisting := fileHelper.OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
 
 			Expect(ordered).To(Equal([]string{"baz", "bar/foo", "foo", "baz2/foo", "bar", "baz2"}))
 			Expect(notExisting).To(Equal([]string{"notexisting"}))
@@ -96,7 +97,7 @@ var _ = Describe("Helpers", func() {
 			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz", "fa"), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			ordered, _ := OrderFiles(testDir, []string{"foo", "foo/bar", "bar", "foo/baz/fa", "foo/baz"})
+			ordered, _ := fileHelper.OrderFiles(testDir, []string{"foo", "foo/bar", "bar", "foo/baz/fa", "foo/baz"})
 			Expect(ordered).To(Equal([]string{"foo/baz/fa", "foo/bar", "foo/baz", "foo", "bar"}))
 		})
 	})

pkg/helpers/helm.go

@@ -2,6 +2,10 @@ package helpers
 
 import (
 	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	"github.com/imdario/mergo"
 	"github.com/pkg/errors"
@@ -11,17 +15,86 @@ import (
 	"helm.sh/helm/v3/pkg/engine"
 )
 
+// ChartFileB is an helper that takes a slice of bytes and construct a chart.File slice from it
+func ChartFileB(s []byte) []*chart.File {
+	return []*chart.File{
+		{Name: "templates", Data: s},
+	}
+}
+
+// ChartFileS is an helper that takes a string and construct a chart.File slice from it
+func ChartFileS(s string) []*chart.File {
+	return []*chart.File{
+		{Name: "templates", Data: []byte(s)},
+	}
+}
+
+// ChartFile reads all the given files and returns a slice of []*chart.File
+// containing the raw content and the file name for each file
+func ChartFile(s ...string) []*chart.File {
+	files := []*chart.File{}
+	for _, c := range s {
+		raw, err := ioutil.ReadFile(c)
+		if err != nil {
+			return files
+		}
+		files = append(files, &chart.File{Name: c, Data: raw})
+	}
+
+	return files
+}
+
+// ChartFiles reads a list of paths and reads all yaml file inside. It returns a
+// slice of pointers of chart.File(s) with the raw content of the yaml
+func ChartFiles(path []string) ([]*chart.File, error) {
+	var chartFiles []*chart.File
+	for _, t := range path {
+		rel, err := fileHelper.Rel2Abs(t)
+		if err != nil {
+			return nil, err
+		}
+
+		if !fileHelper.Exists(rel) {
+			continue
+		}
+		files, err := fileHelper.ListDir(rel)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, f := range files {
+			if strings.ToLower(filepath.Ext(f)) == ".yaml" {
+				raw, err := ioutil.ReadFile(f)
+				if err != nil {
+					return nil, err
+				}
+				chartFiles = append(chartFiles, &chart.File{Name: f, Data: raw})
+			}
+		}
+	}
+	return chartFiles, nil
+}
+
 // RenderHelm renders the template string with helm
-func RenderHelm(template string, values, d map[string]interface{}) (string, error) {
+func RenderHelm(files []*chart.File, values, d map[string]interface{}) (string, error) {
+
+	// We slurp all the files into one here. This is not elegant, but still works.
+	// As a reminder, the files passed here have on the head the templates in the 'templates/' folder
+	// of each luet tree, and it have at the bottom the package buildpsec to be templated.
+	// TODO: Replace by correctly populating the files so that the helm render engine templates it
+	// correctly
+	toTemplate := ""
+	for _, f := range files {
+		toTemplate += string(f.Data)
+	}
+
 	c := &chart.Chart{
 		Metadata: &chart.Metadata{
 			Name:    "",
 			Version: "",
 		},
-		Templates: []*chart.File{
-			{Name: "templates", Data: []byte(template)},
-		},
-		Values: map[string]interface{}{"Values": values},
+		Templates: ChartFileS(toTemplate),
+		Values:    map[string]interface{}{"Values": values},
 	}
 
 	v, err := chartutil.CoalesceValues(c, map[string]interface{}{"Values": d})
@@ -69,23 +142,18 @@ func reverse(s []string) []string {
 	return s
 }
 
-func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string, error) {
-	raw, err := ioutil.ReadFile(toTemplate)
-	if err != nil {
-		return "", errors.Wrap(err, "reading file "+toTemplate)
-	}
-
-	if !Exists(valuesFile) {
-		return "", errors.Wrap(err, "file not existing "+valuesFile)
+func RenderFiles(files []*chart.File, valuesFile string, defaultFile ...string) (string, error) {
+	if !fileHelper.Exists(valuesFile) {
+		return "", errors.New("file does not exist: " + valuesFile)
 	}
 	val, err := ioutil.ReadFile(valuesFile)
 	if err != nil {
-		return "", errors.Wrap(err, "reading file "+valuesFile)
+		return "", errors.Wrap(err, "reading file: "+valuesFile)
 	}
 
 	var values templatedata
 	if err = yaml.Unmarshal(val, &values); err != nil {
-		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
+		return "", errors.Wrap(err, "unmarshalling values")
 	}
 
 	dst, err := UnMarshalValues(defaultFile)
@@ -93,5 +161,5 @@ func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string,
 		return "", errors.Wrap(err, "unmarshalling values")
 	}
 
-	return RenderHelm(string(raw), values, dst)
+	return RenderHelm(files, values, dst)
 }

pkg/helpers/helm_test.go

@@ -30,21 +30,21 @@ func writeFile(path string, content string) {
 	Expect(err).ToNot(HaveOccurred())
 }
 
-var _ = Describe("Helpers", func() {
+var _ = Describe("Helm", func() {
 	Context("RenderHelm", func() {
 		It("Renders templates", func() {
-			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
+			out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).To(Equal("foobar"))
 		})
 		It("Renders templates with overrides", func() {
-			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
+			out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).To(Equal("foobar"))
 		})
 
 		It("Renders templates", func() {
-			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
+			out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).To(Equal("foobar"))
 		})
@@ -68,7 +68,7 @@ foo: "baz"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("baz"))
 
@@ -93,7 +93,7 @@ faa: "baz"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bar"))
 
@@ -114,7 +114,7 @@ foo: "bar"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values)
+			res, err := RenderFiles(ChartFile(toTemplate), values)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bar"))
 		})
@@ -145,11 +145,11 @@ bar: "nei"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d2, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d2, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bazneif"))
 
-			res, err = RenderFiles(toTemplate, values, d, d2)
+			res, err = RenderFiles(ChartFile(toTemplate), values, d, d2)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("doneif"))
 		})
@@ -173,7 +173,7 @@ faa: "baz"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal(""))
 

pkg/helpers/helpers_suite_test.go

@@ -18,15 +18,11 @@ package helpers_test
 import (
 	"testing"
 
-	. "github.com/mudler/luet/cmd"
-	config "github.com/mudler/luet/pkg/config"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestSolver(t *testing.T) {
 	RegisterFailHandler(Fail)
-	LoadConfig(config.LuetCfg)
 	RunSpecs(t, "Helpers Suite")
 }

pkg/helpers/imgworker/auth.go

@@ -1,36 +0,0 @@
-package imgworker
-
-import (
-	"context"
-
-	"github.com/docker/docker/api/types"
-
-	"github.com/moby/buildkit/session"
-	"github.com/moby/buildkit/session/auth"
-	"google.golang.org/grpc"
-)
-
-func NewDockerAuthProvider(auth *types.AuthConfig) session.Attachable {
-	return &authProvider{
-		config: auth,
-	}
-}
-
-type authProvider struct {
-	config *types.AuthConfig
-}
-
-func (ap *authProvider) Register(server *grpc.Server) {
-	// no-op
-}
-
-func (ap *authProvider) Credentials(ctx context.Context, req *auth.CredentialsRequest) (*auth.CredentialsResponse, error) {
-	res := &auth.CredentialsResponse{}
-	if ap.config.IdentityToken != "" {
-		res.Secret = ap.config.IdentityToken
-	} else {
-		res.Username = ap.config.Username
-		res.Secret = ap.config.Password
-	}
-	return res, nil
-}