Tag 0.17.7

Fixes #239

.chglog/CHANGELOG.tpl.md

@@ -0,0 +1,49 @@
+{{ if .Versions -}}
+{{ if .Unreleased.CommitGroups -}}
+{{ range .Unreleased.CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ else }}
+{{ range .Unreleased.Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+
+{{ if .CommitGroups -}}
+{{ range .CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ else }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} (https://github.com/mudler/luet/commit/{{.Hash.Short}})
+{{ end }}
+{{ end -}}
+
+{{- if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+{{ range .Notes }}
+{{ .Body }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}
+
+{{- if .Versions }}
+[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD
+{{ range .Versions -}}
+{{ if .Tag.Previous -}}
+[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}
+{{ end -}}
+{{ end -}}
+{{ end -}}

.chglog/config.yml

@@ -0,0 +1,27 @@
+style: github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://github.com/mudler/luet
+options:
+  commits:
+    # filters:
+    #   Type:
+    #     - feat
+    #     - fix
+    #     - perf
+    #     - refactor
+  commit_groups:
+     title_maps:
+       feat: Features
+       fix: Bug Fixes
+       perf: Performance Improvements
+       refactor: Code Refactoring
+       ci: Continous Integration
+  header:
+    pattern: "(.*)"
+    pattern_maps:
+      - Subject
+  notes:
+    keywords:
+      - BREAKING CHANGE

.github/workflows/release.yml

@@ -1,8 +1,46 @@
 on: push
+concurrency:
+  group: ci-${{ github.head_ref || github.ref }}-${{ github.repository }}
+  cancel-in-progress: true
 name: Build and release on push
 jobs:
-  release:
-    name: Test and Release
+  tests-integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+      - name: Login to quay with img
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Tests with Img backend
+        run: |
+            sudo -E env "PATH=$PATH" \
+                    env "LUET_BACKEND=img" \
+                    make test-integration
+      - name: Tests
+        run: |
+              sudo -E \
+                    env "PATH=$PATH" \
+                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                    make test-integration
+
+  tests-unit:
+    name: Unit tests
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
@@ -22,13 +60,6 @@ jobs:
             sudo chmod a+x "/usr/bin/img"
       - name: Build test
         run: sudo -E env "PATH=$PATH" make multiarch-build-small
-      - name: Login to quay with img
-        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
-      - name: Tests with Img backend
-        run: |
-            sudo -E env "PATH=$PATH" \
-                    env "LUET_BACKEND=img" \
-                    make test-integration
       - name: Tests
         run: |
               sudo -E \
@@ -36,7 +67,28 @@ jobs:
                     env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
                     env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
                     env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
-                    make test-integration test-coverage
+                    make test-coverage
+
+  release:
+    name: Test and Release
+    needs: ["tests-integration","tests-unit"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
       - name: Build
         run: sudo -E env "PATH=$PATH" make multiarch-build-small && sudo chmod -R 777 release/
       - name: Release

.github/workflows/test.yml

@@ -2,7 +2,31 @@
 on: pull_request
 name: Build and Test
 jobs:
-  test:
+  tests-integration:
+    strategy:
+      matrix:
+        go-version: [1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+    - name: Tests with Img backend
+      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make test-integration
+  tests-unit:
     strategy:
       matrix:
         go-version: [1.14.x]
@@ -24,7 +48,5 @@ jobs:
             sudo chmod a+x "/usr/bin/img"
     - name: Build
       run: sudo -E env "PATH=$PATH" make multiarch-build-small
-    - name: Tests with Img backend
-      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
     - name: Tests
-      run: sudo -E env "PATH=$PATH" make test-integration test-coverage
+      run: sudo -E env "PATH=$PATH" make test-coverage

CONTRIBUTING.md

@@ -0,0 +1,53 @@
+
+We love your input! We want to make contributing to this project as easy and transparent as possible, whether it's:
+
+- Reporting a bug
+- Discussing the current state of the code
+- Submitting a fix
+- Proposing new features
+- Becoming a maintainer
+
+## We Develop with Github
+We use github to host code, to track issues and feature requests, as well as accept pull requests.
+
+## Stay in touch
+
+Join us in [slack](https://luet.slack.com/join/shared_invite/enQtOTQxMjcyNDQ0MDUxLWQ5ODVlNTI1MTYzNDRkYzkyYmM1YWE5YjM0NTliNDEzNmQwMTkxNDRhNDIzM2Y5NDBlOTZjZTYxYWQyNDE4YzY#/) and hang out with the community! It will be much easier to get started and do your first steps in contributing to the project.
+
+## All Code Changes Happen Through Pull Requests
+Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests:
+
+1. Fork the repo you want to contribute to and create your branch from `develop`.
+2. If you've added code that should be tested, add tests.
+3. If you've changed APIs, update the [documentation](https://github.com/Luet-lab/docs).
+4. Ensure the test suite passes.
+5. Make sure your code lints.
+6. Issue that pull request!
+
+## Any contributions you make will be under the Software License of the repository
+In short, when you submit code changes, your submissions are understood to be under the same License that covers the project. Feel free to contact the maintainers if that's a concern.
+
+## Report bugs using Github's [issues](https://github.com/mudler/luet/issues)
+We use GitHub issues to track public bugs. Report a bug by [opening a new issue](https://github.com/mudler/luet/issues/new); it's that easy!
+
+## Write bug reports with detail, background, and sample code
+Try to be as more descriptive as possible. When opening a new issue you will be prompted to choose between a bug or a feature request, with a small template to fill details with. Be specific!
+
+**Great Bug Reports** tend to have:
+
+- A quick summary and/or background
+- Steps to reproduce
+  - Be specific!
+  - Give sample code if you can. 
+- What you expected would happen
+- What actually happens
+- Notes (possibly including why you think this might be happening, or stuff you tried that didn't work)
+
+People *love* thorough bug reports.
+
+
+## License
+By contributing, you agree that your contributions will be licensed under the project Licenses.
+
+## References
+This document was adapted from the open-source contribution guidelines from https://gist.github.com/briandk/3d2e8b3ec8daf5a27a62

README.md

@@ -1,19 +1,24 @@
+
+<p align="center">
+  <img width=150 height=150 src="https://user-images.githubusercontent.com/2420543/119691600-0293d700-be4b-11eb-827f-49ff1174a07a.png">
+</p>
+
 # luet - Container-based Package manager
 
 [![Docker Repository on Quay](https://quay.io/repository/luet/base/status "Docker Repository on Quay")](https://quay.io/repository/luet/base)
-[![Go Report Card](https://goreportcard.com/badge/github.com/mudler/luet)](https://goreportcard.com/report/github.com/mudler/luet)
-[![Build Status](https://travis-ci.org/mudler/luet.svg?branch=master)](https://travis-ci.org/mudler/luet)
+[![Build and release on push](https://github.com/mudler/luet/actions/workflows/release.yml/badge.svg)](https://github.com/mudler/luet/actions/workflows/release.yml)
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
 [![codecov](https://codecov.io/gh/mudler/luet/branch/master/graph/badge.svg)](https://codecov.io/gh/mudler/luet)
 
-[![asciicast](https://asciinema.org/a/388348.svg)](https://asciinema.org/a/388348)
-
 Luet is a multi-platform Package Manager based off from containers - it uses Docker (and others) to build packages. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
 
 It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/) in YAML notation to define both [packages](https://luet-lab.github.io/docs/docs/concepts/packages/) and [rootfs](https://luet-lab.github.io/docs/docs/concepts/packages/#package-layers). As it is based on containers, it can be also used to build stages for Linux From Scratch installations and it can build and track updates for those systems.
 
 It is written entirely in Golang and where used as package manager, it can run in from scratch environment, with zero dependencies.
 
+[![asciicast](https://asciinema.org/a/388348.svg)](https://asciinema.org/a/388348)
+
+
 ## In a glance
 
 - Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.

cmd/build.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
@@ -74,7 +75,7 @@ Build packages specifying multiple definition trees:
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
 		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
-		viper.BindPFlag("values", cmd.Flags().Lookup("values"))
+		util.BindValuesFlags(cmd)
 		viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
 
 		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
@@ -83,10 +84,8 @@ Build packages specifying multiple definition trees:
 		viper.BindPFlag("wait", cmd.Flags().Lookup("wait"))
 		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
 
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSolverFlags(cmd)
+
 		LuetCfg.Viper.BindPFlag("general.show_build_output", cmd.Flags().Lookup("live-output"))
 		LuetCfg.Viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
 
@@ -102,7 +101,7 @@ Build packages specifying multiple definition trees:
 		all := viper.GetBool("all")
 		compressionType := viper.GetString("compression")
 		imageRepository := viper.GetString("image-repository")
-		values := viper.GetStringSlice("values")
+		values := util.ValuesFlags()
 		wait := viper.GetBool("wait")
 		push := viper.GetBool("push")
 		pull := viper.GetBool("pull")
@@ -148,21 +147,11 @@ Build packages specifying multiple definition trees:
 
 		Info("Building in", dst)
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		opts := util.SetSolverConfig()
 		pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
 		LuetCfg.GetGeneral().ShowBuildOutput = LuetCfg.Viper.GetBool("general.show_build_output")
 
-		opts := &LuetSolverOptions{
-			Type:        stype,
-			LearnRate:   float32(rate),
-			Discount:    float32(discount),
-			MaxAttempts: attempts,
-		}
-
 		Debug("Solver", opts.CompactString())
 
 		if concurrent {
@@ -171,6 +160,17 @@ Build packages specifying multiple definition trees:
 			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
 		}
 
+		templateFolders := []string{}
+		if !fromRepo {
+			for _, t := range treePaths {
+				templateFolders = append(templateFolders, filepath.Join(t, "templates"))
+			}
+		} else {
+			for _, s := range installer.SystemRepositories(LuetCfg) {
+				templateFolders = append(templateFolders, filepath.Join(s.TreePath, "templates"))
+			}
+		}
+
 		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
 			options.NoDeps(nodeps),
 			options.WithBackendType(backendType),
@@ -179,6 +179,7 @@ Build packages specifying multiple definition trees:
 			options.WithPullRepositories(pullRepo),
 			options.WithPushRepository(imageRepository),
 			options.Rebuild(rebuild),
+			options.WithTemplateFolder(templateFolders),
 			options.WithSolverOptions(*opts),
 			options.Wait(wait),
 			options.OnlyTarget(onlyTarget),

cmd/cleanup.go

@@ -20,9 +20,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
-	config "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/spf13/cobra"
@@ -33,21 +33,13 @@ var cleanupCmd = &cobra.Command{
 	Short: "Clean packages cache.",
 	Long:  `remove downloaded packages tarballs and clean cache directory`,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+		util.BindSystemFlags(cmd)
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var cleaned int = 0
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := config.LuetCfg.Viper.GetString("system.rootfs")
-		engine := config.LuetCfg.Viper.GetString("system.database_engine")
-
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig()
 		// Check if cache dir exists
-		if helpers.Exists(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
+		if fileHelper.Exists(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
 
 			files, err := ioutil.ReadDir(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
 			if err != nil {

cmd/database/create.go

@@ -18,6 +18,7 @@ package cmd_database
 import (
 	"io/ioutil"
 
+	"github.com/mudler/luet/cmd/util"
 	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 
 	. "github.com/mudler/luet/pkg/logger"
@@ -45,21 +46,11 @@ The yaml must contain the package definition, and the file list at least.
 For reference, inspect a "metadata.yaml" file generated while running "luet build"`,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 
-			dbpath := LuetCfg.Viper.GetString("system.database_path")
-			rootfs := LuetCfg.Viper.GetString("system.rootfs")
-			engine := LuetCfg.Viper.GetString("system.database_engine")
-
-			LuetCfg.System.DatabaseEngine = engine
-			LuetCfg.System.DatabasePath = dbpath
-			LuetCfg.System.Rootfs = rootfs
-
+			util.SetSystemConfig()
 			systemDB := LuetCfg.GetSystemDB()
 
 			for _, a := range args {
@@ -74,6 +65,12 @@ For reference, inspect a "metadata.yaml" file generated while running "luet buil
 
 				files := art.Files
 
+				// Check if the package is already present
+				if p, err := systemDB.FindPackage(art.CompileSpec.GetPackage()); err == nil && p.GetName() != "" {
+					Fatal("Package", art.CompileSpec.GetPackage().HumanReadableString(),
+						" already present.")
+				}
+
 				if _, err := systemDB.CreatePackage(art.CompileSpec.GetPackage()); err != nil {
 					Fatal("Failed to create ", a, ": ", err.Error())
 				}

cmd/database/get.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	"gopkg.in/yaml.v2"
 
 	. "github.com/mudler/luet/pkg/config"
@@ -38,20 +39,11 @@ To return also files:
 		$ luet database get --files system/foo`,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			showFiles, _ := cmd.Flags().GetBool("files")
-			dbpath := LuetCfg.Viper.GetString("system.database_path")
-			rootfs := LuetCfg.Viper.GetString("system.rootfs")
-			engine := LuetCfg.Viper.GetString("system.database_engine")
-
-			LuetCfg.System.DatabaseEngine = engine
-			LuetCfg.System.DatabasePath = dbpath
-			LuetCfg.System.Rootfs = rootfs
+			util.SetSystemConfig()
 
 			systemDB := LuetCfg.GetSystemDB()
 

cmd/database/remove.go

@@ -19,6 +19,7 @@ import (
 	. "github.com/mudler/luet/pkg/logger"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 
 	"github.com/spf13/cobra"
@@ -36,19 +37,10 @@ This commands takes multiple packages as arguments and prunes their entries from
 `,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			dbpath := LuetCfg.Viper.GetString("system.database_path")
-			rootfs := LuetCfg.Viper.GetString("system.rootfs")
-			engine := LuetCfg.Viper.GetString("system.database_engine")
-
-			LuetCfg.System.DatabaseEngine = engine
-			LuetCfg.System.DatabasePath = dbpath
-			LuetCfg.System.Rootfs = rootfs
+			util.SetSystemConfig()
 
 			systemDB := LuetCfg.GetSystemDB()
 

cmd/install.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -47,13 +48,8 @@ To force install a package:
 `,
 	Aliases: []string{"i"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
@@ -70,10 +66,6 @@ To force install a package:
 			toInstall = append(toInstall, pack)
 		}
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps := LuetCfg.Viper.GetBool("nodeps")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
@@ -81,18 +73,8 @@ To force install a package:
 		yes := LuetCfg.Viper.GetBool("yes")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
-
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
-
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		util.SetSystemConfig()
+		util.SetSolverConfig()
 
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple

cmd/reclaim.go

@@ -15,6 +15,7 @@
 package cmd
 
 import (
+	"github.com/mudler/luet/cmd/util"
 	installer "github.com/mudler/luet/pkg/installer"
 
 	. "github.com/mudler/luet/pkg/config"
@@ -27,9 +28,7 @@ var reclaimCmd = &cobra.Command{
 	Use:   "reclaim",
 	Short: "Reclaim packages to Luet database from available repositories",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+		util.BindSystemFlags(cmd)
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Long: `Reclaim tries to find association between packages in the online repositories and the system one.
@@ -39,13 +38,7 @@ var reclaimCmd = &cobra.Command{
 It scans the target file system, and if finds a match with a package available in the repositories, it marks as installed in the system database.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
-
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig()
 
 		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
 		repos := installer.Repositories{}

cmd/reinstall.go

@@ -0,0 +1,128 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+var reinstallCmd = &cobra.Command{
+	Use:   "reinstall <pkg1> <pkg2> <pkg3>",
+	Short: "reinstall a set of packages",
+	Long: `Reinstall a group of packages in the system:
+
+	$ luet reinstall -y system/busybox shells/bash system/coreutils ...
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("for", cmd.Flags().Lookup("for"))
+
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var toUninstall pkg.Packages
+		var toAdd pkg.Packages
+
+		force := LuetCfg.Viper.GetBool("force")
+		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
+
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		util.SetSystemConfig()
+
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toUninstall = append(toUninstall, pack)
+			toAdd = append(toAdd, pack)
+		}
+
+		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		util.SetSolverConfig()
+
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      true,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
+		})
+		inst.Repositories(repos)
+
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Swap(toUninstall, toAdd, system)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+
+	reinstallCmd.Flags().String("system-dbpath", "", "System db path")
+	reinstallCmd.Flags().String("system-target", "", "System rootpath")
+	reinstallCmd.Flags().String("system-engine", "", "System DB engine")
+
+	reinstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	reinstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	reinstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	reinstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	reinstallCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	reinstallCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	reinstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	reinstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	reinstallCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(reinstallCmd)
+}

cmd/replace.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -35,13 +36,8 @@ var replaceCmd = &cobra.Command{
 	$ luet replace -y system/busybox ... --for shells/bash --for system/coreutils ...
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
@@ -54,24 +50,15 @@ var replaceCmd = &cobra.Command{
 		var toAdd pkg.Packages
 
 		f := LuetCfg.Viper.GetStringSlice("for")
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps := LuetCfg.Viper.GetBool("nodeps")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
-
+		util.SetSystemConfig()
+		util.SetSolverConfig()
 		for _, a := range args {
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
@@ -98,11 +85,6 @@ var replaceCmd = &cobra.Command{
 			repos = append(repos, r)
 		}
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
 		} else {

cmd/root.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/marcsauter/single"
 	bus "github.com/mudler/luet/pkg/bus"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	extensions "github.com/mudler/cobra-extensions"
 	config "github.com/mudler/luet/pkg/config"
@@ -40,8 +41,14 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.16.4"
+	LuetCLIVersion = "0.17.7"
 	LuetEnvPrefix  = "LUET"
+	license        = `
+	Luet Copyright (C) 2019-2021 Ettore Di Giacinto
+	This program comes with ABSOLUTELY NO WARRANTY.
+	This is free software, and you are welcome to redistribute it
+	under certain conditions.
+	`
 )
 
 // Build time and commit information.
@@ -52,6 +59,47 @@ var (
 	BuildCommit string
 )
 
+func version() string {
+	return fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime)
+}
+
+var noBannerCommands = []string{"search", "exec", "tree", "database", "box", "cleanup"}
+
+func displayVersionBanner() {
+	display := true
+	if len(os.Args) > 1 {
+		for _, c := range noBannerCommands {
+			if os.Args[1] == c {
+				display = false
+			}
+		}
+	}
+	if display {
+		Info("Luet version", version())
+		Info(license)
+	}
+}
+
+func handleLock() {
+	if os.Getenv("LUET_NOLOCK") != "true" {
+		if len(os.Args) > 1 {
+			for _, lockedCmd := range LockedCommands {
+				if os.Args[1] == lockedCmd {
+					s := single.New("luet")
+					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
+						Fatal("another instance of the app is already running, exiting")
+					} else if err != nil {
+						// Another error occurred, might be worth handling it as well
+						Fatal("failed to acquire exclusive app lock:", err.Error())
+					}
+					defer s.TryUnlock()
+					break
+				}
+			}
+		}
+	}
+}
+
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
 	Use:   "luet",
@@ -79,8 +127,9 @@ To build a package, from a tree definition:
 	$ luet build --tree tree/path package
 	
 `,
-	Version: fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime),
+	Version: version(),
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+
 		err := LoadConfig(config.LuetCfg)
 		if err != nil {
 			Fatal("failed to load configuration:", err.Error())
@@ -154,23 +203,8 @@ func LoadConfig(c *config.LuetConfig) error {
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
 
-	if os.Getenv("LUET_NOLOCK") != "true" {
-		if len(os.Args) > 1 {
-			for _, lockedCmd := range LockedCommands {
-				if os.Args[1] == lockedCmd {
-					s := single.New("luet")
-					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
-						Fatal("another instance of the app is already running, exiting")
-					} else if err != nil {
-						// Another error occurred, might be worth handling it as well
-						Fatal("failed to acquire exclusive app lock:", err.Error())
-					}
-					defer s.TryUnlock()
-					break
-				}
-			}
-		}
-	}
+	handleLock()
+	displayVersionBanner()
 
 	if err := RootCmd.Execute(); err != nil {
 		fmt.Println(err)
@@ -252,7 +286,7 @@ func initConfig() {
 		}
 		homeDir := helpers.GetHomeDir()
 
-		if helpers.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && helpers.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
+		if fileHelper.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && fileHelper.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
 			viper.AddConfigPath(".")
 			if homeDir != "" {
 				viper.AddConfigPath(homeDir)

cmd/search.go

@@ -21,6 +21,7 @@ import (
 	"github.com/ghodss/yaml"
 	"github.com/jedib0t/go-pretty/table"
 	"github.com/jedib0t/go-pretty/v6/list"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -306,14 +307,9 @@ Search can also return results in the terminal in different ways: as terminal ou
 `,
 	Aliases: []string{"s"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var results Results
@@ -325,32 +321,20 @@ Search can also return results in the terminal in different ways: as terminal ou
 		hidden, _ := cmd.Flags().GetBool("hidden")
 
 		installed := LuetCfg.Viper.GetBool("installed")
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		searchWithLabel, _ := cmd.Flags().GetBool("by-label")
 		searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex")
 		revdeps, _ := cmd.Flags().GetBool("revdeps")
 		tableMode, _ := cmd.Flags().GetBool("table")
 		files, _ := cmd.Flags().GetBool("files")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig()
+		util.SetSolverConfig()
+
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
 			LuetCfg.GetLogging().SetLogLevel("error")
 		}
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-
 		l := list.NewWriter()
 		t := table.NewWriter()
 		t.AppendHeader(rows)

cmd/tree/images.go

@@ -23,6 +23,7 @@ import (
 	//. "github.com/mudler/luet/pkg/config"
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/compiler/types/options"
@@ -50,6 +51,7 @@ func NewTreeImageCommand() *cobra.Command {
 			if len(args) != 1 {
 				Fatal("Expects one package as parameter")
 			}
+			util.BindValuesFlags(cmd)
 			viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
 
 		},
@@ -59,6 +61,7 @@ func NewTreeImageCommand() *cobra.Command {
 			treePath, _ := cmd.Flags().GetStringArray("tree")
 			imageRepository := viper.GetString("image-repository")
 			pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
+			values := util.ValuesFlags()
 
 			out, _ := cmd.Flags().GetString("output")
 			if out != "terminal" {
@@ -80,6 +83,7 @@ func NewTreeImageCommand() *cobra.Command {
 			luetCompiler := compiler.NewLuetCompiler(
 				compilerBackend,
 				reciper.GetDatabase(),
+				options.WithBuildValues(values),
 				options.WithPushRepository(imageRepository),
 				options.WithPullRepositories(pullRepo),
 				options.WithSolverOptions(opts),

cmd/uninstall.go

@@ -16,6 +16,7 @@ package cmd
 
 import (
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -31,16 +32,11 @@ var uninstallCmd = &cobra.Command{
 	Long:    `Uninstall packages`,
 	Aliases: []string{"rm", "un"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		toRemove := []pkg.Package{}
@@ -53,10 +49,6 @@ var uninstallCmd = &cobra.Command{
 			toRemove = append(toRemove, pack)
 		}
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps, _ := cmd.Flags().GetBool("nodeps")
 		full, _ := cmd.Flags().GetBool("full")
@@ -64,18 +56,12 @@ var uninstallCmd = &cobra.Command{
 		fullClean, _ := cmd.Flags().GetBool("full-clean")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
+		keepProtected, _ := cmd.Flags().GetBool("keep-protected-files")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig()
+		util.SetSolverConfig()
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		LuetCfg.ConfigProtectSkip = !keepProtected
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
 		} else {
@@ -123,6 +109,7 @@ func init() {
 	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
 	uninstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	uninstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	uninstallCmd.Flags().BoolP("keep-protected-files", "k", false, "Keep package protected files around")
 
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -15,6 +15,7 @@
 package cmd
 
 import (
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -28,15 +29,10 @@ var upgradeCmd = &cobra.Command{
 	Short:   "Upgrades the system",
 	Aliases: []string{"u"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", installCmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", installCmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 	},
 	Long: `Upgrades packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
@@ -51,10 +47,6 @@ var upgradeCmd = &cobra.Command{
 			repos = append(repos, r)
 		}
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps, _ := cmd.Flags().GetBool("nodeps")
 		full, _ := cmd.Flags().GetBool("full")
@@ -63,18 +55,11 @@ var upgradeCmd = &cobra.Command{
 		sync, _ := cmd.Flags().GetBool("sync")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		util.SetSystemConfig()
+		util.SetSolverConfig()
+
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
 		} else {

cmd/util/cli.go

@@ -0,0 +1,74 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/config"
+)
+
+func BindSystemFlags(cmd *cobra.Command) {
+	LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+	LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+	LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+}
+
+func BindSolverFlags(cmd *cobra.Command) {
+	LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+	LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+	LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+	LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+}
+
+func BindValuesFlags(cmd *cobra.Command) {
+	viper.BindPFlag("values", cmd.Flags().Lookup("values"))
+}
+
+func ValuesFlags() []string {
+	return viper.GetStringSlice("values")
+}
+
+func SetSystemConfig() {
+	dbpath := LuetCfg.Viper.GetString("system.database_path")
+	rootfs := LuetCfg.Viper.GetString("system.rootfs")
+	engine := LuetCfg.Viper.GetString("system.database_engine")
+
+	LuetCfg.System.DatabaseEngine = engine
+	LuetCfg.System.DatabasePath = dbpath
+	LuetCfg.System.SetRootFS(rootfs)
+}
+
+func SetSolverConfig() (c *config.LuetSolverOptions) {
+	stype := LuetCfg.Viper.GetString("solver.type")
+	discount := LuetCfg.Viper.GetFloat64("solver.discount")
+	rate := LuetCfg.Viper.GetFloat64("solver.rate")
+	attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+
+	LuetCfg.GetSolverOptions().Type = stype
+	LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+	LuetCfg.GetSolverOptions().Discount = float32(discount)
+	LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+	return &config.LuetSolverOptions{
+		Type:        stype,
+		LearnRate:   float32(rate),
+		Discount:    float32(discount),
+		MaxAttempts: attempts,
+	}
+}

cmd/util/unpack.go

@@ -23,7 +23,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/go-units"
 	"github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/helpers/docker"
 	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/spf13/cobra"
@@ -77,7 +77,7 @@ func NewUnpackCommand() *cobra.Command {
 				RegistryToken: registryToken,
 			}
 
-			info, err := helpers.DownloadAndExtractDockerImage(temp, image, destination, auth, verify)
+			info, err := docker.DownloadAndExtractDockerImage(temp, image, destination, auth, verify)
 			if err != nil {
 				Error(err.Error())
 				os.Exit(1)

contrib/config/get_luet_root.sh

@@ -7,7 +7,7 @@ fi
 set -ex
 export LUET_NOLOCK=true
 
-LUET_VERSION=$(curl -s https://api.github.com/repos/mudler/luet/releases/latest | ( grep -oP '"tag_name": "\K(.*)(?=")' || echo "0.9.24" ))
+LUET_VERSION=$(curl -s https://api.github.com/repos/mudler/luet/releases/latest | grep tag_name | awk '{ print $2 }' | sed -e 's/\"//g' -e 's/,//g' || echo "0.9.24" )
 LUET_ROOTFS=${LUET_ROOTFS:-/}
 LUET_DATABASE_PATH=${LUET_DATABASE_PATH:-/var/luet/db}
 LUET_DATABASE_ENGINE=${LUET_DATABASE_ENGINE:-boltdb}

go.mod

@@ -10,10 +10,11 @@ require (
 	github.com/briandowns/spinner v1.12.1-0.20201220203425-e201aaea0a31
 	github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec
 	github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc
-	github.com/crillab/gophersat v1.3.2-0.20201023142334-3fc2ac466765
-	github.com/docker/cli v0.0.0-20200227165822-2298e6a3fe24
+	github.com/crillab/gophersat v1.3.2-0.20210701121804-72b19f5b6b38
+	github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
+	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
 	github.com/genuinetools/img v0.5.11
@@ -22,7 +23,7 @@ require (
 	github.com/google/renameio v1.0.0
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/hashicorp/go-version v1.2.1
-	github.com/imdario/mergo v0.3.8
+	github.com/imdario/mergo v0.3.9
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/jedib0t/go-pretty/v6 v6.0.5
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
@@ -55,8 +56,10 @@ require (
 	go.uber.org/atomic v1.5.1 // indirect
 	go.uber.org/multierr v1.4.0
 	go.uber.org/zap v1.13.0
+	golang.org/x/mod v0.4.2
 	google.golang.org/grpc v1.29.1
 	gopkg.in/yaml.v2 v2.3.0
+	gotest.tools/v3 v3.0.2 // indirect
 	helm.sh/helm/v3 v3.3.4
 
 )

pkg/box/exec.go

@@ -23,9 +23,9 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/pkg/errors"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
-	helpers "github.com/mudler/luet/pkg/helpers"
+	"github.com/pkg/errors"
 )
 
 type Box interface {
@@ -107,7 +107,7 @@ func (b *DefaultBox) Exec() error {
 
 func (b *DefaultBox) Run() error {
 
-	if !helpers.Exists(b.Root) {
+	if !fileHelper.Exists(b.Root) {
 		return errors.New(b.Root + " does not exist")
 	}
 

pkg/bus/events.go

@@ -1,9 +1,8 @@
 package bus
 
 import (
-	. "github.com/mudler/luet/pkg/logger"
-
 	"github.com/mudler/go-pluggable"
+	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -46,6 +45,13 @@ var (
 	EventRepositoryPreBuild pluggable.EventType = "repository.pre.build"
 	// EventRepositoryPostBuild is the event fired after a repository was built
 	EventRepositoryPostBuild pluggable.EventType = "repository.post.build"
+
+	// Image unpack
+
+	// EventImagePreUnPack is the event fired before unpacking an image to a local dir
+	EventImagePreUnPack pluggable.EventType = "image.pre.unpack"
+	// EventImagePostUnPack is the event fired after unpacking an image to a local dir
+	EventImagePostUnPack pluggable.EventType = "image.post.unpack"
 )
 
 // Manager is the bus instance manager, which subscribes plugins to events emitted by Luet
@@ -66,6 +72,8 @@ var Manager *Bus = &Bus{
 			EventImagePostBuild,
 			EventImagePostPull,
 			EventImagePostPush,
+			EventImagePreUnPack,
+			EventImagePostUnPack,
 		},
 	),
 }
@@ -80,9 +88,9 @@ func (b *Bus) Initialize(plugin ...string) {
 	for _, e := range b.Manager.Events {
 		b.Manager.Response(e, func(p *pluggable.Plugin, r *pluggable.EventResponse) {
 			if r.Errored() {
-				Fatal("Plugin", p.Name, "at", p.Executable, "Error", r.Error)
+				logrus.Fatal("Plugin", p.Name, "at", p.Executable, "Error", r.Error)
 			}
-			Debug(
+			logrus.Debug(
 				"plugin_event",
 				"received from",
 				p.Name,

pkg/compiler/backend/simpledocker.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	bus "github.com/mudler/luet/pkg/bus"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	capi "github.com/mudler/docker-companion/api"
 
@@ -237,7 +238,7 @@ func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
 		return errors.Wrap(err, "Error met while unpacking rootfs")
 	}
 
-	manifest, err := helpers.Read(filepath.Join(rootfs, "manifest.json"))
+	manifest, err := fileHelper.Read(filepath.Join(rootfs, "manifest.json"))
 	if err != nil {
 		return errors.Wrap(err, "Error met while reading image manifest")
 	}

pkg/compiler/backend/simpledocker_test.go

@@ -21,12 +21,12 @@ import (
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
-	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -60,7 +60,7 @@ var _ = Describe("Docker backend", func() {
 
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -79,11 +79,11 @@ ENV PACKAGE_CATEGORY=app-admin`))
 
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
@@ -103,7 +103,7 @@ RUN echo bar > /test2`))
 
 			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
 			artifacts := []artifact.ArtifactNode{{
 				Name: "/luetbuild/LuetDockerfile",
@@ -132,7 +132,7 @@ RUN echo bar > /test2`))
 			}
 
 			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
 			Expect(b.ImageExists(opts2.ImageName)).To(BeFalse())
 		})
 

pkg/compiler/compiler.go

@@ -36,11 +36,13 @@ import (
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
+	"helm.sh/helm/v3/pkg/chart"
 )
 
 const BuildFile = "build.yaml"
@@ -319,7 +321,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	defer os.RemoveAll(buildDir) // clean up
 
 	// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
-	err = helpers.CopyDir(p.GetPackage().GetPath(), buildDir)
+	err = fileHelper.CopyDir(p.GetPackage().GetPath(), buildDir)
 	if err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not copy package sources")
 	}
@@ -749,13 +751,13 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS
 		p.BuildOptions.PullImageRepository = append(p.BuildOptions.PullImageRepository, cs.Options.PullImageRepository...)
 		Debug("Inheriting pull repository from PullImageRepository buildoptions", p.BuildOptions.PullImageRepository)
 	}
+
 	Debug(p.GetPackage().HumanReadableString(), "Build options after inherit", p.BuildOptions)
 }
 
 func (cs *LuetCompiler) getSpecHash(pkgs pkg.DefaultPackages, salt string) (string, error) {
 	ht := NewHashTree(cs.Database)
 	overallFp := ""
-
 	for _, p := range pkgs {
 		compileSpec, err := cs.FromPackage(p)
 		if err != nil {
@@ -773,22 +775,31 @@ func (cs *LuetCompiler) getSpecHash(pkgs pkg.DefaultPackages, salt string) (stri
 	return fmt.Sprintf("%x", h.Sum(nil)), nil
 }
 
-func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
 
-	joinTag := ">:loop: join<"
-	if len(p.Join) != 0 {
-		Info(joinTag, "Generating a joint parent image from final packages")
+	joinTag := ">:loop: final images<"
+	var fromPackages pkg.DefaultPackages
+
+	if len(p.Join) > 0 {
+		fromPackages = p.Join
+		Warning(joinTag, `
+	Attention! the 'join' keyword is going to be deprecated in Luet >=0.18.x. 
+	Use 'requires_final_images: true' instead in the build.yaml file`)
+	} else if p.RequiresFinalImages {
+		Info(joinTag, "Generating a parent image from final packages")
+		fromPackages = p.Package.GetRequires()
 	} else {
+		// No source image to resolve
 		return nil
 	}
 
 	// First compute a hash and check if image is available. if it is, then directly consume that
-	overallFp, err := cs.getSpecHash(p.Join, "join")
+	overallFp, err := cs.getSpecHash(fromPackages, "join")
 	if err != nil {
 		return errors.Wrap(err, "could not generate image hash")
 	}
 
-	Info(joinTag, "Searching existing image with hash ", overallFp)
+	Info(joinTag, "Searching existing image with hash", overallFp)
 
 	image := cs.findImageHash(overallFp, p)
 	if image != "" {
@@ -810,12 +821,12 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 	}
 	defer os.RemoveAll(joinDir) // clean up
 
-	for _, p := range p.Join { //highly dependent on the order
+	for _, p := range fromPackages {
 		Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
 	}
 
 	current := 0
-	for _, c := range p.Join {
+	for _, c := range fromPackages {
 		current++
 		if c != nil && c.Name != "" && c.Version != "" {
 			joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(p.Join), c.HumanReadableString())
@@ -923,7 +934,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 
 	//Before multistage : join - same as multistage, but keep artifacts, join them, create a new one and generate a final image.
 	// When the image is there, use it as a source here, in place of GetImage().
-	if err := cs.resolveJoinImages(concurrency, keepPermissions, p); err != nil {
+	if err := cs.resolveFinalImages(concurrency, keepPermissions, p); err != nil {
 		return nil, errors.Wrap(err, "while resolving join images")
 	}
 
@@ -1027,7 +1038,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 				Assert:      assertion,
 			})
 
-			if err := cs.resolveJoinImages(concurrency, keepPermissions, compileSpec); err != nil {
+			if err := cs.resolveFinalImages(concurrency, keepPermissions, compileSpec); err != nil {
 				return nil, errors.Wrap(err, "while resolving join images")
 			}
 
@@ -1118,6 +1129,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 type templatedata map[string]interface{}
 
 func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.Package, dst templatedata) ([]byte, error) {
+	// Grab shared templates first
+	var chartFiles []*chart.File
+	if len(cs.Options.TemplatesFolder) != 0 {
+		c, err := helpers.ChartFiles(cs.Options.TemplatesFolder)
+		if err == nil {
+			chartFiles = c
+		}
+	}
 
 	var dataresult []byte
 	val := pack.Rel(DefinitionFile)
@@ -1155,7 +1174,7 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
 			return nil, errors.Wrap(err, "merging values maps")
 		}
 
-		dat, err := helpers.RenderHelm(string(dataBuild), td, dst)
+		dat, err := helpers.RenderHelm(append(chartFiles, helpers.ChartFileB(dataBuild)...), td, dst)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}
@@ -1173,14 +1192,20 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
 				if err != nil {
 					return nil, errors.Wrap(err, "while marshalling values file")
 				}
-				f := filepath.Join(valuesdir, helpers.RandStringRunes(20))
+				f := filepath.Join(valuesdir, fileHelper.RandStringRunes(20))
 				if err := ioutil.WriteFile(f, out, os.ModePerm); err != nil {
 					return nil, errors.Wrap(err, "while writing temporary values file")
 				}
 				bv = append([]string{f}, bv...)
 			}
 		}
-		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, bv...)
+
+		raw, err := ioutil.ReadFile(pack.Rel(BuildFile))
+		if err != nil {
+			return nil, err
+		}
+
+		out, err := helpers.RenderFiles(append(chartFiles, helpers.ChartFileB(raw)...), val, bv...)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}
@@ -1246,6 +1271,11 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio
 
 	cs.inheritSpecBuildOptions(newSpec)
 
+	// Update the package in the compiler database to catch updates from NewLuetCompilationSpec
+	if err := cs.Database.UpdatePackage(newSpec.Package); err != nil {
+		return nil, errors.Wrap(err, "failed updating new package entry in compiler database")
+	}
+
 	return newSpec, err
 }
 

pkg/compiler/compiler_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	helpers "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -59,15 +60,15 @@ var _ = Describe("Compiler", func() {
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
@@ -99,11 +100,11 @@ var _ = Describe("Compiler", func() {
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("result"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("bina/busybox"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("result"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("bina/busybox"))).To(BeTrue())
 		})
 
 		It("Compiles it correctly with Join", func() {
@@ -129,11 +130,11 @@ var _ = Describe("Compiler", func() {
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("newc"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
 		})
 	})
 
@@ -164,7 +165,7 @@ var _ = Describe("Compiler", func() {
 			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
@@ -227,23 +228,23 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(3))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
-			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("c"))
+			content1, err := fileHelper.Read(spec.Rel("c"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("cd"))
+			content2, err := fileHelper.Read(spec.Rel("cd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("c\n"))
 			Expect(content2).To(Equal("c\n"))
 
-			content1, err = helpers.Read(spec.Rel("d"))
+			content1, err = fileHelper.Read(spec.Rel("d"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err = helpers.Read(spec.Rel("dd"))
+			content2, err = fileHelper.Read(spec.Rel("dd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("s\n"))
 			Expect(content2).To(Equal("dd\n"))
@@ -277,17 +278,17 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts2)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			for _, artifact := range artifacts2 {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
-			Expect(helpers.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test1"))).To(BeTrue())
 		})
 
 		It("Compiles and includes ony wanted files", func() {
@@ -316,12 +317,12 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
 		It("Compiles and excludes files", func() {
@@ -350,13 +351,13 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 		})
 
 		It("Compiles includes and excludes files", func() {
@@ -385,13 +386,13 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
 		It("Compiles and excludes ony wanted files also from unpacked packages", func() {
@@ -419,12 +420,12 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 		})
 
 		It("Compiles includes and excludes ony wanted files also from unpacked packages", func() {
@@ -452,12 +453,12 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 		})
 
 		It("Compiles and includes ony wanted files also from unpacked packages", func() {
@@ -485,16 +486,16 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test2"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel("lib/firmware"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test2"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("lib/firmware"))).ToNot(BeTrue())
 		})
 
 		It("Compiles a more complex tree", func() {
@@ -523,18 +524,18 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("extra-layer"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("base-layer-0.1.metadata.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.metadata.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
 		It("Compiles with provides support", func() {
@@ -564,19 +565,19 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("dd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("cd"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
 		})
 
 		It("Compiles with provides and selectors support", func() {
@@ -607,19 +608,19 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("dd"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("cd"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
 		})
 		It("Compiles revdeps", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -647,16 +648,16 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(2))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("extra-layer"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer"))).To(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
 		It("Compiles complex dependencies trees with best matches", func() {
@@ -685,17 +686,17 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].Dependencies)).To(Equal(6))
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("build-sabayon-overlays-layer-0.1.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("linux-sabayon-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("sabayon-sources-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("vhba"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("build-sabayon-overlays-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("linux-sabayon-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("sabayon-sources-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("vhba"))).To(BeTrue())
 		})
 
 		It("Compiles revdeps with seeds", func() {
@@ -720,31 +721,31 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(4))
 
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			// A deps on B, so A artifacts are here:
-			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
 
 			// B
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("artifact42"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("artifact42"))).To(BeTrue())
 
 			// C depends on B, so B is here
-			content1, err := helpers.Read(spec.Rel("c"))
+			content1, err := fileHelper.Read(spec.Rel("c"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("cd"))
+			content2, err := fileHelper.Read(spec.Rel("cd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("c\n"))
 			Expect(content2).To(Equal("c\n"))
 
 			// D is here as it requires C, and C was recompiled
-			content1, err = helpers.Read(spec.Rel("d"))
+			content1, err = fileHelper.Read(spec.Rel("d"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err = helpers.Read(spec.Rel("dd"))
+			content2, err = fileHelper.Read(spec.Rel("dd"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("s\n"))
 			Expect(content2).To(Equal("dd\n"))
@@ -777,19 +778,19 @@ var _ = Describe("Compiler", func() {
 			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
-				Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 				for _, d := range artifact.Dependencies {
-					Expect(helpers.Exists(d.Path)).To(BeTrue())
+					Expect(fileHelper.Exists(d.Path)).To(BeTrue())
 					Expect(helpers.Untar(d.Path, tmpdir, false)).ToNot(HaveOccurred())
 				}
 			}
 
-			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
 		})
 	})
@@ -821,8 +822,8 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 			Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
 	})
 
@@ -869,13 +870,13 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts[0].Dependencies)).To(Equal(0))
 
 			Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test2"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test1"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test2"))).To(BeTrue())
 
 			Expect(helpers.Untar(spec2.Rel("dironly_filter-test-1.0.package.tar"), tmpdir2, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec2.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
 		})
 	})
 
@@ -905,12 +906,12 @@ var _ = Describe("Compiler", func() {
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
-			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
+			Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
 			Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred())
 			//	Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
 	})
 
@@ -961,7 +962,7 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
 			Expect(artifacts[0].Files).To(ContainElement("bin/busybox"))
 
-			Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
 
 			art, err := LoadArtifactFromYaml(spec)
 			Expect(err).ToNot(HaveOccurred())

pkg/compiler/imagehashtree_test.go

@@ -46,13 +46,13 @@ var _ = Describe("ImageHashTree", func() {
 
 			packageHash, err := hashtree.Query(compiler, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(packageHash.Target.Hash.BuildHash).To(Equal("4db24406e8db30a3310a1cf8c4d4e19597745e6d41b189dc51a73ac4a50cc9e6"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("4c867c9bab6c71d9420df75806e7a2f171dbc08487852ab4e2487bab04066cf2"))
-			Expect(packageHash.BuilderImageHash).To(Equal("builder-e6f9c5552a67c463215b0a9e4f7c7fc8"))
+			Expect(packageHash.Target.Hash.BuildHash).To(Equal("53993e5a02da4c21ad845371c872f5836fe45ff3a4e3c5ccb6296d0faee2b107"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("a786d3fd29d0b8bdfe5f304c8bf8be909d5c764cd7059c0e63294a8bff17f3ef"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-0cd3c0d07fc9be568377b3bf1b699e06"))
 		})
 	})
 
-	expectedPackageHash := "15811d83d0f8360318c54d91dcae3714f8efb39bf872572294834880f00ee7a8"
+	expectedPackageHash := "0d568ac04c4ca528a4e5b67978f2ad3a75d31d443ab20f9d7683b9608cc0d494"
 
 	Context("complex package definition", func() {
 		BeforeEach(func() {
@@ -73,23 +73,26 @@ var _ = Describe("ImageHashTree", func() {
 
 			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(expectedPackageHash))
 			Expect(packageHash.SourceHash).To(Equal(expectedPackageHash))
-			Expect(packageHash.BuilderImageHash).To(Equal("builder-3d739cab442aec15a6da238481df73c5"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-0f45c345f59103e84fc8bebbf02f2e2b"))
 
 			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("99c4ebb4bc4754985fcc28677badf90f525aa231b1db0fe75659f11b86dc20e8"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("2e8159583ac825acada763358290cfbea919a33873a926cab84f4f1a67ecf111"))
 			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
 			hash, err := packageHash.DependencyBuildImage(a)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(hash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+
+			Expect(hash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))
 
 			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
 			Expect(assertionA.Hash.PackageHash).To(Equal(expectedPackageHash))
 			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
 			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
-			Expect(assertionB.Hash.PackageHash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+
+			Expect(assertionB.Hash.PackageHash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))
 			hashB, err := packageHash.DependencyBuildImage(b)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(hashB).To(Equal("2668e418eab6861404834ad617713e39b8e58f68016a1fbfcc9384efdd037376"))
+
+			Expect(hashB).To(Equal("315075265aeb2e3c04c5428d31911f53c194ec9fa3db1421e8478f44b1e0def8"))
 		})
 	})
 
@@ -112,33 +115,35 @@ var _ = Describe("ImageHashTree", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).ToNot(Equal(expectedPackageHash))
-			sourceHash := "1d91b13d0246fa000085a1071c63397d21546300b17f69493f22315a64b717d4"
+			sourceHash := "66ec001fe72052d0e605ca96f607ae39ea4f8b53f0b7f762e377622d9c654de3"
 			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(sourceHash))
 			Expect(packageHash.SourceHash).To(Equal(sourceHash))
+
 			Expect(packageHash.SourceHash).ToNot(Equal(expectedPackageHash))
 
-			Expect(packageHash.BuilderImageHash).To(Equal("builder-03ee108a7c56b17ee568ace0800dd16d"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-ffc02fd8aaa916d0e17249885b3226b1"))
 
 			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("7677da23b2cc866c2d07aa4a58fbf703340f2f78c0efbb1ba9faf8979f250c87"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("b9c0286ebf6d28be831926ec7da9cb3cda6b489722d656aefc363ebd7173f937"))
 			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
 			hash, err := packageHash.DependencyBuildImage(a)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(hash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+
+			Expect(hash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))
 
 			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
 
-			Expect(assertionA.Hash.PackageHash).To(Equal("1d91b13d0246fa000085a1071c63397d21546300b17f69493f22315a64b717d4"))
+			Expect(assertionA.Hash.PackageHash).To(Equal("66ec001fe72052d0e605ca96f607ae39ea4f8b53f0b7f762e377622d9c654de3"))
 			Expect(assertionA.Hash.PackageHash).ToNot(Equal(expectedPackageHash))
 
 			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
 			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
 
-			Expect(assertionB.Hash.PackageHash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+			Expect(assertionB.Hash.PackageHash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))
 			hashB, err := packageHash.DependencyBuildImage(b)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(hashB).To(Equal("2668e418eab6861404834ad617713e39b8e58f68016a1fbfcc9384efdd037376"))
+			Expect(hashB).To(Equal("315075265aeb2e3c04c5428d31911f53c194ec9fa3db1421e8478f44b1e0def8"))
 		})
 	})
 

pkg/compiler/types/artifact/artifact.go

@@ -42,6 +42,7 @@ import (
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
@@ -168,7 +169,7 @@ func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageAr
 	}
 	defer os.RemoveAll(archive) // clean up
 	dst := filepath.Join(archive, fileName)
-	if err := helpers.CopyFile(s, dst); err != nil {
+	if err := fileHelper.CopyFile(s, dst); err != nil {
 		return nil, errors.Wrapf(err, "error while copying %s to %s", s, dst)
 	}
 
@@ -209,7 +210,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, k
 		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
 	}
 
-	empty, err := helpers.DirectoryIsEmpty(uncompressedFiles)
+	empty, err := fileHelper.DirectoryIsEmpty(uncompressedFiles)
 	if err != nil {
 		return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles)
 	}
@@ -218,7 +219,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, k
 	// We can't generate FROM scratch empty images. Docker will refuse to export them
 	// workaround: Inject a .virtual empty file
 	if empty {
-		helpers.Touch(filepath.Join(uncompressedFiles, ".virtual"))
+		fileHelper.Touch(filepath.Join(uncompressedFiles, ".virtual"))
 	}
 
 	data := a.genDockerfile()
@@ -401,12 +402,12 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 		// We want to protect file only if the hash of the files are differing OR the file size are
 		differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
 		// Check if exists
-		if helpers.Exists(destPath) && differs {
+		if fileHelper.Exists(destPath) && differs {
 			for i := 1; i < 1000; i++ {
 				name := filepath.Join(filepath.Join(filepath.Dir(path),
 					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
 
-				if helpers.Exists(name) {
+				if fileHelper.Exists(name) {
 					continue
 				}
 				Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
@@ -452,6 +453,9 @@ func (a *PackageArtifact) GetProtectFiles() []string {
 
 // Unpack Untar and decompress (TODO) to the given path
 func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+	if !strings.HasPrefix(dst, "/") {
+		return errors.New("destination must be an absolute path")
+	}
 
 	// Create
 	protectedFiles := a.GetProtectFiles()
@@ -628,7 +632,7 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 		_, err := os.Lstat(job.Dst)
 		if err != nil {
 			Debug("Copying ", job.Src)
-			if err := helpers.DeepCopyFile(job.Src, job.Dst); err != nil {
+			if err := fileHelper.DeepCopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
 		}

pkg/compiler/types/artifact/artifact_test.go

@@ -29,6 +29,7 @@ import (
 
 	. "github.com/mudler/luet/pkg/compiler"
 	helpers "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -71,7 +72,7 @@ var _ = Describe("Artifact", func() {
 
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -89,12 +90,12 @@ ENV PACKAGE_CATEGORY=app-admin`))
 			}
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
@@ -113,7 +114,7 @@ RUN echo bar > /test2`))
 			}
 			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 			diffs, err := compiler.GenerateChanges(b, opts, opts2)
 			Expect(err).ToNot(HaveOccurred())
 
@@ -140,15 +141,15 @@ RUN echo bar > /test2`))
 
 			a, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
 			err = helpers.Untar(a.Path, unpacked, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
-			content1, err := helpers.Read(filepath.Join(unpacked, "test"))
+			Expect(fileHelper.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
+			content1, err := fileHelper.Read(filepath.Join(unpacked, "test"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("foo\n"))
-			content2, err := helpers.Read(filepath.Join(unpacked, "test2"))
+			content2, err := fileHelper.Read(filepath.Join(unpacked, "test2"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content2).To(Equal("bar\n"))
 
@@ -156,7 +157,7 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			err = a.Verify()
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
+			Expect(fileHelper.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
 
 			err = a.Verify()
 			Expect(err).To(HaveOccurred())
@@ -244,7 +245,7 @@ RUN echo bar > /test2`))
 			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.DirectoryIsEmpty(result)).To(BeFalse())
+			Expect(fileHelper.DirectoryIsEmpty(result)).To(BeFalse())
 			content, err := ioutil.ReadFile(filepath.Join(result, ".virtual"))
 			Expect(err).ToNot(HaveOccurred())
 

pkg/compiler/types/options/compiler_options.go

@@ -43,6 +43,9 @@ type Compiler struct {
 	BackendArgs []string
 
 	BackendType string
+
+	// TemplatesFolder. should default to tree/templates
+	TemplatesFolder []string
 }
 
 func NewDefaultCompiler() *Compiler {
@@ -87,6 +90,13 @@ func WithBackendType(r string) func(cfg *Compiler) error {
 	}
 }
 
+func WithTemplateFolder(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.TemplatesFolder = r
+		return nil
+	}
+}
+
 func WithBuildValues(r []string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		cfg.BuildValuesFile = r

pkg/compiler/types/spec/spec.go

@@ -23,10 +23,11 @@ import (
 	"github.com/mitchellh/hashstructure/v2"
 	options "github.com/mudler/luet/pkg/compiler/types/options"
 
+	"github.com/ghodss/yaml"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/otiai10/copy"
-	yaml "gopkg.in/yaml.v2"
+	dirhash "golang.org/x/mod/sumdb/dirhash"
 )
 
 type LuetCompilationspecs []LuetCompilationSpec
@@ -115,49 +116,72 @@ type LuetCompilationSpec struct {
 
 	Copy []CopyField `json:"copy"`
 
-	Join pkg.DefaultPackages `json:"join"`
+	Join                pkg.DefaultPackages `json:"join"`
+	RequiresFinalImages bool                `json:"requires_final_images" yaml:"requires_final_images"`
 }
 
 // Signature is a portion of the spec that yields a signature for the hash
 type Signature struct {
-	Image      string
-	Steps      []string
-	PackageDir string
-	Prelude    []string
-	Seed       string
-	Env        []string
-	Retrieve   []string
-	Unpack     bool
-	Includes   []string
-	Excludes   []string
-	Copy       []CopyField
-	Join       pkg.DefaultPackages
+	Image               string
+	Steps               []string
+	PackageDir          string
+	Prelude             []string
+	Seed                string
+	Env                 []string
+	Retrieve            []string
+	Unpack              bool
+	Includes            []string
+	Excludes            []string
+	Copy                []CopyField
+	Join                pkg.DefaultPackages
+	RequiresFinalImages bool
 }
 
 func (cs *LuetCompilationSpec) signature() Signature {
 	return Signature{
-		Image:      cs.Image,
-		Steps:      cs.Steps,
-		PackageDir: cs.PackageDir,
-		Prelude:    cs.Prelude,
-		Seed:       cs.Seed,
-		Env:        cs.Env,
-		Retrieve:   cs.Retrieve,
-		Unpack:     cs.Unpack,
-		Includes:   cs.Includes,
-		Excludes:   cs.Excludes,
-		Copy:       cs.Copy,
-		Join:       cs.Join,
+		Image:               cs.Image,
+		Steps:               cs.Steps,
+		PackageDir:          cs.PackageDir,
+		Prelude:             cs.Prelude,
+		Seed:                cs.Seed,
+		Env:                 cs.Env,
+		Retrieve:            cs.Retrieve,
+		Unpack:              cs.Unpack,
+		Includes:            cs.Includes,
+		Excludes:            cs.Excludes,
+		Copy:                cs.Copy,
+		Join:                cs.Join,
+		RequiresFinalImages: cs.RequiresFinalImages,
 	}
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
 	var spec LuetCompilationSpec
+	var packageDefinition pkg.DefaultPackage
 	err := yaml.Unmarshal(b, &spec)
 	if err != nil {
 		return &spec, err
 	}
-	spec.Package = p.(*pkg.DefaultPackage)
+	err = yaml.Unmarshal(b, &packageDefinition)
+	if err != nil {
+		return &spec, err
+	}
+
+	// Update requires/conflict/provides
+	// When we have been passed a bytes slice, parse it as a package
+	// and updates requires/conflicts/provides.
+	// This is required in order to allow manipulation of such fields with templating
+	copy := *p.(*pkg.DefaultPackage)
+	spec.Package = &copy
+	if len(packageDefinition.GetRequires()) != 0 {
+		spec.Package.Requires(packageDefinition.GetRequires())
+	}
+	if len(packageDefinition.GetConflicts()) != 0 {
+		spec.Package.Conflicts(packageDefinition.GetConflicts())
+	}
+	if len(packageDefinition.GetProvides()) != 0 {
+		spec.Package.SetProvides(packageDefinition.GetProvides())
+	}
 	return &spec, nil
 }
 func (cs *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
@@ -242,7 +266,7 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 }
 
 func (cs *LuetCompilationSpec) EmptyPackage() bool {
-	return len(cs.BuildSteps()) == 0 && len(cs.GetPreBuildSteps()) == 0 && !cs.UnpackedPackage()
+	return len(cs.BuildSteps()) == 0 && !cs.UnpackedPackage()
 }
 
 func (cs *LuetCompilationSpec) UnpackedPackage() bool {
@@ -266,7 +290,14 @@ func (cs *LuetCompilationSpec) Hash() (string, error) {
 	// build a signature, we want to be part of the hash only the fields that are relevant for build purposes
 	signature := cs.signature()
 	h, err := hashstructure.Hash(signature, hashstructure.FormatV2, nil)
-	return fmt.Sprint(h), err
+	if err != nil {
+		return "", err
+	}
+	sum, err := dirhash.HashDir(cs.Package.Path, "", dirhash.DefaultHash)
+	if err != nil {
+		return fmt.Sprint(h), err
+	}
+	return fmt.Sprint(h, sum), err
 }
 
 func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {

pkg/compiler/types/spec/spec_test.go

@@ -22,9 +22,9 @@ import (
 
 	options "github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	. "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -154,7 +154,7 @@ var _ = Describe("Spec", func() {
 			lspec.Env = []string{"test=1"}
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -167,7 +167,7 @@ ENV test=1`))
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
@@ -205,7 +205,7 @@ RUN echo bar > /test2`))
 
 		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
-		dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
 		Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -222,7 +222,7 @@ ENV test=1`))
 
 		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
-		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
 		Expect(dockerfile).To(Equal(`
 FROM alpine
@@ -237,7 +237,7 @@ ENV test=1`))
 
 		err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
-		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 		Expect(err).ToNot(HaveOccurred())
 
 		Expect(dockerfile).To(Equal(`

pkg/config/config.go

@@ -17,7 +17,6 @@
 package config
 
 import (
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -27,7 +26,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mudler/luet/pkg/helpers"
+	"github.com/pkg/errors"
+
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	pkg "github.com/mudler/luet/pkg/package"
 	solver "github.com/mudler/luet/pkg/solver"
 
@@ -92,7 +94,7 @@ func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
 		return solver.SimpleQLearningSolver()
 	}
 
-	return &solver.DummyPackageResolver{}
+	return &solver.Explainer{}
 }
 
 func (opts *LuetSolverOptions) CompactString() string {
@@ -108,6 +110,16 @@ type LuetSystemConfig struct {
 	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
 }
 
+func (s *LuetSystemConfig) SetRootFS(path string) error {
+	p, err := fileHelper.Rel2Abs(path)
+	if err != nil {
+		return err
+	}
+
+	s.Rootfs = p
+	return nil
+}
+
 func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
 	dbpath = filepath.Join(dbpath, "repos/"+name)
@@ -406,8 +418,21 @@ system:
 }
 
 func (c *LuetSystemConfig) InitTmpDir() error {
-	if !helpers.Exists(c.TmpDirBase) {
-		return os.MkdirAll(c.TmpDirBase, os.ModePerm)
+	if !filepath.IsAbs(c.TmpDirBase) {
+		abs, err := fileHelper.Rel2Abs(c.TmpDirBase)
+		if err != nil {
+			return errors.Wrap(err, "while converting relative path to absolute path")
+		}
+		c.TmpDirBase = abs
+	}
+
+	if _, err := os.Stat(c.TmpDirBase); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(c.TmpDirBase, os.ModePerm)
+			if err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }

pkg/config/config_test.go

@@ -22,7 +22,7 @@ import (
 	"strings"
 
 	config "github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -38,7 +38,7 @@ var _ = Describe("Config", func() {
 			tmpDir, err := config.LuetCfg.GetSystem().TempDir("test1")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(strings.HasPrefix(tmpDir, filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
-			Expect(helpers.Exists(tmpDir)).To(BeTrue())
+			Expect(fileHelper.Exists(tmpDir)).To(BeTrue())
 
 			defer os.RemoveAll(tmpDir)
 		})
@@ -49,7 +49,7 @@ var _ = Describe("Config", func() {
 			tmpFile, err := config.LuetCfg.GetSystem().TempFile("testfile1")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(strings.HasPrefix(tmpFile.Name(), filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
-			Expect(helpers.Exists(tmpFile.Name())).To(BeTrue())
+			Expect(fileHelper.Exists(tmpFile.Name())).To(BeTrue())
 
 			defer os.Remove(tmpFile.Name())
 		})

pkg/helpers/archive_test.go

@@ -25,6 +25,8 @@ import (
 	"os"
 	"path/filepath"
 
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	"github.com/docker/docker/pkg/archive"
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
@@ -128,7 +130,7 @@ var _ = Describe("Helpers Archive", func() {
 			err = archive.Untar(replacerArchive, targetDir, opts)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
+			Expect(fileHelper.Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
 		})
 	})
 })

pkg/helpers/docker/docker.go

@@ -13,20 +13,32 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package docker
 
 import (
 	"context"
 	"encoding/hex"
+	"fmt"
 	"os"
 	"strings"
 
+	"github.com/containerd/containerd/images"
+	"github.com/mudler/luet/pkg/helpers/imgworker"
+
+	continerdarchive "github.com/containerd/containerd/archive"
 	"github.com/docker/cli/cli/trust"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/registry"
-	"github.com/mudler/luet/pkg/helpers/imgworker"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/mudler/luet/pkg/bus"
 	"github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/theupdateframework/notary/tuf/data"
 )
@@ -94,18 +106,31 @@ func trustedResolveDigest(ctx context.Context, ref reference.NamedTagged, authCo
 	return reference.WithDigest(ref, dgst)
 }
 
-// DownloadAndExtractDockerImage is a re-adaption
-// from genuinetools/img https://github.com/genuinetools/img/blob/54d0ca981c1260546d43961a538550eef55c87cf/pull.go
-func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) {
+type staticAuth struct {
+	auth *types.AuthConfig
+}
 
-	if verify {
-		img, err := verifyImage(image, auth)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed verifying image")
-		}
-		image = img
+func (s staticAuth) Authorization() (*authn.AuthConfig, error) {
+	if s.auth == nil {
+		return nil, nil
 	}
+	return &authn.AuthConfig{
+		Username:      s.auth.Username,
+		Password:      s.auth.Password,
+		Auth:          s.auth.Auth,
+		IdentityToken: s.auth.IdentityToken,
+		RegistryToken: s.auth.RegistryToken,
+	}, nil
+}
 
+// UnpackEventData is the data structure to pass for the bus events
+type UnpackEventData struct {
+	Image string
+	Dest  string
+}
+
+// privilegedExtractImage uses the imgworker (which requires privileges) to extract a container image
+func privilegedExtractImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) {
 	defer os.RemoveAll(temp)
 	c, err := imgworker.New(temp, auth)
 	if err != nil {
@@ -116,14 +141,121 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
 	listedImage, err := c.Pull(image)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed listing images")
-
 	}
 
 	os.RemoveAll(dest)
+
+	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
+
 	err = c.Unpack(image, dest)
+
+	bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
+
 	return listedImage, err
 }
 
+// UnarchiveLayers extract layers with archive.Untar from docker instead of containerd
+func UnarchiveLayers(temp string, img v1.Image, image, dest string, auth *types.AuthConfig, verify bool) (int64, error) {
+	layers, err := img.Layers()
+	if err != nil {
+		return 0, fmt.Errorf("reading layers from '%s' image failed: %v", image, err)
+	}
+	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	var size int64
+	for _, l := range layers {
+		s, err := l.Size()
+		if err != nil {
+			return 0, fmt.Errorf("reading layer size from '%s' image failed: %v", image, err)
+		}
+		size += s
+
+		layerReader, err := l.Uncompressed()
+		if err != nil {
+			return 0, fmt.Errorf("reading uncompressed layer from '%s' image failed: %v", image, err)
+		}
+		defer layerReader.Close()
+
+		// Unpack the tarfile to the rootfs path.
+		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
+		if err := archive.Untar(layerReader, dest, &archive.TarOptions{
+			NoLchown:        false,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+		}); err != nil {
+			return 0, fmt.Errorf("extracting '%s' image to directory %s failed: %v", image, dest, err)
+		}
+	}
+	bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	return size, nil
+}
+
+// DownloadAndExtractDockerImage extracts a container image natively. It supports privileged/unprivileged mode
+func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) {
+	if verify {
+		img, err := verifyImage(image, auth)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed verifying image")
+		}
+		image = img
+	}
+
+	if os.Getenv("LUET_PRIVILEGED_EXTRACT") == "true" {
+		return privilegedExtractImage(temp, image, dest, auth, verify)
+	}
+
+	ref, err := name.ParseReference(image)
+	if err != nil {
+		return nil, err
+	}
+
+	img, err := remote.Image(ref, remote.WithAuth(staticAuth{auth}))
+	if err != nil {
+		return nil, err
+	}
+
+	m, err := img.Manifest()
+	if err != nil {
+		return nil, err
+	}
+
+	mt, err := img.MediaType()
+	if err != nil {
+		return nil, err
+	}
+
+	d, err := img.Digest()
+	if err != nil {
+		return nil, err
+	}
+
+	reader := mutate.Extract(img)
+	defer reader.Close()
+	defer os.RemoveAll(temp)
+
+	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	c, err := continerdarchive.Apply(context.TODO(), dest, reader)
+	if err != nil {
+		return nil, err
+	}
+
+	bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
+
+	return &imgworker.ListedImage{
+		Image: images.Image{
+			Name:   image,
+			Labels: m.Annotations,
+			Target: specs.Descriptor{
+				MediaType: string(mt),
+				Digest:    digest.Digest(d.String()),
+				Size:      c,
+			},
+		},
+		ContentSize: c,
+	}, nil
+}
+
 func StripInvalidStringsFromImage(s string) string {
 	return strings.ReplaceAll(s, "+", "-")
 }

pkg/helpers/docker/docker_test.go

@@ -13,10 +13,10 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers_test
+package docker_test
 
 import (
-	. "github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/helpers/docker"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -24,7 +24,7 @@ import (
 var _ = Describe("StripInvalidStringsFromImage", func() {
 	Context("Image names", func() {
 		It("strips invalid chars", func() {
-			Expect(StripInvalidStringsFromImage("foo+bar")).To(Equal("foo-bar"))
+			Expect(docker.StripInvalidStringsFromImage("foo+bar")).To(Equal("foo-bar"))
 		})
 	})
 })

pkg/helpers/file/file.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package file
 
 import (
 	"fmt"
@@ -287,3 +287,15 @@ func CopyDir(src string, dst string) (err error) {
 		Sync:      true,
 		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }
+
+func Rel2Abs(s string) (string, error) {
+	pathToSet := s
+	if !filepath.IsAbs(s) {
+		abs, err := filepath.Abs(s)
+		if err != nil {
+			return "", err
+		}
+		pathToSet = abs
+	}
+	return pathToSet, nil
+}

pkg/helpers/file/file_test.go

@@ -13,14 +13,15 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers_test
+package file_test
 
 import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
-	. "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -28,8 +29,8 @@ import (
 var _ = Describe("Helpers", func() {
 	Context("Exists", func() {
 		It("Detect existing and not-existing files", func() {
-			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml")).To(BeTrue())
-			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
+			Expect(fileHelper.Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml")).To(BeTrue())
+			Expect(fileHelper.Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
 		})
 	})
 
@@ -38,15 +39,15 @@ var _ = Describe("Helpers", func() {
 			testDir, err := ioutil.TempDir(os.TempDir(), "test")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(testDir)
-			Expect(DirectoryIsEmpty(testDir)).To(BeTrue())
+			Expect(fileHelper.DirectoryIsEmpty(testDir)).To(BeTrue())
 		})
 		It("Detects directory with files", func() {
 			testDir, err := ioutil.TempDir(os.TempDir(), "test")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(testDir)
-			err = Touch(filepath.Join(testDir, "foo"))
+			err = fileHelper.Touch(filepath.Join(testDir, "foo"))
 			Expect(err).ToNot(HaveOccurred())
-			Expect(DirectoryIsEmpty(testDir)).To(BeFalse())
+			Expect(fileHelper.DirectoryIsEmpty(testDir)).To(BeFalse())
 		})
 	})
 
@@ -72,7 +73,7 @@ var _ = Describe("Helpers", func() {
 			err = ioutil.WriteFile(filepath.Join(testDir, "baz2", "foo"), []byte("test\n"), 0644)
 			Expect(err).ToNot(HaveOccurred())
 
-			ordered, notExisting := OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
+			ordered, notExisting := fileHelper.OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
 
 			Expect(ordered).To(Equal([]string{"baz", "bar/foo", "foo", "baz2/foo", "bar", "baz2"}))
 			Expect(notExisting).To(Equal([]string{"notexisting"}))
@@ -96,7 +97,7 @@ var _ = Describe("Helpers", func() {
 			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz", "fa"), os.ModePerm)
 			Expect(err).ToNot(HaveOccurred())
 
-			ordered, _ := OrderFiles(testDir, []string{"foo", "foo/bar", "bar", "foo/baz/fa", "foo/baz"})
+			ordered, _ := fileHelper.OrderFiles(testDir, []string{"foo", "foo/bar", "bar", "foo/baz/fa", "foo/baz"})
 			Expect(ordered).To(Equal([]string{"foo/baz/fa", "foo/bar", "foo/baz", "foo", "bar"}))
 		})
 	})

pkg/helpers/helm.go

@@ -2,6 +2,10 @@ package helpers
 
 import (
 	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	"github.com/imdario/mergo"
 	"github.com/pkg/errors"
@@ -11,17 +15,86 @@ import (
 	"helm.sh/helm/v3/pkg/engine"
 )
 
+// ChartFileB is an helper that takes a slice of bytes and construct a chart.File slice from it
+func ChartFileB(s []byte) []*chart.File {
+	return []*chart.File{
+		{Name: "templates", Data: s},
+	}
+}
+
+// ChartFileS is an helper that takes a string and construct a chart.File slice from it
+func ChartFileS(s string) []*chart.File {
+	return []*chart.File{
+		{Name: "templates", Data: []byte(s)},
+	}
+}
+
+// ChartFile reads all the given files and returns a slice of []*chart.File
+// containing the raw content and the file name for each file
+func ChartFile(s ...string) []*chart.File {
+	files := []*chart.File{}
+	for _, c := range s {
+		raw, err := ioutil.ReadFile(c)
+		if err != nil {
+			return files
+		}
+		files = append(files, &chart.File{Name: c, Data: raw})
+	}
+
+	return files
+}
+
+// ChartFiles reads a list of paths and reads all yaml file inside. It returns a
+// slice of pointers of chart.File(s) with the raw content of the yaml
+func ChartFiles(path []string) ([]*chart.File, error) {
+	var chartFiles []*chart.File
+	for _, t := range path {
+		rel, err := fileHelper.Rel2Abs(t)
+		if err != nil {
+			return nil, err
+		}
+
+		if !fileHelper.Exists(rel) {
+			continue
+		}
+		files, err := fileHelper.ListDir(rel)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, f := range files {
+			if strings.ToLower(filepath.Ext(f)) == ".yaml" {
+				raw, err := ioutil.ReadFile(f)
+				if err != nil {
+					return nil, err
+				}
+				chartFiles = append(chartFiles, &chart.File{Name: f, Data: raw})
+			}
+		}
+	}
+	return chartFiles, nil
+}
+
 // RenderHelm renders the template string with helm
-func RenderHelm(template string, values, d map[string]interface{}) (string, error) {
+func RenderHelm(files []*chart.File, values, d map[string]interface{}) (string, error) {
+
+	// We slurp all the files into one here. This is not elegant, but still works.
+	// As a reminder, the files passed here have on the head the templates in the 'templates/' folder
+	// of each luet tree, and it have at the bottom the package buildpsec to be templated.
+	// TODO: Replace by correctly populating the files so that the helm render engine templates it
+	// correctly
+	toTemplate := ""
+	for _, f := range files {
+		toTemplate += string(f.Data)
+	}
+
 	c := &chart.Chart{
 		Metadata: &chart.Metadata{
 			Name:    "",
 			Version: "",
 		},
-		Templates: []*chart.File{
-			{Name: "templates", Data: []byte(template)},
-		},
-		Values: map[string]interface{}{"Values": values},
+		Templates: ChartFileS(toTemplate),
+		Values:    map[string]interface{}{"Values": values},
 	}
 
 	v, err := chartutil.CoalesceValues(c, map[string]interface{}{"Values": d})
@@ -69,23 +142,18 @@ func reverse(s []string) []string {
 	return s
 }
 
-func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string, error) {
-	raw, err := ioutil.ReadFile(toTemplate)
-	if err != nil {
-		return "", errors.Wrap(err, "reading file "+toTemplate)
-	}
-
-	if !Exists(valuesFile) {
-		return "", errors.Wrap(err, "file not existing "+valuesFile)
+func RenderFiles(files []*chart.File, valuesFile string, defaultFile ...string) (string, error) {
+	if !fileHelper.Exists(valuesFile) {
+		return "", errors.New("file does not exist: " + valuesFile)
 	}
 	val, err := ioutil.ReadFile(valuesFile)
 	if err != nil {
-		return "", errors.Wrap(err, "reading file "+valuesFile)
+		return "", errors.Wrap(err, "reading file: "+valuesFile)
 	}
 
 	var values templatedata
 	if err = yaml.Unmarshal(val, &values); err != nil {
-		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
+		return "", errors.Wrap(err, "unmarshalling values")
 	}
 
 	dst, err := UnMarshalValues(defaultFile)
@@ -93,5 +161,5 @@ func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string,
 		return "", errors.Wrap(err, "unmarshalling values")
 	}
 
-	return RenderHelm(string(raw), values, dst)
+	return RenderHelm(files, values, dst)
 }

pkg/helpers/helm_test.go

@@ -30,21 +30,21 @@ func writeFile(path string, content string) {
 	Expect(err).ToNot(HaveOccurred())
 }
 
-var _ = Describe("Helpers", func() {
+var _ = Describe("Helm", func() {
 	Context("RenderHelm", func() {
 		It("Renders templates", func() {
-			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
+			out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).To(Equal("foobar"))
 		})
 		It("Renders templates with overrides", func() {
-			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
+			out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).To(Equal("foobar"))
 		})
 
 		It("Renders templates", func() {
-			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
+			out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).To(Equal("foobar"))
 		})
@@ -68,7 +68,7 @@ foo: "baz"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("baz"))
 
@@ -93,7 +93,7 @@ faa: "baz"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bar"))
 
@@ -114,7 +114,7 @@ foo: "bar"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values)
+			res, err := RenderFiles(ChartFile(toTemplate), values)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bar"))
 		})
@@ -145,11 +145,11 @@ bar: "nei"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d2, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d2, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bazneif"))
 
-			res, err = RenderFiles(toTemplate, values, d, d2)
+			res, err = RenderFiles(ChartFile(toTemplate), values, d, d2)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("doneif"))
 		})
@@ -173,7 +173,7 @@ faa: "baz"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal(""))
 

pkg/helpers/imgworker/unpack.go

@@ -5,6 +5,7 @@ package imgworker
 import (
 	"errors"
 	"fmt"
+	"github.com/mudler/luet/pkg/bus"
 	"os"
 
 	"github.com/containerd/containerd/content"
@@ -18,6 +19,12 @@ import (
 // TODO: this requires root permissions to mount/unmount layers, althrought it shouldn't be required.
 // See how backends are unpacking images without asking for root permissions.
 
+// UnpackEventData is the data structure to pass for the bus events
+type UnpackEventData struct {
+	Image string
+	Dest  string
+}
+
 // Unpack exports an image to a rootfs destination directory.
 func (c *Client) Unpack(image, dest string) error {
 
@@ -59,6 +66,8 @@ func (c *Client) Unpack(image, dest string) error {
 		return fmt.Errorf("getting image manifest failed: %v", err)
 	}
 
+	_,_ = bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
+
 	for _, desc := range manifest.Layers {
 		logrus.Debugf("Unpacking layer %s", desc.Digest.String())
 
@@ -78,5 +87,7 @@ func (c *Client) Unpack(image, dest string) error {
 		}
 	}
 
+	_, _ = bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
+
 	return nil
 }

pkg/helpers/match/match.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package match
 
 import (
 	"reflect"

pkg/helpers/slice.go

@@ -0,0 +1,10 @@
+package helpers
+
+func Contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}

pkg/installer/client/docker.go

@@ -28,8 +28,8 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
-	"github.com/mudler/luet/pkg/helpers/imgworker"
+	"github.com/mudler/luet/pkg/helpers/docker"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 )
 
@@ -64,7 +64,7 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
 	artifactName := path.Base(a.Path)
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 	Debug("Cache file", cacheFile)
-	if err := helpers.EnsureDir(cacheFile); err != nil {
+	if err := fileHelper.EnsureDir(cacheFile); err != nil {
 		return nil, errors.Wrapf(err, "could not create cache folder %s for %s", config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), cacheFile)
 	}
 	ok := false
@@ -77,7 +77,7 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
 	// is done in such cases (see repository.go)
 
 	// Check if file is already in cache
-	if helpers.Exists(cacheFile) {
+	if fileHelper.Exists(cacheFile) {
 		Debug("Cache hit for artifact", artifactName)
 		resultingArtifact = a
 		resultingArtifact.Path = cacheFile
@@ -102,7 +102,7 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
 			}
 
 			// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
-			info, err := helpers.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
+			info, err := docker.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
 			if err != nil {
 				Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
 				continue
@@ -140,7 +140,6 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 	var file *os.File = nil
 	var err error
 	var temp, contentstore string
-	var info *imgworker.ListedImage
 	// Files should be in URI/repository:<file>
 	ok := false
 
@@ -161,10 +160,10 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 			continue
 		}
 
-		imageName := fmt.Sprintf("%s:%s", uri, helpers.StripInvalidStringsFromImage(name))
+		imageName := fmt.Sprintf("%s:%s", uri, docker.StripInvalidStringsFromImage(name))
 		Info("Downloading", imageName)
 
-		info, err = helpers.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
+		info, err := docker.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
 		if err != nil {
 			Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
 			continue
@@ -174,7 +173,7 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 		Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
 
 		Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name())
-		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+		err = fileHelper.CopyFile(filepath.Join(temp, name), file.Name())
 		if err != nil {
 			continue
 		}

pkg/installer/client/docker_test.go

@@ -22,8 +22,8 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
-	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	. "github.com/mudler/luet/pkg/installer/client"
@@ -32,7 +32,7 @@ import (
 )
 
 // This test expect that the repository defined in UNIT_TEST_DOCKER_IMAGE is in zstd format.
-// the repository is built by the 01_simple_docker.sh integration test file.
+// the repository is built by the 01_simple_docker.sh integration test fileHelper.
 // This test also require root. At the moment, unpacking docker images with 'img' requires root permission to
 // mount/unmount layers.
 var _ = Describe("Docker client", func() {
@@ -51,7 +51,7 @@ var _ = Describe("Docker client", func() {
 		It("Downloads single files", func() {
 			f, err := c.DownloadFile("repository.yaml")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(f)).To(ContainSubstring("Test Repo"))
+			Expect(fileHelper.Read(f)).To(ContainSubstring("Test Repo"))
 			os.RemoveAll(f)
 		})
 
@@ -71,8 +71,8 @@ var _ = Describe("Docker client", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 			Expect(f.Unpack(tmpdir, false)).ToNot(HaveOccurred())
-			Expect(helpers.Read(filepath.Join(tmpdir, "c"))).To(Equal("c\n"))
-			Expect(helpers.Read(filepath.Join(tmpdir, "cd"))).To(Equal("c\n"))
+			Expect(fileHelper.Read(filepath.Join(tmpdir, "c"))).To(Equal("c\n"))
+			Expect(fileHelper.Read(filepath.Join(tmpdir, "cd"))).To(Equal("c\n"))
 			os.RemoveAll(f.Path)
 		})
 	})

pkg/installer/client/http.go

@@ -18,19 +18,20 @@ package client
 import (
 	"fmt"
 	"math"
+	"net/http"
 	"net/url"
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"time"
 
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 
-	"github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
-
 	"github.com/cavaliercoder/grab"
+	"github.com/mudler/luet/pkg/config"
 
 	"github.com/schollz/progressbar/v3"
 )
@@ -43,6 +44,27 @@ func NewHttpClient(r RepoData) *HttpClient {
 	return &HttpClient{RepoData: r}
 }
 
+func NewGrabClient() *grab.Client {
+	httpTimeout := 120
+	timeout := os.Getenv("HTTP_TIMEOUT")
+	if timeout != "" {
+		timeoutI, err := strconv.Atoi(timeout)
+		if err == nil {
+			httpTimeout = timeoutI
+		}
+	}
+
+	return &grab.Client{
+		UserAgent: "grab",
+		HTTPClient: &http.Client{
+			Timeout: time.Duration(httpTimeout) * time.Second,
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+			},
+		},
+	}
+}
+
 func (c *HttpClient) PrepareReq(dst, url string) (*grab.Request, error) {
 
 	req, err := grab.NewRequest(dst, url)
@@ -77,7 +99,7 @@ func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.Pa
 	ok := false
 
 	// Check if file is already in cache
-	if helpers.Exists(cacheFile) {
+	if fileHelper.Exists(cacheFile) {
 		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 
@@ -87,7 +109,7 @@ func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.Pa
 		}
 		defer os.RemoveAll(temp)
 
-		client := grab.NewClient()
+		client := NewGrabClient()
 
 		for _, uri := range c.RepoData.Urls {
 			Debug("Downloading artifact", artifactName, "from", uri)
@@ -156,7 +178,7 @@ func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.Pa
 				fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
 
 			Debug("\nCopying file ", filepath.Join(temp, artifactName), "to", cacheFile)
-			err = helpers.CopyFile(filepath.Join(temp, artifactName), cacheFile)
+			err = fileHelper.CopyFile(filepath.Join(temp, artifactName), cacheFile)
 
 			bar.Finish()
 			ok = true
@@ -187,7 +209,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 		return "", err
 	}
 
-	client := grab.NewClient()
+	client := NewGrabClient()
 
 	for _, uri := range c.RepoData.Urls {
 
@@ -218,7 +240,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 			fmt.Sprintf("%.2f", (float64(resp.BytesComplete())/1000)/1000), "MB (",
 			fmt.Sprintf("%.2f", (float64(resp.BytesPerSecond())/1024)/1024), "MiB/s )")
 
-		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
+		err = fileHelper.CopyFile(filepath.Join(temp, name), file.Name())
 		if err != nil {
 			continue
 		}

pkg/installer/client/http_test.go

@@ -23,8 +23,7 @@ import (
 	"path/filepath"
 
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
-	helpers "github.com/mudler/luet/pkg/helpers"
-
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer/client"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -47,7 +46,7 @@ var _ = Describe("Http client", func() {
 			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
 			path, err := c.DownloadFile("test.txt")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path)).To(Equal("test"))
+			Expect(fileHelper.Read(path)).To(Equal("test"))
 			os.RemoveAll(path)
 		})
 
@@ -65,7 +64,7 @@ var _ = Describe("Http client", func() {
 			c := NewHttpClient(RepoData{Urls: []string{ts.URL}})
 			path, err := c.DownloadArtifact(&artifact.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path.Path)).To(Equal("test"))
+			Expect(fileHelper.Read(path.Path)).To(Equal("test"))
 			os.RemoveAll(path.Path)
 		})
 

pkg/installer/client/local.go

@@ -22,9 +22,8 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
-
-	"github.com/mudler/luet/pkg/helpers"
 )
 
 type LocalClient struct {
@@ -50,7 +49,7 @@ func (c *LocalClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.P
 	}
 
 	// Check if file is already in cache
-	if helpers.Exists(cacheFile) {
+	if fileHelper.Exists(cacheFile) {
 		Debug("Use artifact", artifactName, "from cache.")
 	} else {
 		ok := false
@@ -61,7 +60,7 @@ func (c *LocalClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.P
 			Info("Downloading artifact", artifactName, "from", uri)
 
 			//defer os.Remove(file.Name())
-			err = helpers.CopyFile(filepath.Join(uri, artifactName), cacheFile)
+			err = fileHelper.CopyFile(filepath.Join(uri, artifactName), cacheFile)
 			if err != nil {
 				continue
 			}
@@ -104,7 +103,7 @@ func (c *LocalClient) DownloadFile(name string) (string, error) {
 		}
 		//defer os.Remove(file.Name())
 
-		err = helpers.CopyFile(filepath.Join(uri, name), file.Name())
+		err = fileHelper.CopyFile(filepath.Join(uri, name), file.Name())
 		if err != nil {
 			continue
 		}

pkg/installer/client/local_test.go

@@ -21,8 +21,7 @@ import (
 	"path/filepath"
 
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
-	helpers "github.com/mudler/luet/pkg/helpers"
-
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer/client"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -42,7 +41,7 @@ var _ = Describe("Local client", func() {
 			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
 			path, err := c.DownloadFile("test.txt")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path)).To(Equal("test"))
+			Expect(fileHelper.Read(path)).To(Equal("test"))
 			os.RemoveAll(path)
 		})
 
@@ -58,7 +57,7 @@ var _ = Describe("Local client", func() {
 			c := NewLocalClient(RepoData{Urls: []string{tmpdir}})
 			path, err := c.DownloadArtifact(&artifact.PackageArtifact{Path: "test.txt"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(path.Path)).To(Equal("test"))
+			Expect(fileHelper.Read(path.Path)).To(Equal("test"))
 			os.RemoveAll(path.Path)
 		})
 

pkg/installer/installer.go

@@ -24,16 +24,17 @@ import (
 	"strings"
 	"sync"
 
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
-
-	. "github.com/logrusorgru/aurora"
 	"github.com/mudler/luet/pkg/bus"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	"github.com/mudler/luet/pkg/helpers/match"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 
+	. "github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 )
 
@@ -271,6 +272,15 @@ func (l *LuetInstaller) swap(o Option, syncedRepos Repositories, toRemove pkg.Pa
 	if err := l.download(syncedRepos, match); err != nil {
 		return errors.Wrap(err, "Pre-downloading packages")
 	}
+
+	if err := l.checkFileconflicts(match, false, s); err != nil {
+		if !l.Options.Force {
+			return errors.Wrap(err, "file conflict found")
+		} else {
+			Warning("file conflict found", err.Error())
+		}
+	}
+
 	if l.Options.DownloadOnly {
 		return nil
 	}
@@ -597,7 +607,7 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 				"from", repo.GetName(), "is installed")
 		FILES:
 			for _, f := range artefact.Files {
-				if helpers.Exists(filepath.Join(s.Target, f)) {
+				if fileHelper.Exists(filepath.Join(s.Target, f)) {
 					p, err := repo.GetTree().GetDatabase().FindPackage(artefact.CompileSpec.GetPackage())
 					if err != nil {
 						return err
@@ -755,12 +765,64 @@ func (l *LuetInstaller) getFinalizers(allRepos pkg.PackageDatabase, solution sol
 	return toFinalize, nil
 }
 
+func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, checkSystem bool, s *System) error {
+	Info("Checking for file conflicts..")
+	defer s.Clean() // Release memory
+
+	filesToInstall := []string{}
+	for _, m := range toInstall {
+		a, err := l.downloadPackage(m)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed downloading package")
+		}
+		files, err := a.FileList()
+		if err != nil && !l.Options.Force {
+			return errors.Wrapf(err, "Could not get filelist for %s", a.CompileSpec.Package.HumanReadableString())
+		}
+
+		for _, f := range files {
+			if helpers.Contains(filesToInstall, f) {
+				return fmt.Errorf(
+					"file conflict between packages to be installed",
+				)
+			}
+			if checkSystem {
+				exists, p, err := s.ExistsPackageFile(f)
+				if err != nil {
+					return errors.Wrap(err, "failed checking into system db")
+				}
+				if exists {
+					return fmt.Errorf(
+						"file conflict between '%s' and '%s' ( file: %s )",
+						p.HumanReadableString(),
+						m.Package.HumanReadableString(),
+						f,
+					)
+				}
+			}
+		}
+		filesToInstall = append(filesToInstall, files...)
+	}
+
+	return nil
+}
+
 func (l *LuetInstaller) install(o Option, syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
-	// Install packages into rootfs in parallel.
+
+	// Download packages in parallel first
 	if err := l.download(syncedRepos, toInstall); err != nil {
 		return errors.Wrap(err, "Downloading packages")
 	}
 
+	// Check file conflicts
+	if err := l.checkFileconflicts(toInstall, true, s); err != nil {
+		if !l.Options.Force {
+			return errors.Wrap(err, "file conflict found")
+		} else {
+			Warning("file conflict found", err.Error())
+		}
+	}
+
 	if l.Options.DownloadOnly {
 		return nil
 	}
@@ -915,7 +977,7 @@ func pruneEmptyFilePath(path string) {
 		currentPath = filepath.Join(currentPath, p)
 		allPaths = append(allPaths, currentPath)
 	}
-	helpers.ReverseAny(allPaths)
+	match.ReverseAny(allPaths)
 	for _, p := range allPaths {
 		checkAndPrunePath(p)
 	}
@@ -943,7 +1005,7 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		cp.Map(files)
 	}
 
-	toRemove, notPresent := helpers.OrderFiles(s.Target, files)
+	toRemove, notPresent := fileHelper.OrderFiles(s.Target, files)
 
 	// Remove from target
 	for _, f := range toRemove {

pkg/installer/installer_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -84,34 +85,34 @@ var _ = Describe("Installer", func() {
 
 			a, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(a.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(a.Path)).To(BeTrue())
 			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -136,8 +137,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -154,8 +155,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
@@ -199,21 +200,21 @@ urls:
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
@@ -223,15 +224,15 @@ urls:
 			repo.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -256,8 +257,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -274,8 +275,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
@@ -319,21 +320,21 @@ urls:
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := GenerateRepository(
 				"test",
@@ -345,15 +346,15 @@ urls:
 				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -383,8 +384,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -401,8 +402,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
@@ -444,21 +445,21 @@ urls:
 
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := GenerateRepository(
 				"test",
@@ -471,15 +472,15 @@ urls:
 				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -531,7 +532,7 @@ urls:
 
 			artifact, err = c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 
 			repo, err = stubRepo(tmpdir2, "../../tests/fixtures/alpine")
 			Expect(err).ToNot(HaveOccurred())
@@ -604,15 +605,15 @@ urls:
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -642,8 +643,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -660,11 +661,11 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			// New version - new files
-			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
 			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
 			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -729,9 +730,9 @@ urls:
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
@@ -774,8 +775,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -793,11 +794,11 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			// New version - new files
-			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
 			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
 			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -856,17 +857,17 @@ urls:
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -896,8 +897,8 @@ urls:
 			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
 			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -914,11 +915,11 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			// New version - new files
-			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
 			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
 			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -1014,17 +1015,17 @@ urls:
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -1059,13 +1060,13 @@ urls:
 			Expect(err).To(HaveOccurred())
 
 			Expect(len(system.Database.World())).To(Equal(0))
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeFalse())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeFalse())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "c"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "c"))).To(BeFalse())
 
-			Expect(helpers.Touch(filepath.Join(fakeroot, "test5"))).ToNot(HaveOccurred())
-			Expect(helpers.Touch(filepath.Join(fakeroot, "test6"))).ToNot(HaveOccurred())
-			Expect(helpers.Touch(filepath.Join(fakeroot, "c"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "test5"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "test6"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "c"))).ToNot(HaveOccurred())
 
 			err = inst.Reclaim(system)
 			Expect(err).ToNot(HaveOccurred())
@@ -1115,15 +1116,15 @@ urls:
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -1158,13 +1159,13 @@ urls:
 			Expect(err).To(HaveOccurred())
 
 			Expect(len(system.Database.World())).To(Equal(0))
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeFalse())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeFalse())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "c"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeFalse())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "c"))).To(BeFalse())
 
-			Expect(helpers.Touch(filepath.Join(fakeroot, "test5"))).ToNot(HaveOccurred())
-			Expect(helpers.Touch(filepath.Join(fakeroot, "test6"))).ToNot(HaveOccurred())
-			Expect(helpers.Touch(filepath.Join(fakeroot, "c"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "test5"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "test6"))).ToNot(HaveOccurred())
+			Expect(fileHelper.Touch(filepath.Join(fakeroot, "c"))).ToNot(HaveOccurred())
 
 			err = inst.Reclaim(system)
 			Expect(err).ToNot(HaveOccurred())
@@ -1218,11 +1219,11 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nothing should be there anymore (files, packagedb entry)
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
-			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
 
 			// New version - new files
-			Expect(helpers.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "newc"))).To(BeTrue())
 			_, err = system.Database.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).To(HaveOccurred())
 			_, err = system.Database.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})

pkg/installer/repository.go

@@ -28,11 +28,11 @@ import (
 
 	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	"go.uber.org/multierr"
 
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
 	"github.com/mudler/luet/pkg/installer/client"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -723,7 +723,7 @@ func (r *LuetSystemRepository) SyncBuildMetadata(path string) error {
 		if err != nil {
 			return errors.Wrapf(err, "while downloading metadata for %s", ai.HumanReadableString())
 		}
-		if err := helpers.Move(file, filepath.Join(path, ai.GetMetadataFilePath())); err != nil {
+		if err := fileHelper.Move(file, filepath.Join(path, ai.GetMetadataFilePath())); err != nil {
 			return err
 		}
 	}
@@ -810,7 +810,7 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
 
 		if r.Cached {
 			// Copy updated repository.yaml file to repo dir now that the tree is synced.
-			err = helpers.CopyFile(file, filepath.Join(repobasedir, REPOSITORY_SPECFILE))
+			err = fileHelper.CopyFile(file, filepath.Join(repobasedir, REPOSITORY_SPECFILE))
 			if err != nil {
 				return nil, errors.Wrap(err, "Error on update "+REPOSITORY_SPECFILE)
 			}

pkg/installer/repository_docker.go

@@ -24,16 +24,16 @@ import (
 	"strings"
 	"time"
 
-	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
-
+	"github.com/mudler/luet/pkg/bus"
+	compiler "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
+	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/helpers/docker"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
-	"github.com/mudler/luet/pkg/bus"
-	compiler "github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/config"
-	"github.com/mudler/luet/pkg/helpers"
 	"github.com/pkg/errors"
 )
 
@@ -166,7 +166,7 @@ func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArt
 	if err != nil {
 		return errors.Wrap(err, "failed generating checksums for tree")
 	}
-	imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, helpers.StripInvalidStringsFromImage(a.GetFileName()))
+	imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, docker.StripInvalidStringsFromImage(a.GetFileName()))
 	if checkIfExists && d.imagePush && d.b.ImageAvailable(imageTree) && !d.force {
 		Info("Image", imageTree, "already present, skipping. use --force-push to override")
 		return nil

pkg/installer/repository_test.go

@@ -26,14 +26,15 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
-	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
-
 	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	config "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -83,34 +84,34 @@ var _ = Describe("Repository", func() {
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 		})
 
 		It("Generate repository metadata of files ONLY referenced in a tree", func() {
@@ -156,41 +157,41 @@ var _ = Describe("Repository", func() {
 
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
 			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			artifact2, err := compiler2.Compile(false, spec2)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(artifact2.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(artifact2.Path)).To(BeTrue())
 			Expect(helpers.Untar(artifact2.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
-			content1, err := helpers.Read(spec.Rel("test5"))
+			content1, err := fileHelper.Read(spec.Rel("test5"))
 			Expect(err).ToNot(HaveOccurred())
-			content2, err := helpers.Read(spec.Rel("test6"))
+			content2, err := fileHelper.Read(spec.Rel("test6"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content1).To(Equal("artifact5\n"))
 			Expect(content2).To(Equal("artifact6\n"))
 
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
-			Expect(helpers.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := stubRepo(tmpdir, "../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 
 			// We check now that the artifact not referenced in the tree
 			// (spec2) is not indexed in the repository
@@ -267,18 +268,18 @@ urls:
 
 			a, err := localcompiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(a.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(a.Path)).To(BeTrue())
 			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
 
 			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/buildable", repoImage, true, true)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(repoImage, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
@@ -295,7 +296,7 @@ urls:
 
 			f, err := c.DownloadFile("repository.yaml")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(f)).To(ContainSubstring("name: test"))
+			Expect(fileHelper.Read(f)).To(ContainSubstring("name: test"))
 
 			a, err = c.DownloadArtifact(&artifact.PackageArtifact{
 				Path: "test.tar",
@@ -310,7 +311,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(a.Unpack(extracted, false)).ToNot(HaveOccurred())
-			Expect(helpers.Read(filepath.Join(extracted, "test6"))).To(Equal("artifact6\n"))
+			Expect(fileHelper.Read(filepath.Join(extracted, "test6"))).To(Equal("artifact6\n"))
 		})
 
 		It("generates images of virtual packages", func() {
@@ -341,15 +342,15 @@ urls:
 
 			a, err := localcompiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(a.Path)).To(BeTrue())
+			Expect(fileHelper.Exists(a.Path)).To(BeTrue())
 			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
 
 			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/virtuals", repoImage, true, true)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(repoImage, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
@@ -366,7 +367,7 @@ urls:
 
 			f, err := c.DownloadFile("repository.yaml")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Read(f)).To(ContainSubstring("name: test"))
+			Expect(fileHelper.Read(f)).To(ContainSubstring("name: test"))
 
 			a, err = c.DownloadArtifact(&artifact.PackageArtifact{
 				Path: "test.tar",
@@ -382,7 +383,7 @@ urls:
 
 			Expect(a.Unpack(extracted, false)).ToNot(HaveOccurred())
 
-			Expect(helpers.DirectoryIsEmpty(extracted)).To(BeFalse())
+			Expect(fileHelper.DirectoryIsEmpty(extracted)).To(BeFalse())
 			content, err := ioutil.ReadFile(filepath.Join(extracted, ".virtual"))
 			Expect(err).ToNot(HaveOccurred())
 

pkg/installer/system.go

@@ -1,16 +1,21 @@
 package installer
 
 import (
+	"sync"
+
 	"github.com/hashicorp/go-multierror"
 	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 )
 
 type System struct {
-	Database pkg.PackageDatabase
-	Target   string
+	Database  pkg.PackageDatabase
+	Target    string
+	fileIndex map[string]pkg.Package
+	sync.Mutex
 }
 
 func (s *System) World() (pkg.Packages, error) {
@@ -23,8 +28,8 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
 	var errs error
 	executedFinalizer := map[string]bool{}
 	for _, p := range packs {
-		if helpers.Exists(p.Rel(tree.FinalizerFile)) {
-			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile))
+		if fileHelper.Exists(p.Rel(tree.FinalizerFile)) {
+			out, err := helpers.RenderFiles(helpers.ChartFile(p.Rel(tree.FinalizerFile)), p.Rel(tree.DefinitionFile))
 			if err != nil {
 				Warning("Failed rendering finalizer for ", p.HumanReadableString(), err.Error())
 				errs = multierror.Append(errs, err)
@@ -51,3 +56,38 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
 	}
 	return errs
 }
+
+func (s *System) buildFileIndex() {
+	s.Lock()
+	defer s.Unlock()
+	// Check if cache is empty or if it got modified
+	if s.fileIndex == nil { //|| len(s.Database.GetPackages()) != len(s.fileIndex) {
+		s.fileIndex = make(map[string]pkg.Package)
+		for _, p := range s.Database.World() {
+			files, _ := s.Database.GetPackageFiles(p)
+			for _, f := range files {
+				s.fileIndex[f] = p
+			}
+		}
+	}
+}
+
+func (s *System) Clean() {
+	s.Lock()
+	defer s.Unlock()
+	s.fileIndex = nil
+}
+
+func (s *System) ExistsPackageFile(file string) (bool, pkg.Package, error) {
+	Debug("Checking if file ", file, "belongs to any package")
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	if p, exists := s.fileIndex[file]; exists {
+		Debug(file, "belongs already to", p.HumanReadableString())
+
+		return exists, p, nil
+	}
+	Debug(file, "doesn't belong to any package")
+	return false, nil, nil
+}

pkg/installer/system_test.go

@@ -0,0 +1,70 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer_test
+
+import (
+
+	//	. "github.com/mudler/luet/pkg/installer"
+
+	. "github.com/mudler/luet/pkg/installer"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("System", func() {
+	Context("Files", func() {
+		var s *System
+		var db pkg.PackageDatabase
+		var a, b *pkg.DefaultPackage
+
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			s = &System{Database: db}
+
+			a = &pkg.DefaultPackage{Name: "test", Version: "1", Category: "t"}
+
+			db.CreatePackage(a)
+			db.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.GetFingerPrint(), Files: []string{"foo", "f"}})
+
+			b = &pkg.DefaultPackage{Name: "test2", Version: "1", Category: "t"}
+
+			db.CreatePackage(b)
+			db.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: b.GetFingerPrint(), Files: []string{"barz", "f"}})
+		})
+
+		It("detects when are already shipped by other packages", func() {
+			r, p, err := s.ExistsPackageFile("foo")
+			Expect(r).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(Equal(a))
+			r, p, err = s.ExistsPackageFile("baz")
+			Expect(r).To(BeFalse())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(BeNil())
+
+			r, p, err = s.ExistsPackageFile("f")
+			Expect(r).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(Equal(b))
+			r, p, err = s.ExistsPackageFile("barz")
+			Expect(r).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(Equal(b))
+		})
+	})
+})

pkg/package/database_mem.go

@@ -143,6 +143,7 @@ func (db *InMemoryDatabase) getRevdeps(p Package, visited map[string]interface{}
 	if err != nil {
 		return res, err
 	}
+
 	for _, pp := range packs {
 		//	db.Lock()
 		list := db.RevDepsDatabase[pp.GetFingerPrint()]
@@ -195,6 +196,14 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 	return ID, nil
 }
 
+func (db *InMemoryDatabase) updateRevDep(k, v string, b Package) {
+	_, ok := db.RevDepsDatabase[k]
+	if !ok {
+		db.RevDepsDatabase[k] = make(map[string]Package)
+	}
+	db.RevDepsDatabase[k][v] = b.Clone()
+}
+
 func (db *InMemoryDatabase) populateCaches(p Package) {
 	pd, _ := p.(*DefaultPackage)
 
@@ -226,20 +235,19 @@ func (db *InMemoryDatabase) populateCaches(p Package) {
 	}
 	db.CacheNoVersion[p.GetPackageName()][p.GetVersion()] = nil
 
+	db.Unlock()
+
 	// Updating Revdeps
 	// Given that when we populate the cache we don't have the full db at hand
 	// We cycle over reverse dependency of a package to update their entry if they are matching
 	// the version selector
+	db.Lock()
 	toUpdate, ok := db.RevDepsDatabase[pd.GetPackageName()]
 	if ok {
 		for _, pp := range toUpdate {
 			for _, re := range pp.GetRequires() {
 				if match, _ := pd.VersionMatchSelector(re.GetVersion(), nil); match {
-					_, ok = db.RevDepsDatabase[pd.GetFingerPrint()]
-					if !ok {
-						db.RevDepsDatabase[pd.GetFingerPrint()] = make(map[string]Package)
-					}
-					db.RevDepsDatabase[pd.GetFingerPrint()][pp.GetFingerPrint()] = pp
+					db.updateRevDep(pd.GetFingerPrint(), pp.GetFingerPrint(), pp)
 				}
 			}
 		}
@@ -249,30 +257,12 @@ func (db *InMemoryDatabase) populateCaches(p Package) {
 	for _, re := range pd.GetRequires() {
 		packages, _ := db.FindPackages(re)
 		db.Lock()
-
 		for _, pa := range packages {
-			_, ok := db.RevDepsDatabase[pa.GetFingerPrint()]
-			if !ok {
-				db.RevDepsDatabase[pa.GetFingerPrint()] = make(map[string]Package)
-			}
-			db.RevDepsDatabase[pa.GetFingerPrint()][pd.GetFingerPrint()] = pd
-			_, ok = db.RevDepsDatabase[pa.GetPackageName()]
-			if !ok {
-				db.RevDepsDatabase[pa.GetPackageName()] = make(map[string]Package)
-			}
-			db.RevDepsDatabase[pa.GetPackageName()][pd.GetPackageName()] = pd
+			db.updateRevDep(pa.GetFingerPrint(), pd.GetFingerPrint(), pd)
+			db.updateRevDep(pa.GetPackageName(), pd.GetPackageName(), pd)
 		}
-		_, ok := db.RevDepsDatabase[re.GetFingerPrint()]
-		if !ok {
-			db.RevDepsDatabase[re.GetFingerPrint()] = make(map[string]Package)
-		}
-		db.RevDepsDatabase[re.GetFingerPrint()][pd.GetFingerPrint()] = pd
-		_, ok = db.RevDepsDatabase[re.GetPackageName()]
-		if !ok {
-			db.RevDepsDatabase[re.GetPackageName()] = make(map[string]Package)
-		}
-		db.RevDepsDatabase[re.GetPackageName()][pd.GetPackageName()] = pd
-
+		db.updateRevDep(re.GetFingerPrint(), pd.GetFingerPrint(), pd)
+		db.updateRevDep(re.GetPackageName(), pd.GetPackageName(), pd)
 		db.Unlock()
 	}
 }

pkg/package/package.go

@@ -26,7 +26,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/helpers/docker"
+	"github.com/mudler/luet/pkg/helpers/match"
 	version "github.com/mudler/luet/pkg/versioner"
 
 	gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
@@ -135,6 +136,18 @@ type Packages []Package
 
 type DefaultPackages []*DefaultPackage
 
+type PackageMap map[string]Package
+
+func (pm PackageMap) String() string {
+	rr := []string{}
+	for _, r := range pm {
+
+		rr = append(rr, r.HumanReadableString())
+
+	}
+	return fmt.Sprint(rr)
+}
+
 func (d DefaultPackages) Hash(salt string) string {
 
 	overallFp := ""
@@ -324,7 +337,7 @@ func (p *DefaultPackage) GetPackageName() string {
 }
 
 func (p *DefaultPackage) ImageID() string {
-	return helpers.StripInvalidStringsFromImage(p.GetFingerPrint())
+	return docker.StripInvalidStringsFromImage(p.GetFingerPrint())
 }
 
 // GetBuildTimestamp returns the package build timestamp
@@ -359,19 +372,19 @@ func (p *DefaultPackage) IsHidden() bool {
 }
 
 func (p *DefaultPackage) HasLabel(label string) bool {
-	return helpers.MapHasKey(&p.Labels, label)
+	return match.MapHasKey(&p.Labels, label)
 }
 
 func (p *DefaultPackage) MatchLabel(r *regexp.Regexp) bool {
-	return helpers.MapMatchRegex(&p.Labels, r)
+	return match.MapMatchRegex(&p.Labels, r)
 }
 
 func (p *DefaultPackage) HasAnnotation(label string) bool {
-	return helpers.MapHasKey(&p.Annotations, label)
+	return match.MapHasKey(&p.Annotations, label)
 }
 
 func (p *DefaultPackage) MatchAnnotation(r *regexp.Regexp) bool {
-	return helpers.MapMatchRegex(&p.Annotations, r)
+	return match.MapMatchRegex(&p.Annotations, r)
 }
 
 // AddUse adds a use to a package

pkg/solver/resolver.go

@@ -1,4 +1,4 @@
-// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+// Copyright © 2020-2021 Ettore Di Giacinto <mudler@mocaccino.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -16,11 +16,16 @@
 package solver
 
 import (
+	"bufio"
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"strconv"
+	"strings"
 
 	"github.com/crillab/gophersat/bf"
+	"github.com/crillab/gophersat/explain"
 	"github.com/mudler/luet/pkg/helpers"
 	"gopkg.in/yaml.v2"
 
@@ -51,18 +56,115 @@ const (
 	QLearningResolverType = "qlearning"
 )
 
-//. "github.com/mudler/luet/pkg/logger"
-
 // PackageResolver assists PackageSolver on unsat cases
 type PackageResolver interface {
 	Solve(bf.Formula, PackageSolver) (PackagesAssertions, error)
 }
 
-type DummyPackageResolver struct {
+type Explainer struct{}
+
+func decodeDimacs(vars map[string]string, dimacs string) (string, error) {
+	res := ""
+	sc := bufio.NewScanner(bytes.NewBufferString(dimacs))
+	lines := strings.Split(dimacs, "\n")
+	linenum := 1
+SCAN:
+	for sc.Scan() {
+
+		line := sc.Text()
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			continue
+		}
+		switch fields[0] {
+		case "p":
+			continue SCAN
+		default:
+			for i := 0; i < len(fields)-1; i++ {
+				v := fields[i]
+				negative := false
+				if strings.HasPrefix(fields[i], "-") {
+					v = strings.TrimLeft(fields[i], "-")
+					negative = true
+				}
+				variable := vars[v]
+				if negative {
+					res += fmt.Sprintf("!(%s)", variable)
+				} else {
+					res += variable
+				}
+
+				if i != len(fields)-2 {
+					res += fmt.Sprintf(" or ")
+				}
+			}
+			if linenum != len(lines)-1 {
+				res += fmt.Sprintf(" and \n")
+			}
+		}
+		linenum++
+	}
+	if err := sc.Err(); err != nil {
+		return res, fmt.Errorf("could not parse problem: %v", err)
+	}
+	return res, nil
 }
 
-func (*DummyPackageResolver) Solve(bf.Formula, PackageSolver) (PackagesAssertions, error) {
-	return nil, errors.New("Could not satisfy the constraints. Try again by removing deps ")
+func parseVars(r io.Reader) (map[string]string, error) {
+	sc := bufio.NewScanner(r)
+	res := map[string]string{}
+	for sc.Scan() {
+		line := sc.Text()
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			continue
+		}
+		switch fields[0] {
+		case "c":
+			data := strings.Split(fields[1], "=")
+			res[data[1]] = data[0]
+
+		default:
+			continue
+
+		}
+	}
+	if err := sc.Err(); err != nil {
+		return nil, fmt.Errorf("could not parse problem: %v", err)
+	}
+	return res, nil
+}
+
+// Solve tries to find the MUS (minimum unsat) formula from the original problem.
+// it returns an error with the decoded dimacs
+func (*Explainer) Solve(f bf.Formula, s PackageSolver) (PackagesAssertions, error) {
+	buf := bytes.NewBufferString("")
+	if err := bf.Dimacs(f, buf); err != nil {
+		return nil, errors.Wrap(err, "cannot extract dimacs from formula")
+	}
+
+	copy := *buf
+
+	pb, err := explain.ParseCNF(&copy)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not parse problem")
+	}
+	pb2, err := pb.MUS()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not extract subset")
+	}
+
+	variables, err := parseVars(buf)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not parse variables")
+	}
+
+	res, err := decodeDimacs(variables, pb2.CNF())
+	if err != nil {
+		return nil, errors.Wrap(err, "could not parse dimacs")
+	}
+
+	return nil, fmt.Errorf("could not satisfy the constraints: \n%s", res)
 }
 
 type QLearningResolver struct {
@@ -103,8 +205,8 @@ func (resolver *QLearningResolver) Solve(f bf.Formula, s PackageSolver) (Package
 	//	Info("Using QLearning solver to resolve conflicts. Please be patient.")
 	resolver.Solver = s
 
-	s.SetResolver(&DummyPackageResolver{}) // Set dummy. Otherwise the attempts will run again a QLearning instance.
-	defer s.SetResolver(resolver)          // Set back ourselves as resolver
+	s.SetResolver(&Explainer{})   // Set dummy. Otherwise the attempts will run again a QLearning instance.
+	defer s.SetResolver(resolver) // Set back ourselves as resolver
 
 	resolver.Formula = f
 

pkg/solver/resolver_test.go

@@ -38,7 +38,7 @@ var _ = Describe("Resolver", func() {
 	})
 
 	Context("Conflict set", func() {
-		Context("DummyPackageResolver", func() {
+		Context("Explainer", func() {
 			It("is unsolvable - as we something we ask to install conflict with system stuff", func() {
 				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
@@ -152,7 +152,7 @@ var _ = Describe("Resolver", func() {
 			})
 		})
 
-		Context("DummyPackageResolver", func() {
+		Context("Explainer", func() {
 			It("cannot find a solution", func() {
 				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
@@ -171,6 +171,11 @@ var _ = Describe("Resolver", func() {
 
 				solution, err := s.Install([]pkg.Package{A, D})
 				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(Equal(`could not satisfy the constraints: 
+A-- and 
+C-- and 
+!(A--) or B-- and 
+!(B--) or !(C--)`))
 
 				Expect(len(solution)).To(Equal(0))
 			})

pkg/solver/solver.go

@@ -71,7 +71,7 @@ type Options struct {
 // NewSolver accepts as argument two lists of packages, the first is the initial set,
 // the second represent all the known packages.
 func NewSolver(t Options, installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase) PackageSolver {
-	return NewResolver(t, installed, definitiondb, solverdb, &DummyPackageResolver{})
+	return NewResolver(t, installed, definitiondb, solverdb, &Explainer{})
 }
 
 // NewResolver accepts as argument two lists of packages, the first is the initial set,

pkg/tree/compiler_recipe.go

@@ -26,6 +26,7 @@ import (
 	"path/filepath"
 
 	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/pkg/errors"
 )
@@ -61,7 +62,7 @@ type CompilerRecipe struct {
 // and the build context required for reproducible builds
 func (r *CompilerRecipe) Save(path string) error {
 	for _, p := range r.SourcePath {
-		if err := helpers.CopyDir(p, filepath.Join(path, filepath.Base(p))); err != nil {
+		if err := fileHelper.CopyDir(p, filepath.Join(path, filepath.Base(p))); err != nil {
 			return errors.Wrap(err, "while copying source tree")
 		}
 	}
@@ -75,6 +76,10 @@ func (r *CompilerRecipe) Load(path string) error {
 	//if err != nil {
 	//	return err
 	//}
+	c, err := helpers.ChartFiles([]string{filepath.Join(path, "templates")})
+	if err != nil {
+		return err
+	}
 
 	//r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))
 	// TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean()
@@ -102,9 +107,8 @@ func (r *CompilerRecipe) Load(path string) error {
 
 			// Instead of rdeps, have a different tree for build deps.
 			compileDefPath := pack.Rel(CompilerDefinitionFile)
-			if helpers.Exists(compileDefPath) {
-
-				dat, err := helpers.RenderFiles(compileDefPath, currentpath)
+			if fileHelper.Exists(compileDefPath) {
+				dat, err := helpers.RenderFiles(append(c, helpers.ChartFile(compileDefPath)...), currentpath)
 				if err != nil {
 					return errors.Wrap(err,
 						"Error templating file "+CompilerDefinitionFile+" from "+
@@ -149,14 +153,14 @@ func (r *CompilerRecipe) Load(path string) error {
 
 				// Instead of rdeps, have a different tree for build deps.
 				compileDefPath := pack.Rel(CompilerDefinitionFile)
-				if helpers.Exists(compileDefPath) {
+				if fileHelper.Exists(compileDefPath) {
 
 					raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
 					buildyaml, err := ioutil.ReadFile(compileDefPath)
 					if err != nil {
 						return errors.Wrap(err, "Error reading file "+currentpath)
 					}
-					dat, err := helpers.RenderHelm(string(buildyaml), raw, map[string]interface{}{})
+					dat, err := helpers.RenderHelm(append(c, helpers.ChartFileB(buildyaml)...), raw, map[string]interface{}{})
 					if err != nil {
 						return errors.Wrap(err,
 							"Error templating file "+CompilerDefinitionFile+" from "+
@@ -182,7 +186,7 @@ func (r *CompilerRecipe) Load(path string) error {
 		return nil
 	}
 
-	err := filepath.Walk(path, ff)
+	err = filepath.Walk(path, ff)
 	if err != nil {
 		return err
 	}

pkg/tree/installer_recipe.go

@@ -26,8 +26,9 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
+
 	"github.com/pkg/errors"
 )
 
@@ -61,8 +62,8 @@ func (r *InstallerRecipe) Save(path string) error {
 		}
 		// Instead of rdeps, have a different tree for build deps.
 		finalizerPath := p.Rel(FinalizerFile)
-		if helpers.Exists(finalizerPath) { // copy finalizer file from the source tree
-			helpers.CopyFile(finalizerPath, filepath.Join(dir, FinalizerFile))
+		if fileHelper.Exists(finalizerPath) { // copy finalizer file from the source tree
+			fileHelper.CopyFile(finalizerPath, filepath.Join(dir, FinalizerFile))
 		}
 
 	}
@@ -71,7 +72,7 @@ func (r *InstallerRecipe) Save(path string) error {
 
 func (r *InstallerRecipe) Load(path string) error {
 
-	if !helpers.Exists(path) {
+	if !fileHelper.Exists(path) {
 		return errors.New(fmt.Sprintf(
 			"Path %s doesn't exit.", path,
 		))

pkg/tree/recipes.go

@@ -26,9 +26,10 @@ import (
 	"os"
 	"path/filepath"
 
-	helpers "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	spectooling "github.com/mudler/luet/pkg/spectooling"
+
 	"github.com/pkg/errors"
 )
 
@@ -77,7 +78,7 @@ func (r *Recipe) Load(path string) error {
 	// if err != nil {
 	// 	return err
 	// }
-	if !helpers.Exists(path) {
+	if !fileHelper.Exists(path) {
 		return errors.New(fmt.Sprintf(
 			"Path %s doesn't exit.", path,
 		))

tests/fixtures/fileconflicts/conflict1/build.yaml

@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+  - mkdir /foo
+steps:
+  - echo conflict > /foo/test1
+package_dir: /foo

tests/fixtures/fileconflicts/conflict1/collection.yaml

@@ -0,0 +1,7 @@
+packages:
+- category: "test1"
+  name: "conflict"
+  version: "1.0"
+- category: "test2"
+  name: "conflict"
+  version: "1.0"

tests/fixtures/fileconflicts_upgrade/conflict1/build.yaml

@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+  - mkdir /foo
+steps:
+  - echo conflict > /foo/test1
+package_dir: /foo

tests/fixtures/fileconflicts_upgrade/conflict1/collection.yaml

@@ -0,0 +1,13 @@
+packages:
+- category: "test1"
+  name: "conflict"
+  version: "1.0"
+- category: "test2"
+  name: "conflict"
+  version: "1.0"
+- category: "test1"
+  name: "conflict"
+  version: "1.1"
+- category: "test2"
+  name: "conflict"
+  version: "1.1"

tests/fixtures/nofileconflicts_upgrade/noconflict/build.yaml

@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+  - mkdir /foo
+steps:
+  - echo conflict > /foo/test1
+package_dir: /foo

tests/fixtures/nofileconflicts_upgrade/noconflict/collection.yaml

@@ -0,0 +1,7 @@
+packages:
+- category: "test1"
+  name: "noconflict"
+  version: "1.0"
+- category: "test1"
+  name: "noconflict"
+  version: "1.1"

tests/fixtures/shared_templates/templates/writefile.yaml

@@ -0,0 +1,3 @@
+{{ define "writefile" }}
+- echo conflict > /foo/{{.}}
+{{end}}

tests/fixtures/shared_templates/test/build.yaml

@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+  - mkdir /foo
+steps:
+{{ template "writefile" .Values.name }}
+package_dir: /foo

tests/fixtures/shared_templates/test/collection.yaml

@@ -0,0 +1,7 @@
+packages:
+- category: "test"
+  name: "foo"
+  version: "1.0"
+- category: "test"
+  name: "bar"
+  version: "1.0"

tests/integration/12_config_protect.sh

@@ -92,7 +92,7 @@ testUnInstall() {
     assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
     # TODO: we need remove it or not??
     assertTrue 'config protect created' "[ -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
-    assertTrue 'config protect maintains the protected files' "[ -e '$tmpdir/testrootfs/etc/a/conf' ]"
+    assertTrue 'config protect maintains the protected files' "[ ! -e '$tmpdir/testrootfs/etc/a/conf' ]"
 }
 
 

tests/integration/12_config_protect_samefile.sh

@@ -87,7 +87,7 @@ testInstall() {
 
 
 testUnInstall() {
-    luet uninstall -y --full --config $tmpdir/luet.yaml test/a
+    luet uninstall -y --full --keep-protected-files --config $tmpdir/luet.yaml test/a
     installst=$?
     assertEquals 'uninstall test successfully' "$installst" "0"
     assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"

tests/integration/13_config_protect_annotation.sh

@@ -86,7 +86,7 @@ testInstall() {
 
 
 testUnInstall() {
-    luet uninstall -y --full --config $tmpdir/luet.yaml test/a
+    luet uninstall -y --full --keep-protected-files --config $tmpdir/luet.yaml test/a
     installst=$?
     assertEquals 'uninstall test successfully' "$installst" "0"
     assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"

tests/integration/28_nobuildtreedocker.sh

@@ -51,9 +51,9 @@ testBuild() {
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available remotely for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:ac34beb3fc2752bab54549db095dd6994d7531b88e1ff7f902d01ae80fdd030d"
-    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available remotely for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:d8b8881d50ae83646728bb28ad678fc14e4e003e4b0d0a66f8e6167c6116e024"
-    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2aaccd929ebe5683f95c70c6ec8d68f240ceea8633f3350b94904ad73da5fd47"
+    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available remotely for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:7a46e5c8893b574d9f984eaade75b0e68cc891bb6d01860ba91a62f9e4de2b62"
+    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available remotely for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:4277934ece4de17856b87294301cdfcdf3c6f956e682ff64697be67178b8a4b1"
+    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:132c097844b4e809efa8ae234452893b0214a1c860371e21564d89655ab10a56"
 }
 
 testRepo() {

tests/integration/29_nobuildtreeinheritvalues.sh

@@ -56,8 +56,8 @@ EOF
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertNotContains 'Does NOT use the upstream cache without specifying it' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2aaccd929ebe5683f95c70c6ec8d68f240ceea8633f3350b94904ad73da5fd47"
-    assertContains 'Does generate a new hash as values changed build.yaml for test/interpolated-1.0+2 package image' "$build_output" "Building image luet/cache:c34b533cf76886c332fe9b2d3208f04265360a465a90c996cb4fcdaf959dee36 done"
+    assertNotContains 'Does NOT use the upstream cache without specifying it' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:132c097844b4e809efa8ae234452893b0214a1c860371e21564d89655ab10a56"
+    assertContains 'Does generate a new hash as values changed build.yaml for test/interpolated-1.0+2 package image' "$build_output" "Building image luet/cache:a7edf5c9ab219e406b64b2692ce08d56a8bcd212a43bb70df3737f13a008dfd4 done"
 }
 
 testRepo() {

tests/integration/29_nobuildtreeinheritvalues_noignored.sh

@@ -56,7 +56,7 @@ EOF
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Downloading image quay.io/mocaccinoos/integration-test-cache:4db24406e8db30a3310a1cf8c4d4e19597745e6d41b189dc51a73ac4a50cc9e6"
+    assertContains 'Does use the upstream cache without specifying it (test/c-1.0)' "$build_output" "quay.io/mocaccinoos/integration-test-cache:a786d3fd29d0b8bdfe5f304c8bf8be909d5c764cd7059c0e63294a8bff17f3ef"
 }
 
 testRepo() {

tests/integration/32_fileconflicts.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/fileconflicts" --destination $tmpdir/testbuild --compression gzip --all
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test1-1.0.package.tar.gz' ]"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test2-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/fileconflicts" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install -y --config $tmpdir/luet.yaml test1/conflict test2/conflict
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test failed' "$installst" "1"
+    #assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testReInstall() {
+    luet install -y --config $tmpdir/luet.yaml test1/conflict
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "0"
+    luet install -y --config $tmpdir/luet.yaml test2/conflict
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "1"
+    luet install -y --force --config $tmpdir/luet.yaml test2/conflict
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/32_fileconflicts_upgrade.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/fileconflicts_upgrade" --destination $tmpdir/testbuild --compression gzip --all
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test1-1.0.package.tar.gz' ]"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test2-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/fileconflicts_upgrade" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install -y --force --config $tmpdir/luet.yaml test1/conflict@1.0 test2/conflict@1.0
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test succeded' "$installst" "0"
+    #assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testUpgrade() {
+    out=$(luet upgrade -y --config $tmpdir/luet.yaml)
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "1"
+    assertContains 'does find conflicts' "$out" "Error: file conflict found: file conflict between packages to be installed"
+
+    luet upgrade -y --config $tmpdir/luet.yaml --force
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/32_nofileconflicts_upgrade.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/nofileconflicts_upgrade" --destination $tmpdir/testbuild --compression gzip --all
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/noconflict-test1-1.0.package.tar.gz' ]"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/noconflict-test1-1.1.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/nofileconflicts_upgrade" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install -y --config $tmpdir/luet.yaml test1/noconflict@1.0
+    installst=$?
+    #assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testUpgrade() {
+    out=$(luet upgrade -y --config $tmpdir/luet.yaml)
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "0"
+    assertNotContains 'does find conflicts' "$out" "Error: file conflict found: file conflict between packages to be installed"
+
+    installed=$(luet --config $tmpdir/luet.yaml search --installed)
+    assertContains 'does upgrade' "$installed" "test1/noconflict-1.1"
+
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/33_reinstall.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/fileconflicts" --destination $tmpdir/testbuild --compression gzip --all
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test1-1.0.package.tar.gz' ]"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test2-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/fileconflicts" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testReInstall() {
+    luet install -y --config $tmpdir/luet.yaml test1/conflict
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "0"
+    luet reinstall -y --config $tmpdir/luet.yaml test1/conflict
+    installst=$?
+    assertEquals 'reinstall test succeeded' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/34_templates.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/shared_templates" --destination $tmpdir/testbuild --compression gzip --all
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/foo-test-1.0.package.tar.gz' ]"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/bar-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/shared_templates" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install -y --config $tmpdir/luet.yaml test/foo test/bar
+    installst=$?
+    assertEquals 'install test failed' "$installst" "0"
+    assertTrue 'package bar installed' "[ -e '$tmpdir/testrootfs/bar' ]"
+    assertTrue 'package foo installed' "[ -e '$tmpdir/testrootfs/foo' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

vendor/github.com/crillab/gophersat/explain/check.go

@@ -0,0 +1,149 @@
+// Package explain provides facilities to check and understand UNSAT instances.
+package explain
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/crillab/gophersat/solver"
+)
+
+// Options is a set of options that can be set to true during the checking process.
+type Options struct {
+	// If Verbose is true, information about resolution will be written on stdout.
+	Verbose bool
+}
+
+// Checks whether the clause satisfies the problem or not.
+// Will return true if the problem is UNSAT, false if it is SAT or indeterminate.
+func unsat(pb *Problem, clause []int) bool {
+	oldUnits := make([]int, len(pb.units))
+	copy(oldUnits, pb.units)
+	// lits is supposed to be implied by the problem.
+	// We add the negation of each lit as a unit clause to see if this is true.
+	for _, lit := range clause {
+		if lit > 0 {
+			pb.units[lit-1] = -1
+		} else {
+			pb.units[-lit-1] = 1
+		}
+	}
+	res := pb.unsat()
+	pb.units = oldUnits // We must restore the previous state
+	return res
+}
+
+// UnsatChan will wait RUP clauses from ch and use them as a certificate.
+// It will return true iff the certificate is valid, i.e iff it makes the problem UNSAT
+// through unit propagation.
+// If pb.Options.ExtractSubset is true, a subset will also be extracted for that problem.
+func (pb *Problem) UnsatChan(ch chan string) (valid bool, err error) {
+	defer pb.restore()
+	pb.initTagged()
+	for line := range ch {
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			continue
+		}
+		if _, err := strconv.Atoi(fields[0]); err != nil {
+			// This is not a clause: ignore the line
+			continue
+		}
+		clause, err := parseClause(fields)
+		if err != nil {
+			return false, err
+		}
+		if !unsat(pb, clause) {
+			return false, nil
+		}
+		if len(clause) == 0 {
+			// This is the empty and unit propagation made the problem UNSAT: we're done.
+			return true, nil
+		}
+		// Since clause is a logical consequence, append it to the problem
+		pb.Clauses = append(pb.Clauses, clause)
+	}
+
+	// If we did not send any information through the channel
+	// It implies that the problem is trivially unsatisfiable
+	// Since we had only unit clauses inside the channel.
+	return true, nil
+}
+
+// Unsat will parse a certificate, and return true iff the certificate is valid, i.e iff it makes the problem UNSAT
+// through unit propagation.
+// If pb.Options.ExtractSubset is true, a subset will also be extracted for that problem.
+func (pb *Problem) Unsat(cert io.Reader) (valid bool, err error) {
+	defer pb.restore()
+	pb.initTagged()
+	sc := bufio.NewScanner(cert)
+	for sc.Scan() {
+		line := sc.Text()
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			continue
+		}
+		if _, err := strconv.Atoi(fields[0]); err != nil {
+			// This is not a clause: ignore the line
+			continue
+		}
+		clause, err := parseClause(fields)
+		if err != nil {
+			return false, err
+		}
+		if !unsat(pb, clause) {
+			return false, nil
+		}
+		// Since clause is a logical consequence, append it to the problem
+		pb.Clauses = append(pb.Clauses, clause)
+	}
+	if err := sc.Err(); err != nil {
+		return false, fmt.Errorf("could not parse certificate: %v", err)
+	}
+	return true, nil
+}
+
+// ErrNotUnsat is the error returned when trying to get the MUS or UnsatSubset of a satisfiable problem.
+var ErrNotUnsat = fmt.Errorf("problem is not UNSAT")
+
+// UnsatSubset returns an unsatisfiable subset of the problem.
+// The subset is not guaranteed to be a MUS, meaning some clauses of the resulting
+// problem might be removed while still keeping the unsatisfiability of the problem.
+// However, this method is much more efficient than extracting a MUS, as it only calls
+// the SAT solver once.
+func (pb *Problem) UnsatSubset() (subset *Problem, err error) {
+	solverPb := solver.ParseSlice(pb.Clauses)
+	if solverPb.Status == solver.Unsat {
+		// Problem is trivially UNSAT
+		// Make a copy so that pb and pb2 are not the same value.
+		pb2 := *pb
+		return &pb2, nil
+	}
+	s := solver.New(solver.ParseSlice(pb.Clauses))
+	s.Certified = true
+	s.CertChan = make(chan string)
+	status := solver.Unsat
+	go func() {
+		status = s.Solve()
+		close(s.CertChan)
+	}()
+	if valid, err := pb.UnsatChan(s.CertChan); !valid || status == solver.Sat {
+		return nil, ErrNotUnsat
+	} else if err != nil {
+		return nil, fmt.Errorf("could not solve problem: %v", err)
+	}
+	subset = &Problem{
+		NbVars: pb.NbVars,
+	}
+	for i, clause := range pb.Clauses {
+		if pb.tagged[i] {
+			// clause was used to prove pb is UNSAT: it's part of the subset
+			subset.Clauses = append(subset.Clauses, clause)
+			subset.NbClauses++
+		}
+	}
+	return subset, nil
+}

vendor/github.com/crillab/gophersat/explain/mus.go

@@ -0,0 +1,213 @@
+package explain
+
+import (
+	"fmt"
+
+	"github.com/crillab/gophersat/solver"
+)
+
+// MUSMaxSat returns a Minimal Unsatisfiable Subset for the problem using the MaxSat strategy.
+// A MUS is an unsatisfiable subset such that, if any of its clause is removed,
+// the problem becomes satisfiable.
+// A MUS can be useful to understand why a problem is UNSAT, but MUSes are expensive to compute since
+// a SAT solver must be called several times on parts of the original problem to find them.
+// With the MaxSat strategy, the function computes the MUS through several calls to MaxSat.
+func (pb *Problem) MUSMaxSat() (mus *Problem, err error) {
+	pb2 := pb.clone()
+	nbVars := pb2.NbVars
+	NbClauses := pb2.NbClauses
+	weights := make([]int, NbClauses)          // Weights of each clause
+	relaxLits := make([]solver.Lit, NbClauses) // Set of all relax lits
+	relaxLit := nbVars + 1                     // Index of last used relax lit
+	for i, clause := range pb2.Clauses {
+		pb2.Clauses[i] = append(clause, relaxLit)
+		relaxLits[i] = solver.IntToLit(int32(relaxLit))
+		weights[i] = 1
+		relaxLit++
+	}
+	prob := solver.ParseSlice(pb2.Clauses)
+	prob.SetCostFunc(relaxLits, weights)
+	s := solver.New(prob)
+	s.Verbose = pb.Options.Verbose
+	var musClauses [][]int
+	done := make([]bool, NbClauses) // Indicates whether a clause is already part of MUS or not yet
+	for {
+		cost := s.Minimize()
+		if cost == -1 {
+			return makeMus(nbVars, musClauses), nil
+		}
+		if cost == 0 {
+			return nil, fmt.Errorf("cannot extract MUS from satisfiable problem")
+		}
+		model := s.Model()
+		for i, clause := range pb.Clauses {
+			if !done[i] && !satClause(clause, model) {
+				// The clause is part of the MUS
+				pb2.Clauses = append(pb2.Clauses, []int{-(nbVars + i + 1)}) // Now, relax lit has to be false
+				pb2.NbClauses++
+				musClauses = append(musClauses, clause)
+				done[i] = true
+				// Make it a hard clause before restarting solver
+				lits := make([]solver.Lit, len(clause))
+				for j, lit := range clause {
+					lits[j] = solver.IntToLit(int32(lit))
+				}
+				s.AppendClause(solver.NewClause(lits))
+			}
+		}
+		if pb.Options.Verbose {
+			fmt.Printf("c Currently %d/%d clauses in MUS\n", len(musClauses), NbClauses)
+		}
+		prob = solver.ParseSlice(pb2.Clauses)
+		prob.SetCostFunc(relaxLits, weights)
+		s = solver.New(prob)
+		s.Verbose = pb.Options.Verbose
+	}
+}
+
+// true iff the clause is satisfied by the model
+func satClause(clause []int, model []bool) bool {
+	for _, lit := range clause {
+		if (lit > 0 && model[lit-1]) || (lit < 0 && !model[-lit-1]) {
+			return true
+		}
+	}
+	return false
+}
+
+func makeMus(nbVars int, clauses [][]int) *Problem {
+	mus := &Problem{
+		Clauses:   clauses,
+		NbVars:    nbVars,
+		NbClauses: len(clauses),
+		units:     make([]int, nbVars),
+	}
+	for _, clause := range clauses {
+		if len(clause) == 1 {
+			lit := clause[0]
+			if lit > 0 {
+				mus.units[lit-1] = 1
+			} else {
+				mus.units[-lit-1] = -1
+			}
+		}
+	}
+	return mus
+}
+
+// MUSInsertion returns a Minimal Unsatisfiable Subset for the problem using the insertion method.
+// A MUS is an unsatisfiable subset such that, if any of its clause is removed,
+// the problem becomes satisfiable.
+// A MUS can be useful to understand why a problem is UNSAT, but MUSes are expensive to compute since
+// a SAT solver must be called several times on parts of the original problem to find them.
+// The insertion algorithm is efficient is many cases, as it calls the same solver several times in a row.
+// However, in some cases, the number of calls will be higher than using other methods.
+// For instance, if  called on a formula that is already a MUS, it will perform n*(n-1) calls to SAT, where
+// n is the number of clauses of the problem.
+func (pb *Problem) MUSInsertion() (mus *Problem, err error) {
+	pb2, err := pb.UnsatSubset()
+	if err != nil {
+		return nil, fmt.Errorf("could not extract MUS: %v", err)
+	}
+	mus = &Problem{NbVars: pb2.NbVars}
+	clauses := pb2.Clauses
+	for {
+		if pb.Options.Verbose {
+			fmt.Printf("c mus currently contains %d clauses\n", mus.NbClauses)
+		}
+		s := solver.New(solver.ParseSliceNb(mus.Clauses, mus.NbVars))
+		s.Verbose = pb.Options.Verbose
+		st := s.Solve()
+		if st == solver.Unsat { // Found the MUS
+			return mus, nil
+		}
+		// Add clauses until the problem becomes UNSAT
+		idx := 0
+		for st == solver.Sat {
+			clause := clauses[idx]
+			lits := make([]solver.Lit, len(clause))
+			for i, lit := range clause {
+				lits[i] = solver.IntToLit(int32(lit))
+			}
+			cl := solver.NewClause(lits)
+			s.AppendClause(cl)
+			idx++
+			st = s.Solve()
+		}
+		idx--                                           // We went one step too far, go back
+		mus.Clauses = append(mus.Clauses, clauses[idx]) // Last clause is part of the MUS
+		mus.NbClauses++
+		if pb.Options.Verbose {
+			fmt.Printf("c removing %d/%d clause(s)\n", len(clauses)-idx, len(clauses))
+		}
+		clauses = clauses[:idx] // Remaining clauses are not part of the MUS
+	}
+}
+
+// MUSDeletion returns a Minimal Unsatisfiable Subset for the problem using the insertion method.
+// A MUS is an unsatisfiable subset such that, if any of its clause is removed,
+// the problem becomes satisfiable.
+// A MUS can be useful to understand why a problem is UNSAT, but MUSes are expensive to compute since
+// a SAT solver must be called several times on parts of the original problem to find them.
+// The deletion algorithm is guaranteed to call exactly n SAT solvers, where n is the number of clauses in the problem.
+// It can be quite efficient, but each time the solver is called, it is starting from scratch.
+// Other methods keep the solver "hot", so despite requiring more calls, these methods can be more efficient in practice.
+func (pb *Problem) MUSDeletion() (mus *Problem, err error) {
+	pb2, err := pb.UnsatSubset()
+	if err != nil {
+		if err == ErrNotUnsat {
+			return nil, err
+		}
+		return nil, fmt.Errorf("could not extract MUS: %v", err)
+	}
+	pb2.NbVars += pb2.NbClauses          // Add one relax var for each clause
+	for i, clause := range pb2.Clauses { // Add relax lit to each clause
+		newClause := make([]int, len(clause)+1)
+		copy(newClause, clause)
+		newClause[len(clause)] = pb.NbVars + i + 1 // Add relax lit to the clause
+		pb2.Clauses[i] = newClause
+	}
+	s := solver.New(solver.ParseSlice(pb2.Clauses))
+	asumptions := make([]solver.Lit, pb2.NbClauses)
+	for i := 0; i < pb2.NbClauses; i++ {
+		asumptions[i] = solver.IntToLit(int32(-(pb.NbVars + i + 1))) // At first, all asumptions are false
+	}
+	for i := range pb2.Clauses {
+		// Relax current clause
+		asumptions[i] = asumptions[i].Negation()
+		s.Assume(asumptions)
+		if s.Solve() == solver.Sat {
+			// It is now sat; reinsert the clause, i.e re-falsify the relax lit
+			asumptions[i] = asumptions[i].Negation()
+			if pb.Options.Verbose {
+				fmt.Printf("c clause %d/%d: kept\n", i+1, pb2.NbClauses)
+			}
+		} else if pb.Options.Verbose {
+			fmt.Printf("c clause %d/%d: removed\n", i+1, pb2.NbClauses)
+		}
+	}
+	mus = &Problem{
+		NbVars: pb.NbVars,
+	}
+	for i, val := range asumptions {
+		if !val.IsPositive() {
+			// Lit is not relaxed, meaning the clause is part of the MUS
+			clause := pb2.Clauses[i]
+			clause = clause[:len(clause)-1] // Remove relax lit
+			mus.Clauses = append(mus.Clauses, clause)
+		}
+		mus.NbClauses = len(mus.Clauses)
+	}
+	return mus, nil
+}
+
+// MUS returns a Minimal Unsatisfiable Subset for the problem.
+// A MUS is an unsatisfiable subset such that, if any of its clause is removed,
+// the problem becomes satisfiable.
+// A MUS can be useful to understand why a problem is UNSAT, but MUSes are expensive to compute since
+// a SAT solver must be called several times on parts of the original problem to find them.
+// The exact algorithm used to compute the MUS is not guaranteed. If you want to use a given algorithm,
+// use the relevant functions.
+func (pb *Problem) MUS() (mus *Problem, err error) {
+	return pb.MUSDeletion()
+}

vendor/github.com/crillab/gophersat/explain/parser.go

@@ -0,0 +1,104 @@
+package explain
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+)
+
+// parseClause parses a line representing a clause in the DIMACS CNF syntax.
+func parseClause(fields []string) ([]int, error) {
+	clause := make([]int, 0, len(fields)-1)
+	for _, rawLit := range fields {
+		lit, err := strconv.Atoi(rawLit)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse clause %v: %v", fields, err)
+		}
+		if lit != 0 {
+			clause = append(clause, lit)
+		}
+	}
+	return clause, nil
+}
+
+// ParseCNF parses a CNF and returns the associated problem.
+func ParseCNF(r io.Reader) (*Problem, error) {
+	sc := bufio.NewScanner(r)
+	var pb Problem
+	for sc.Scan() {
+		line := sc.Text()
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			continue
+		}
+		switch fields[0] {
+		case "c":
+			continue
+		case "p":
+			if err := pb.parseHeader(fields); err != nil {
+				return nil, fmt.Errorf("could not parse header %q: %v", line, err)
+			}
+		default:
+			if err := pb.parseClause(fields); err != nil {
+				return nil, fmt.Errorf("could not parse clause %q: %v", line, err)
+			}
+		}
+	}
+	if err := sc.Err(); err != nil {
+		return nil, fmt.Errorf("could not parse problem: %v", err)
+	}
+	return &pb, nil
+}
+
+func (pb *Problem) parseHeader(fields []string) error {
+	if len(fields) != 4 {
+		return fmt.Errorf("expected 4 fields, got %d", len(fields))
+	}
+	strVars := fields[2]
+	strClauses := fields[3]
+	var err error
+	pb.NbVars, err = strconv.Atoi(fields[2])
+	if err != nil {
+		return fmt.Errorf("invalid number of vars %q: %v", strVars, err)
+	}
+	if pb.NbVars < 0 {
+		return fmt.Errorf("negative number of vars %d", pb.NbVars)
+	}
+	pb.units = make([]int, pb.NbVars)
+	pb.NbClauses, err = strconv.Atoi(fields[3])
+	if err != nil {
+		return fmt.Errorf("invalid number of clauses %s: %v", strClauses, err)
+	}
+	if pb.NbClauses < 0 {
+		return fmt.Errorf("negative number of clauses %d", pb.NbClauses)
+	}
+	pb.Clauses = make([][]int, 0, pb.NbClauses)
+	return nil
+}
+
+func (pb *Problem) parseClause(fields []string) error {
+	clause, err := parseClause(fields)
+	if err != nil {
+		return err
+	}
+	pb.Clauses = append(pb.Clauses, clause)
+	if len(clause) == 1 {
+		lit := clause[0]
+		v := lit
+		if lit < 0 {
+			v = -v
+		}
+		if v > pb.NbVars {
+			// There was an error in the header
+			return fmt.Errorf("found lit %d but problem was supposed to hold only %d vars", lit, pb.NbVars)
+		}
+		if lit > 0 {
+			pb.units[v-1] = 1
+		} else {
+			pb.units[v-1] = -1
+		}
+	}
+	return nil
+}

vendor/github.com/crillab/gophersat/explain/problem.go

@@ -0,0 +1,125 @@
+package explain
+
+import (
+	"fmt"
+	"strings"
+)
+
+// A Problem is a conjunction of Clauses.
+// This package does not use solver's representation.
+// We want this code to be as simple as possible to be easy to audit.
+// On the other hand, solver's code must be as efficient as possible.
+type Problem struct {
+	Clauses   [][]int
+	NbVars    int
+	NbClauses int
+	units     []int // For each var, 0 if the var is unbound, 1 if true, -1 if false
+	Options   Options
+	tagged    []bool // List of claused used whil proving the problem is unsat. Initialized lazily
+}
+
+func (pb *Problem) initTagged() {
+	pb.tagged = make([]bool, pb.NbClauses)
+	for i, clause := range pb.Clauses {
+		// Unit clauses are tagged as they will probably be used during resolution
+		pb.tagged[i] = len(clause) == 1
+	}
+}
+
+func (pb *Problem) clone() *Problem {
+	pb2 := &Problem{
+		Clauses:   make([][]int, pb.NbClauses),
+		NbVars:    pb.NbVars,
+		NbClauses: pb.NbClauses,
+		units:     make([]int, pb.NbVars),
+	}
+	copy(pb2.units, pb.units)
+	for i, clause := range pb.Clauses {
+		pb2.Clauses[i] = make([]int, len(clause))
+		copy(pb2.Clauses[i], clause)
+	}
+	return pb2
+}
+
+// restore removes all learned clauses, if any.
+func (pb *Problem) restore() {
+	pb.Clauses = pb.Clauses[:pb.NbClauses]
+}
+
+// unsat will be true iff the problem can be proven unsat through unit propagation.
+// This methods modifies pb.units.
+func (pb *Problem) unsat() bool {
+	done := make([]bool, len(pb.Clauses)) // clauses that were deemed sat during propagation
+	modified := true
+	for modified {
+		modified = false
+		for i, clause := range pb.Clauses {
+			if done[i] { // That clause was already proved true
+				continue
+			}
+			unbound := 0
+			var unit int // An unbound literal, if any
+			sat := false
+			for _, lit := range clause {
+				v := lit
+				if v < 0 {
+					v = -v
+				}
+				binding := pb.units[v-1]
+				if binding == 0 {
+					unbound++
+					if unbound == 1 {
+						unit = lit
+					} else {
+						break
+					}
+				} else if binding*lit == v { // (binding == -1 && lit < 0) || (binding == 1 && lit > 0) {
+					sat = true
+					break
+				}
+			}
+			if sat {
+				done[i] = true
+				continue
+			}
+			if unbound == 0 {
+				// All lits are false: problem is UNSAT
+				if i < pb.NbClauses {
+					pb.tagged[i] = true
+				}
+				return true
+			}
+			if unbound == 1 {
+				if unit < 0 {
+					pb.units[-unit-1] = -1
+				} else {
+					pb.units[unit-1] = 1
+				}
+				done[i] = true
+				if i < pb.NbClauses {
+					pb.tagged[i] = true
+				}
+				modified = true
+			}
+		}
+	}
+	// Problem is either sat or could not be proven unsat through unit propagation
+	return false
+}
+
+// CNF returns a representation of the problem using the Dimacs syntax.
+func (pb *Problem) CNF() string {
+	lines := make([]string, 1, pb.NbClauses+1)
+	lines[0] = fmt.Sprintf("p cnf %d %d", pb.NbVars, pb.NbClauses)
+	for i := 0; i < pb.NbClauses; i++ {
+		clause := pb.Clauses[i]
+		strClause := make([]string, len(clause)+1)
+		for i, lit := range clause {
+			strClause[i] = fmt.Sprintf("%d", lit)
+		}
+		strClause[len(clause)] = "0"
+		line := strings.Join(strClause, " ")
+		lines = append(lines, line)
+	}
+	return strings.Join(lines, "\n")
+}

vendor/github.com/crillab/gophersat/solver/solver.go

@@ -127,6 +127,25 @@ func New(problem *Problem) *Solver {
 	return s
 }
 
+// newVar is used to indicate a new variable must be added to the solver.
+// This can be used when new clauses are appended and these clauses contain vars that were unseen so far.
+// If the var already existed, nothing will happen.
+func (s *Solver) newVar(v Var) {
+	if cnfVar := int(v.Int()); cnfVar > s.nbVars {
+		// If the var already existed, do nothing
+		for i := s.nbVars; i < cnfVar; i++ {
+			s.model = append(s.model, 0)
+			s.activity = append(s.activity, 0.)
+			s.polarity = append(s.polarity, false)
+			s.reason = append(s.reason, nil)
+			s.trailBuf = append(s.trailBuf, 0)
+		}
+		s.varQueue = newQueue(s.activity)
+		s.addVarWatcherList(v)
+		s.nbVars = cnfVar
+	}
+}
+
 // sets initial activity for optimization variables, if any.
 func (s *Solver) initOptimActivity() {
 	for i, lit := range s.minLits {
@@ -682,6 +701,7 @@ func (s *Solver) AppendClause(clause *Clause) {
 	i := 0
 	for i < clause.Len() {
 		lit := clause.Get(i)
+		s.newVar(lit.Var())
 		switch s.litStatus(lit) {
 		case Sat:
 			w := clause.Weight(i)

vendor/github.com/crillab/gophersat/solver/types.go

@@ -71,6 +71,11 @@ func (v Var) Lit() Lit {
 	return Lit(v * 2)
 }
 
+// Int converts a Var to a CNF variable.
+func (v Var) Int() int32 {
+	return int32(v + 1)
+}
+
 // SignedLit returns the Lit associated to v, negated if 'signed', positive else.
 func (v Var) SignedLit(signed bool) Lit {
 	if signed {

vendor/github.com/crillab/gophersat/solver/watcher.go

@@ -39,6 +39,16 @@ func (s *Solver) initWatcherList(clauses []*Clause) {
 	}
 }
 
+// Should be called when new vars are added to the problem (see Solver.newVar)
+func (s *Solver) addVarWatcherList(v Var) {
+	cnfVar := int(v.Int())
+	for i := s.nbVars; i < cnfVar; i++ {
+		s.wl.wlistBin = append(s.wl.wlistBin, nil, nil)
+		s.wl.wlist = append(s.wl.wlist, nil, nil)
+		s.wl.wlistPb = append(s.wl.wlistPb, nil, nil)
+	}
+}
+
 // appendClause appends the clause without checking whether the clause is already satisfiable, unit, or unsatisfiable.
 // To perform those checks, call s.AppendClause.
 // clause is supposed to be a problem clause, not a learned one.

vendor/github.com/docker/cli/AUTHORS

@@ -8,6 +8,7 @@ Aaron.L.Xu <likexu@harmonycloud.cn>
 Abdur Rehman <abdur_rehman@mentor.com>
 Abhinandan Prativadi <abhi@docker.com>
 Abin Shahab <ashahab@altiscale.com>
+Abreto FU <public@abreto.email>
 Ace Tang <aceapril@126.com>
 Addam Hardy <addam.hardy@gmail.com>
 Adolfo Ochagavía <aochagavia92@gmail.com>
@@ -17,12 +18,15 @@ Adrien Folie <folie.adrien@gmail.com>
 Ahmet Alp Balkan <ahmetb@microsoft.com>
 Aidan Feldman <aidan.feldman@gmail.com>
 Aidan Hobson Sayers <aidanhs@cantab.net>
-AJ Bowen <aj@gandi.net>
-Akihiro Suda <suda.akihiro@lab.ntt.co.jp>
+AJ Bowen <aj@soulshake.net>
+Akhil Mohan <akhil.mohan@mayadata.io>
+Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
 Akim Demaille <akim.demaille@docker.com>
 Alan Thompson <cloojure@gmail.com>
 Albert Callarisa <shark234@gmail.com>
+Albin Kerouanton <albin@akerouanton.name>
 Aleksa Sarai <asarai@suse.de>
+Aleksander Piotrowski <apiotrowski312@gmail.com>
 Alessandro Boch <aboch@tetrationanalytics.com>
 Alex Mavrogiannis <alex.mavrogiannis@docker.com>
 Alex Mayer <amayer5125@gmail.com>
@@ -40,6 +44,7 @@ Amir Goldstein <amir73il@aquasec.com>
 Amit Krishnan <amit.krishnan@oracle.com>
 Amit Shukla <amit.shukla@docker.com>
 Amy Lindburg <amy.lindburg@docker.com>
+Anca Iordache <anca.iordache@docker.com>
 Anda Xu <anda.xu@docker.com>
 Andrea Luzzardi <aluzzardi@gmail.com>
 Andreas Köhler <andi5.py@gmx.net>
@@ -49,6 +54,7 @@ Andrew Macpherson <hopscotch23@gmail.com>
 Andrew McDonnell <bugs@andrewmcdonnell.net>
 Andrew Po <absourd.noise@gmail.com>
 Andrey Petrov <andrey.petrov@shazow.net>
+Andrii Berehuliak <berkusandrew@gmail.com>
 André Martins <aanm90@gmail.com>
 Andy Goldstein <agoldste@redhat.com>
 Andy Rothfusz <github@developersupport.net>
@@ -61,7 +67,9 @@ Antonis Kalipetis <akalipetis@gmail.com>
 Anusha Ragunathan <anusha.ragunathan@docker.com>
 Ao Li <la9249@163.com>
 Arash Deshmeh <adeshmeh@ca.ibm.com>
+Arko Dasgupta <arko.dasgupta@docker.com>
 Arnaud Porterie <arnaud.porterie@docker.com>
+Arthur Peka <arthur.peka@outlook.com>
 Ashwini Oruganti <ashwini.oruganti@gmail.com>
 Azat Khuyiyakhmetov <shadow_uz@mail.ru>
 Bardia Keyoumarsi <bkeyouma@ucsc.edu>
@@ -87,6 +95,7 @@ Brent Salisbury <brent.salisbury@docker.com>
 Bret Fisher <bret@bretfisher.com>
 Brian (bex) Exelbierd <bexelbie@redhat.com>
 Brian Goff <cpuguy83@gmail.com>
+Brian Wieder <brian@4wieders.com>
 Bryan Bess <squarejaw@bsbess.com>
 Bryan Boreham <bjboreham@gmail.com>
 Bryan Murphy <bmurphy1976@gmail.com>
@@ -95,6 +104,7 @@ Cameron Spear <cameronspear@gmail.com>
 Cao Weiwei <cao.weiwei30@zte.com.cn>
 Carlo Mion <mion00@gmail.com>
 Carlos Alexandro Becker <caarlos0@gmail.com>
+Carlos de Paula <me@carlosedp.com>
 Ce Gao <ce.gao@outlook.com>
 Cedric Davies <cedricda@microsoft.com>
 Cezar Sa Espinola <cezarsa@gmail.com>
@@ -128,26 +138,31 @@ Coenraad Loubser <coenraad@wish.org.za>
 Colin Hebert <hebert.colin@gmail.com>
 Collin Guarino <collin.guarino@gmail.com>
 Colm Hally <colmhally@gmail.com>
+Comical Derskeal <27731088+derskeal@users.noreply.github.com>
 Corey Farrell <git@cfware.com>
 Corey Quon <corey.quon@docker.com>
 Craig Wilhite <crwilhit@microsoft.com>
 Cristian Staretu <cristian.staretu@gmail.com>
 Daehyeok Mun <daehyeok@gmail.com>
 Dafydd Crosby <dtcrsby@gmail.com>
+Daisuke Ito <itodaisuke00@gmail.com>
 dalanlan <dalanlan925@gmail.com>
 Damien Nadé <github@livna.org>
 Dan Cotora <dan@bluevision.ro>
+Daniel Artine <daniel.artine@ufrj.br>
 Daniel Cassidy <mail@danielcassidy.me.uk>
 Daniel Dao <dqminh@cloudflare.com>
 Daniel Farrell <dfarrell@redhat.com>
 Daniel Gasienica <daniel@gasienica.ch>
 Daniel Goosen <daniel.goosen@surveysampling.com>
+Daniel Helfand <dhelfand@redhat.com>
 Daniel Hiltgen <daniel.hiltgen@docker.com>
 Daniel J Walsh <dwalsh@redhat.com>
 Daniel Nephin <dnephin@docker.com>
 Daniel Norberg <dano@spotify.com>
 Daniel Watkins <daniel@daniel-watkins.co.uk>
 Daniel Zhang <jmzwcn@gmail.com>
+Daniil Nikolenko <qoo2p5@gmail.com>
 Danny Berger <dpb587@gmail.com>
 Darren Shepherd <darren.s.shepherd@gmail.com>
 Darren Stahl <darst@microsoft.com>
@@ -180,13 +195,15 @@ Dima Stopel <dima@twistlock.com>
 Dimitry Andric <d.andric@activevideo.com>
 Ding Fei <dingfei@stars.org.cn>
 Diogo Monica <diogo@docker.com>
+Djordje Lukic <djordje.lukic@docker.com>
 Dmitry Gusev <dmitry.gusev@gmail.com>
 Dmitry Smirnov <onlyjob@member.fsf.org>
 Dmitry V. Krivenok <krivenok.dmitry@gmail.com>
+Dominik Braun <dominik.braun@nbsp.de>
 Don Kjer <don.kjer@gmail.com>
 Dong Chen <dongluo.chen@docker.com>
 Doug Davis <dug@us.ibm.com>
-Drew Erny <drew.erny@docker.com>
+Drew Erny <derny@mirantis.com>
 Ed Costello <epc@epcostello.com>
 Elango Sivanandam <elango.siva@docker.com>
 Eli Uriegas <eli.uriegas@docker.com>
@@ -249,6 +266,7 @@ Harald Albers <github@albersweb.de>
 Harold Cooper <hrldcpr@gmail.com>
 Harry Zhang <harryz@hyper.sh>
 He Simei <hesimei@zju.edu.cn>
+Hector S <hfsam88@gmail.com>
 Helen Xie <chenjg@harmonycloud.cn>
 Henning Sprang <henning.sprang@gmail.com>
 Henry N <henrynmail-github@yahoo.de>
@@ -256,6 +274,7 @@ Hernan Garcia <hernandanielg@gmail.com>
 Hongbin Lu <hongbin034@gmail.com>
 Hu Keping <hukeping@huawei.com>
 Huayi Zhang <irachex@gmail.com>
+Hugo Gabriel Eyherabide <hugogabriel.eyherabide@gmail.com>
 huqun <huqun@zju.edu.cn>
 Huu Nguyen <huu@prismskylabs.com>
 Hyzhou Zhy <hyzhou.zhy@alibaba-inc.com>
@@ -297,7 +316,7 @@ Jeremy Unruh <jeremybunruh@gmail.com>
 Jeremy Yallop <yallop@docker.com>
 Jeroen Franse <jeroenfranse@gmail.com>
 Jesse Adametz <jesseadametz@gmail.com>
-Jessica Frazelle <jessfraz@google.com>
+Jessica Frazelle <jess@oxide.computer>
 Jezeniel Zapanta <jpzapanta22@gmail.com>
 Jian Zhang <zhangjian.fnst@cn.fujitsu.com>
 Jie Luo <luo612@zju.edu.cn>
@@ -308,6 +327,7 @@ Jimmy Song <rootsongjc@gmail.com>
 jimmyxian <jimmyxian2004@yahoo.com.cn>
 Jintao Zhang <zhangjintao9020@gmail.com>
 Joao Fernandes <joao.fernandes@docker.com>
+Joe Abbey <joe.abbey@gmail.com>
 Joe Doliner <jdoliner@pachyderm.io>
 Joe Gordon <joe.gordon0@gmail.com>
 Joel Handwell <joelhandwell@gmail.com>
@@ -317,7 +337,7 @@ Johan Euphrosine <proppy@google.com>
 Johannes 'fish' Ziemke <github@freigeist.org>
 John Feminella <jxf@jxf.me>
 John Harris <john@johnharris.io>
-John Howard (VM) <John.Howard@microsoft.com>
+John Howard <github@lowenna.com>
 John Laswell <john.n.laswell@gmail.com>
 John Maguire <jmaguire@duosecurity.com>
 John Mulhausen <john@docker.com>
@@ -326,12 +346,15 @@ John Stephens <johnstep@docker.com>
 John Tims <john.k.tims@gmail.com>
 John V. Martinez <jvmatl@gmail.com>
 John Willis <john.willis@docker.com>
+Jon Johnson <jonjohnson@google.com>
+Jonatas Baldin <jonatas.baldin@gmail.com>
 Jonathan Boulle <jonathanboulle@gmail.com>
 Jonathan Lee <jonjohn1232009@gmail.com>
 Jonathan Lomas <jonathan@floatinglomas.ca>
 Jonathan McCrohan <jmccrohan@gmail.com>
 Jonh Wendell <jonh.wendell@redhat.com>
 Jordan Jennings <jjn2009@gmail.com>
+Jose J. Escobar <53836904+jescobar-docker@users.noreply.github.com>
 Joseph Kern <jkern@semafour.net>
 Josh Bodah <jb3689@yahoo.com>
 Josh Chorlton <jchorlton@gmail.com>
@@ -369,6 +392,7 @@ Kevin Kern <kaiwentan@harmonycloud.cn>
 Kevin Kirsche <Kev.Kirsche+GitHub@gmail.com>
 Kevin Meredith <kevin.m.meredith@gmail.com>
 Kevin Richardson <kevin@kevinrichardson.co>
+Kevin Woblick <mail@kovah.de>
 khaled souf <khaled.souf@gmail.com>
 Kim Eik <kim@heldig.org>
 Kir Kolyshkin <kolyshkin@gmail.com>
@@ -406,13 +430,16 @@ Luca Favatella <luca.favatella@erlang-solutions.com>
 Luca Marturana <lucamarturana@gmail.com>
 Lucas Chan <lucas-github@lucaschan.com>
 Luka Hartwig <mail@lukahartwig.de>
+Lukas Heeren <lukas-heeren@hotmail.com>
 Lukasz Zajaczkowski <Lukasz.Zajaczkowski@ts.fujitsu.com>
 Lydell Manganti <LydellManganti@users.noreply.github.com>
 Lénaïc Huard <lhuard@amadeus.com>
 Ma Shimiao <mashimiao.fnst@cn.fujitsu.com>
 Mabin <bin.ma@huawei.com>
+Maciej Kalisz <maciej.d.kalisz@gmail.com>
 Madhav Puri <madhav.puri@gmail.com>
 Madhu Venugopal <madhu@socketplane.io>
+Madhur Batra <madhurbatra097@gmail.com>
 Malte Janduda <mail@janduda.net>
 Manjunath A Kumatagi <mkumatag@in.ibm.com>
 Mansi Nahar <mmn4185@rit.edu>
@@ -422,6 +449,7 @@ Marco Mariani <marco.mariani@alterway.fr>
 Marco Vedovati <mvedovati@suse.com>
 Marcus Martins <marcus@docker.com>
 Marianna Tessel <mtesselh@gmail.com>
+Marius Ileana <marius.ileana@gmail.com>
 Marius Sturm <marius@graylog.com>
 Mark Oates <fl0yd@me.com>
 Marsh Macy <marsma@microsoft.com>
@@ -467,12 +495,14 @@ mikelinjie <294893458@qq.com>
 Mikhail Vasin <vasin@cloud-tv.ru>
 Milind Chawre <milindchawre@gmail.com>
 Mindaugas Rukas <momomg@gmail.com>
+Miroslav Gula <miroslav.gula@naytrolabs.com>
 Misty Stanley-Jones <misty@docker.com>
 Mohammad Banikazemi <mb@us.ibm.com>
 Mohammed Aaqib Ansari <maaquib@gmail.com>
 Mohini Anne Dsouza <mohini3917@gmail.com>
 Moorthy RS <rsmoorthy@gmail.com>
 Morgan Bauer <mbauer@us.ibm.com>
+Morten Hekkvang <morten.hekkvang@sbab.se>
 Moysés Borges <moysesb@gmail.com>
 Mrunal Patel <mrunalp@gmail.com>
 muicoder <muicoder@gmail.com>
@@ -503,9 +533,11 @@ Nishant Totla <nishanttotla@gmail.com>
 NIWA Hideyuki <niwa.niwa@nifty.ne.jp>
 Noah Treuhaft <noah.treuhaft@docker.com>
 O.S. Tezer <ostezer@gmail.com>
+Odin Ugedal <odin@ugedal.com>
 ohmystack <jun.jiang02@ele.me>
 Olle Jonsson <olle.jonsson@gmail.com>
 Olli Janatuinen <olli.janatuinen@gmail.com>
+Oscar Wieman <oscrx@icloud.com>
 Otto Kekäläinen <otto@seravo.fi>
 Ovidio Mallo <ovidio.mallo@gmail.com>
 Pascal Borreli <pascal@borreli.com>
@@ -515,6 +547,7 @@ Patrick Lang <plang@microsoft.com>
 Paul <paul9869@gmail.com>
 Paul Kehrer <paul.l.kehrer@gmail.com>
 Paul Lietar <paul@lietar.net>
+Paul Mulders <justinkb@gmail.com>
 Paul Weaver <pauweave@cisco.com>
 Pavel Pospisil <pospispa@gmail.com>
 Paweł Szczekutowicz <pszczekutowicz@gmail.com>
@@ -541,6 +574,7 @@ Qiang Huang <h.huangqiang@huawei.com>
 Qinglan Peng <qinglanpeng@zju.edu.cn>
 qudongfang <qudongfang@gmail.com>
 Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
+Rahul Zoldyck <rahulzoldyck@gmail.com>
 Ravi Shekhar Jethani <rsjethani@gmail.com>
 Ray Tsang <rayt@google.com>
 Reficul <xuzhenglun@gmail.com>
@@ -553,6 +587,7 @@ Richard Scothern <richard.scothern@gmail.com>
 Rick Wieman <git@rickw.nl>
 Ritesh H Shukla <sritesh@vmware.com>
 Riyaz Faizullabhoy <riyaz.faizullabhoy@docker.com>
+Rob Gulewich <rgulewich@netflix.com>
 Robert Wallis <smilingrob@gmail.com>
 Robin Naundorf <r.naundorf@fh-muenster.de>
 Robin Speekenbrink <robin@kingsquare.nl>
@@ -574,10 +609,14 @@ Sainath Grandhi <sainath.grandhi@intel.com>
 Sakeven Jiang <jc5930@sina.cn>
 Sally O'Malley <somalley@redhat.com>
 Sam Neirinck <sam@samneirinck.com>
+Samarth Shah <samashah@microsoft.com>
 Sambuddha Basu <sambuddhabasu1@gmail.com>
 Sami Tabet <salph.tabet@gmail.com>
+Samuel Cochran <sj26@sj26.com>
 Samuel Karp <skarp@amazon.com>
 Santhosh Manohar <santhosh@docker.com>
+Sargun Dhillon <sargun@netflix.com>
+Saswat Bhattacharya <sas.saswat@gmail.com>
 Scott Brenner <scott@scottbrenner.me>
 Scott Collier <emailscottcollier@gmail.com>
 Sean Christopherson <sean.j.christopherson@intel.com>
@@ -598,6 +637,7 @@ sidharthamani <sid@rancher.com>
 Silvin Lubecki <silvin.lubecki@docker.com>
 Simei He <hesimei@zju.edu.cn>
 Simon Ferquel <simon.ferquel@docker.com>
+Simon Heimberg <simon.heimberg@heimberg-ea.ch>
 Sindhu S <sindhus@live.in>
 Slava Semushin <semushin@redhat.com>
 Solomon Hykes <solomon@docker.com>
@@ -627,7 +667,10 @@ TAGOMORI Satoshi <tagomoris@gmail.com>
 taiji-tech <csuhqg@foxmail.com>
 Taylor Jones <monitorjbl@gmail.com>
 Tejaswini Duggaraju <naduggar@microsoft.com>
+Tengfei Wang <tfwang@alauda.io>
+Teppei Fukuda <knqyf263@gmail.com>
 Thatcher Peskens <thatcher@docker.com>
+Thibault Coupin <thibault.coupin@gmail.com>
 Thomas Gazagnaire <thomas@gazagnaire.org>
 Thomas Krzero <thomas.kovatchitch@gmail.com>
 Thomas Leonard <thomas.leonard@docker.com>
@@ -639,6 +682,7 @@ Tianyi Wang <capkurmagati@gmail.com>
 Tibor Vass <teabee89@gmail.com>
 Tim Dettrick <t.dettrick@uq.edu.au>
 Tim Hockin <thockin@google.com>
+Tim Sampson <tim@sampson.fi>
 Tim Smith <timbot@google.com>
 Tim Waugh <twaugh@redhat.com>
 Tim Wraight <tim.wraight@tangentlabs.co.uk>
@@ -663,9 +707,11 @@ Tristan Carel <tristan@cogniteev.com>
 Tycho Andersen <tycho@docker.com>
 Tycho Andersen <tycho@tycho.ws>
 uhayate <uhayate.gong@daocloud.io>
+Ulrich Bareth <ulrich.bareth@gmail.com>
 Ulysses Souza <ulysses.souza@docker.com>
 Umesh Yadav <umesh4257@gmail.com>
 Valentin Lorentz <progval+git@progval.net>
+Venkateswara Reddy Bukkasamudram <bukkasamudram@outlook.com>
 Veres Lajos <vlajos@gmail.com>
 Victor Vieux <victor.vieux@docker.com>
 Victoria Bialas <victoria.bialas@docker.com>
@@ -683,6 +729,7 @@ Wang Long <long.wanglong@huawei.com>
 Wang Ping <present.wp@icloud.com>
 Wang Xing <hzwangxing@corp.netease.com>
 Wang Yuexiao <wang.yuexiao@zte.com.cn>
+Wang Yumu <37442693@qq.com>
 Wataru Ishida <ishida.wataru@lab.ntt.co.jp>
 Wayne Song <wsong@docker.com>
 Wen Cheng Ma <wenchma@cn.ibm.com>
@@ -691,6 +738,7 @@ Wes Morgan <cap10morgan@gmail.com>
 Wewang Xiaorenfine <wang.xiaoren@zte.com.cn>
 William Henry <whenry@redhat.com>
 Xianglin Gao <xlgao@zju.edu.cn>
+Xiaodong Liu <liuxiaodong@loongson.cn>
 Xiaodong Zhang <a4012017@sina.com>
 Xiaoxi He <xxhe@alauda.io>
 Xinbo Weng <xihuanbo_0521@zju.edu.cn>

vendor/github.com/docker/cli/cli/config/config.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/docker/cli/cli/config/configfile"
 	"github.com/docker/cli/cli/config/credentials"
@@ -23,10 +24,15 @@ const (
 )
 
 var (
-	configDir = os.Getenv("DOCKER_CONFIG")
+	initConfigDir sync.Once
+	configDir     string
 )
 
-func init() {
+func setConfigDir() {
+	if configDir != "" {
+		return
+	}
+	configDir = os.Getenv("DOCKER_CONFIG")
 	if configDir == "" {
 		configDir = filepath.Join(homedir.Get(), configFileDir)
 	}
@@ -34,6 +40,7 @@ func init() {
 
 // Dir returns the directory the configuration file is stored in
 func Dir() string {
+	initConfigDir.Do(setConfigDir)
 	return configDir
 }
 
@@ -88,11 +95,7 @@ func Load(configDir string) (*configfile.ConfigFile, error) {
 	configFile := configfile.New(filename)
 
 	// Try happy path first - latest config file
-	if _, err := os.Stat(filename); err == nil {
-		file, err := os.Open(filename)
-		if err != nil {
-			return configFile, errors.Wrap(err, filename)
-		}
+	if file, err := os.Open(filename); err == nil {
 		defer file.Close()
 		err = configFile.LoadFromReader(file)
 		if err != nil {
@@ -106,22 +109,16 @@ func Load(configDir string) (*configfile.ConfigFile, error) {
 	}
 
 	// Can't find latest config file so check for the old one
-	homedir, err := os.UserHomeDir()
+	home, err := os.UserHomeDir()
 	if err != nil {
 		return configFile, errors.Wrap(err, oldConfigfile)
 	}
-	confFile := filepath.Join(homedir, oldConfigfile)
-	if _, err := os.Stat(confFile); err != nil {
-		return configFile, nil // missing file is not an error
-	}
-	file, err := os.Open(confFile)
-	if err != nil {
-		return configFile, errors.Wrap(err, filename)
-	}
-	defer file.Close()
-	err = configFile.LegacyLoadFromReader(file)
-	if err != nil {
-		return configFile, errors.Wrap(err, filename)
+	filename = filepath.Join(home, oldConfigfile)
+	if file, err := os.Open(filename); err == nil {
+		defer file.Close()
+		if err := configFile.LegacyLoadFromReader(file); err != nil {
+			return configFile, errors.Wrap(err, filename)
+		}
 	}
 	return configFile, nil
 }

vendor/github.com/docker/cli/cli/config/configfile/file.go

@@ -13,6 +13,7 @@ import (
 	"github.com/docker/cli/cli/config/credentials"
 	"github.com/docker/cli/cli/config/types"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -118,7 +119,7 @@ func (configFile *ConfigFile) LegacyLoadFromReader(configData io.Reader) error {
 // LoadFromReader reads the configuration data given and sets up the auth config
 // information with given directory and populates the receiver object
 func (configFile *ConfigFile) LoadFromReader(configData io.Reader) error {
-	if err := json.NewDecoder(configData).Decode(&configFile); err != nil {
+	if err := json.NewDecoder(configData).Decode(&configFile); err != nil && !errors.Is(err, io.EOF) {
 		return err
 	}
 	var err error
@@ -168,6 +169,13 @@ func (configFile *ConfigFile) SaveToWriter(writer io.Writer) error {
 	configFile.AuthConfigs = tmpAuthConfigs
 	defer func() { configFile.AuthConfigs = saveAuthConfigs }()
 
+	// User-Agent header is automatically set, and should not be stored in the configuration
+	for v := range configFile.HTTPHeaders {
+		if strings.EqualFold(v, "User-Agent") {
+			delete(configFile.HTTPHeaders, v)
+		}
+	}
+
 	data, err := json.MarshalIndent(configFile, "", "\t")
 	if err != nil {
 		return err
@@ -177,7 +185,7 @@ func (configFile *ConfigFile) SaveToWriter(writer io.Writer) error {
 }
 
 // Save encodes and writes out all the authorization information
-func (configFile *ConfigFile) Save() error {
+func (configFile *ConfigFile) Save() (retErr error) {
 	if configFile.Filename == "" {
 		return errors.Errorf("Can't save config with empty filename")
 	}
@@ -190,16 +198,33 @@ func (configFile *ConfigFile) Save() error {
 	if err != nil {
 		return err
 	}
+	defer func() {
+		temp.Close()
+		if retErr != nil {
+			if err := os.Remove(temp.Name()); err != nil {
+				logrus.WithError(err).WithField("file", temp.Name()).Debug("Error cleaning up temp file")
+			}
+		}
+	}()
+
 	err = configFile.SaveToWriter(temp)
-	temp.Close()
 	if err != nil {
-		os.Remove(temp.Name())
 		return err
 	}
-	// Try copying the current config file (if any) ownership and permissions
-	copyFilePermissions(configFile.Filename, temp.Name())
 
-	return os.Rename(temp.Name(), configFile.Filename)
+	if err := temp.Close(); err != nil {
+		return errors.Wrap(err, "error closing temp file")
+	}
+
+	// Handle situation where the configfile is a symlink
+	cfgFile := configFile.Filename
+	if f, err := os.Readlink(cfgFile); err == nil {
+		cfgFile = f
+	}
+
+	// Try copying the current config file (if any) ownership and permissions
+	copyFilePermissions(cfgFile, temp.Name())
+	return os.Rename(temp.Name(), cfgFile)
 }
 
 // ParseProxyConfig computes proxy configuration by retrieving the config for the provided host and