Tag 0.17.2

Speedup conflict check by caching in memory files
Tag 0.17.1
2025-09-04 00:34:41 +00:00 · 2021-07-30 13:33:29 +02:00 · 2021-07-30 12:40:55 +02:00 · 2021-07-29 15:07:32 +02:00 · 2021-07-29 11:03:02 +02:00 · 2021-07-29 11:03:02 +02:00
30 changed files with 934 additions and 99 deletions
--- a/.chglog/CHANGELOG.tpl.md
+++ b/.chglog/CHANGELOG.tpl.md
@@ -0,0 +1,49 @@
+{{ if .Versions -}}
+{{ if .Unreleased.CommitGroups -}}
+{{ range .Unreleased.CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ else }}
+{{ range .Unreleased.Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+
+{{ if .CommitGroups -}}
+{{ range .CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ else }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} (https://github.com/mudler/luet/commit/{{.Hash.Short}})
+{{ end }}
+{{ end -}}
+
+{{- if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+{{ range .Notes }}
+{{ .Body }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}
+
+{{- if .Versions }}
+[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD
+{{ range .Versions -}}
+{{ if .Tag.Previous -}}
+[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}
+{{ end -}}
+{{ end -}}
+{{ end -}}
--- a/.chglog/config.yml
+++ b/.chglog/config.yml
@@ -0,0 +1,27 @@
+style: github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://github.com/mudler/luet
+options:
+  commits:
+    # filters:
+    #   Type:
+    #     - feat
+    #     - fix
+    #     - perf
+    #     - refactor
+  commit_groups:
+     title_maps:
+       feat: Features
+       fix: Bug Fixes
+       perf: Performance Improvements
+       refactor: Code Refactoring
+       ci: Continous Integration
+  header:
+    pattern: "(.*)"
+    pattern_maps:
+      - Subject
+  notes:
+    keywords:
+      - BREAKING CHANGE
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,8 +1,46 @@
 on: push
+concurrency:
+  group: ci-${{ github.head_ref || github.ref }}-${{ github.repository }}
+  cancel-in-progress: true
 name: Build and release on push
 jobs:
-  release:
-    name: Test and Release
+  tests-integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+      - name: Login to quay with img
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Tests with Img backend
+        run: |
+            sudo -E env "PATH=$PATH" \
+                    env "LUET_BACKEND=img" \
+                    make test-integration
+      - name: Tests
+        run: |
+              sudo -E \
+                    env "PATH=$PATH" \
+                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                    make test-integration
+
+  tests-unit:
+    name: Unit tests
    runs-on: ubuntu-latest
    steps:
      - name: Install Go
@@ -22,13 +60,6 @@ jobs:
            sudo chmod a+x "/usr/bin/img"
      - name: Build test
        run: sudo -E env "PATH=$PATH" make multiarch-build-small
-      - name: Login to quay with img
-        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
-      - name: Tests with Img backend
-        run: |
-            sudo -E env "PATH=$PATH" \
-                    env "LUET_BACKEND=img" \
-                    make test-integration
      - name: Tests
        run: |
              sudo -E \
@@ -36,7 +67,28 @@ jobs:
                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
-                    make test-integration test-coverage
+                    make test-coverage
+
+  release:
+    name: Test and Release
+    needs: ["tests-integration","tests-unit"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
      - name: Build
        run: sudo -E env "PATH=$PATH" make multiarch-build-small && sudo chmod -R 777 release/
      - name: Release
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -2,7 +2,31 @@
 on: pull_request
 name: Build and Test
 jobs:
-  test:
+  tests-integration:
+    strategy:
+      matrix:
+        go-version: [1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+    - name: Tests with Img backend
+      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make test-integration
+  tests-unit:
    strategy:
      matrix:
        go-version: [1.14.x]
@@ -24,7 +48,5 @@ jobs:
            sudo chmod a+x "/usr/bin/img"
    - name: Build
      run: sudo -E env "PATH=$PATH" make multiarch-build-small
-    - name: Tests with Img backend
-      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
    - name: Tests
-      run: sudo -E env "PATH=$PATH" make test-integration test-coverage
+      run: sudo -E env "PATH=$PATH" make test-coverage
--- a/cmd/database/create.go
+++ b/cmd/database/create.go
@@ -74,6 +74,12 @@ For reference, inspect a "metadata.yaml" file generated while running "luet buil

 				files := art.Files

+				// Check if the package is already present
+				if p, err := systemDB.FindPackage(art.CompileSpec.GetPackage()); err == nil && p.GetName() != "" {
+					Fatal("Package", art.CompileSpec.GetPackage().HumanReadableString(),
+						" already present.")
+				}
+
 				if _, err := systemDB.CreatePackage(art.CompileSpec.GetPackage()); err != nil {
 					Fatal("Failed to create ", a, ": ", err.Error())
 				}
--- a/cmd/reinstall.go
+++ b/cmd/reinstall.go
@@ -0,0 +1,143 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+var reinstallCmd = &cobra.Command{
+	Use:   "reinstall <pkg1> <pkg2> <pkg3>",
+	Short: "reinstall a set of packages",
+	Long: `Reinstall a group of packages in the system:
+
+	$ luet reinstall -y system/busybox shells/bash system/coreutils ...
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("for", cmd.Flags().Lookup("for"))
+
+		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var toUninstall pkg.Packages
+		var toAdd pkg.Packages
+
+		stype := LuetCfg.Viper.GetString("solver.type")
+		discount := LuetCfg.Viper.GetFloat64("solver.discount")
+		rate := LuetCfg.Viper.GetFloat64("solver.rate")
+		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
+		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		yes := LuetCfg.Viper.GetBool("yes")
+		dbpath := LuetCfg.Viper.GetString("system.database_path")
+		rootfs := LuetCfg.Viper.GetString("system.rootfs")
+		engine := LuetCfg.Viper.GetString("system.database_engine")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		LuetCfg.System.DatabaseEngine = engine
+		LuetCfg.System.DatabasePath = dbpath
+		LuetCfg.System.Rootfs = rootfs
+
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toUninstall = append(toUninstall, pack)
+			toAdd = append(toAdd, pack)
+		}
+
+		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
+		repos := installer.Repositories{}
+		for _, repo := range LuetCfg.SystemRepositories {
+			if !repo.Enable {
+				continue
+			}
+			r := installer.NewSystemRepository(repo)
+			repos = append(repos, r)
+		}
+
+		LuetCfg.GetSolverOptions().Type = stype
+		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+		LuetCfg.GetSolverOptions().Discount = float32(discount)
+		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+		if concurrent {
+			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
+		} else {
+			LuetCfg.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		}
+
+		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      true,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
+		})
+		inst.Repositories(repos)
+
+		system := &installer.System{Database: LuetCfg.GetSystemDB(), Target: LuetCfg.GetSystem().Rootfs}
+		err := inst.Swap(toUninstall, toAdd, system)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+
+	reinstallCmd.Flags().String("system-dbpath", "", "System db path")
+	reinstallCmd.Flags().String("system-target", "", "System rootpath")
+	reinstallCmd.Flags().String("system-engine", "", "System DB engine")
+
+	reinstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
+	reinstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	reinstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	reinstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	reinstallCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	reinstallCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	reinstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	reinstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	reinstallCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(reinstallCmd)
+}
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -41,8 +41,14 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}

 const (
-	LuetCLIVersion = "0.16.7"
+	LuetCLIVersion = "0.17.2"
 	LuetEnvPrefix  = "LUET"
+	license        = `
+	Luet Copyright (C) 2019-2021 Ettore Di Giacinto
+	This program comes with ABSOLUTELY NO WARRANTY.
+	This is free software, and you are welcome to redistribute it
+	under certain conditions.
+	`
 )

 // Build time and commit information.
@@ -53,6 +59,47 @@ var (
 	BuildCommit string
 )

+func version() string {
+	return fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime)
+}
+
+var noBannerCommands = []string{"search", "exec", "tree", "database", "box", "cleanup"}
+
+func displayVersionBanner() {
+	display := true
+	if len(os.Args) > 1 {
+		for _, c := range noBannerCommands {
+			if os.Args[1] == c {
+				display = false
+			}
+		}
+	}
+	if display {
+		Info("Luet version", version())
+		Info(license)
+	}
+}
+
+func handleLock() {
+	if os.Getenv("LUET_NOLOCK") != "true" {
+		if len(os.Args) > 1 {
+			for _, lockedCmd := range LockedCommands {
+				if os.Args[1] == lockedCmd {
+					s := single.New("luet")
+					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
+						Fatal("another instance of the app is already running, exiting")
+					} else if err != nil {
+						// Another error occurred, might be worth handling it as well
+						Fatal("failed to acquire exclusive app lock:", err.Error())
+					}
+					defer s.TryUnlock()
+					break
+				}
+			}
+		}
+	}
+}
+
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
 	Use:   "luet",
@@ -80,8 +127,9 @@ To build a package, from a tree definition:
 	$ luet build --tree tree/path package
 	
 `,
-	Version: fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime),
+	Version: version(),
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+
 		err := LoadConfig(config.LuetCfg)
 		if err != nil {
 			Fatal("failed to load configuration:", err.Error())
@@ -155,23 +203,8 @@ func LoadConfig(c *config.LuetConfig) error {
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {

-	if os.Getenv("LUET_NOLOCK") != "true" {
-		if len(os.Args) > 1 {
-			for _, lockedCmd := range LockedCommands {
-				if os.Args[1] == lockedCmd {
-					s := single.New("luet")
-					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
-						Fatal("another instance of the app is already running, exiting")
-					} else if err != nil {
-						// Another error occurred, might be worth handling it as well
-						Fatal("failed to acquire exclusive app lock:", err.Error())
-					}
-					defer s.TryUnlock()
-					break
-				}
-			}
-		}
-	}
+	handleLock()
+	displayVersionBanner()

 	if err := RootCmd.Execute(); err != nil {
 		fmt.Println(err)
--- a/cmd/uninstall.go
+++ b/cmd/uninstall.go
@@ -67,10 +67,12 @@ var uninstallCmd = &cobra.Command{
 		dbpath := LuetCfg.Viper.GetString("system.database_path")
 		rootfs := LuetCfg.Viper.GetString("system.rootfs")
 		engine := LuetCfg.Viper.GetString("system.database_engine")
+		keepProtected, _ := cmd.Flags().GetBool("keep-protected-files")

 		LuetCfg.System.DatabaseEngine = engine
 		LuetCfg.System.DatabasePath = dbpath
 		LuetCfg.System.Rootfs = rootfs
+		LuetCfg.ConfigProtectSkip = !keepProtected

 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
@@ -123,6 +125,7 @@ func init() {
 	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
 	uninstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	uninstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	uninstallCmd.Flags().BoolP("keep-protected-files", "k", false, "Keep package protected files around")

 	RootCmd.AddCommand(uninstallCmd)
 }
--- a/go.mod
+++ b/go.mod
@@ -56,6 +56,7 @@ require (
 	go.uber.org/atomic v1.5.1 // indirect
 	go.uber.org/multierr v1.4.0
 	go.uber.org/zap v1.13.0
+	golang.org/x/mod v0.4.2
 	google.golang.org/grpc v1.29.1
 	gopkg.in/yaml.v2 v2.3.0
 	gotest.tools/v3 v3.0.2 // indirect
--- a/go.sum
+++ b/go.sum
@@ -1493,6 +1493,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
--- a/pkg/compiler/compiler.go
+++ b/pkg/compiler/compiler.go
@@ -774,22 +774,31 @@ func (cs *LuetCompiler) getSpecHash(pkgs pkg.DefaultPackages, salt string) (stri
 	return fmt.Sprintf("%x", h.Sum(nil)), nil
 }

-func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {

-	joinTag := ">:loop: join<"
-	if len(p.Join) != 0 {
-		Info(joinTag, "Generating a joint parent image from final packages")
+	joinTag := ">:loop: final images<"
+	var fromPackages pkg.DefaultPackages
+
+	if len(p.Join) > 0 {
+		fromPackages = p.Join
+		Warning(joinTag, `
+	Attention! the 'join' keyword is going to be deprecated in Luet >=0.18.x. 
+	Use 'requires_final_images: true' instead in the build.yaml file`)
+	} else if p.RequiresFinalImages {
+		Info(joinTag, "Generating a parent image from final packages")
+		fromPackages = p.Package.GetRequires()
 	} else {
+		// No source image to resolve
 		return nil
 	}

 	// First compute a hash and check if image is available. if it is, then directly consume that
-	overallFp, err := cs.getSpecHash(p.Join, "join")
+	overallFp, err := cs.getSpecHash(fromPackages, "join")
 	if err != nil {
 		return errors.Wrap(err, "could not generate image hash")
 	}

-	Info(joinTag, "Searching existing image with hash ", overallFp)
+	Info(joinTag, "Searching existing image with hash", overallFp)

 	image := cs.findImageHash(overallFp, p)
 	if image != "" {
@@ -811,12 +820,12 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 	}
 	defer os.RemoveAll(joinDir) // clean up

-	for _, p := range p.Join { //highly dependent on the order
+	for _, p := range fromPackages {
 		Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
 	}

 	current := 0
-	for _, c := range p.Join {
+	for _, c := range fromPackages {
 		current++
 		if c != nil && c.Name != "" && c.Version != "" {
 			joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(p.Join), c.HumanReadableString())
@@ -924,7 +933,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF

 	//Before multistage : join - same as multistage, but keep artifacts, join them, create a new one and generate a final image.
 	// When the image is there, use it as a source here, in place of GetImage().
-	if err := cs.resolveJoinImages(concurrency, keepPermissions, p); err != nil {
+	if err := cs.resolveFinalImages(concurrency, keepPermissions, p); err != nil {
 		return nil, errors.Wrap(err, "while resolving join images")
 	}

@@ -1028,7 +1037,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 				Assert:      assertion,
 			})

-			if err := cs.resolveJoinImages(concurrency, keepPermissions, compileSpec); err != nil {
+			if err := cs.resolveFinalImages(concurrency, keepPermissions, compileSpec); err != nil {
 				return nil, errors.Wrap(err, "while resolving join images")
 			}

--- a/pkg/compiler/imagehashtree_test.go
+++ b/pkg/compiler/imagehashtree_test.go
@@ -46,13 +46,13 @@ var _ = Describe("ImageHashTree", func() {

 			packageHash, err := hashtree.Query(compiler, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(packageHash.Target.Hash.BuildHash).To(Equal("4db24406e8db30a3310a1cf8c4d4e19597745e6d41b189dc51a73ac4a50cc9e6"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("4c867c9bab6c71d9420df75806e7a2f171dbc08487852ab4e2487bab04066cf2"))
-			Expect(packageHash.BuilderImageHash).To(Equal("builder-e6f9c5552a67c463215b0a9e4f7c7fc8"))
+			Expect(packageHash.Target.Hash.BuildHash).To(Equal("53993e5a02da4c21ad845371c872f5836fe45ff3a4e3c5ccb6296d0faee2b107"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("a786d3fd29d0b8bdfe5f304c8bf8be909d5c764cd7059c0e63294a8bff17f3ef"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-0cd3c0d07fc9be568377b3bf1b699e06"))
 		})
 	})

-	expectedPackageHash := "15811d83d0f8360318c54d91dcae3714f8efb39bf872572294834880f00ee7a8"
+	expectedPackageHash := "0d568ac04c4ca528a4e5b67978f2ad3a75d31d443ab20f9d7683b9608cc0d494"

 	Context("complex package definition", func() {
 		BeforeEach(func() {
@@ -73,23 +73,26 @@ var _ = Describe("ImageHashTree", func() {

 			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(expectedPackageHash))
 			Expect(packageHash.SourceHash).To(Equal(expectedPackageHash))
-			Expect(packageHash.BuilderImageHash).To(Equal("builder-3d739cab442aec15a6da238481df73c5"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-0f45c345f59103e84fc8bebbf02f2e2b"))

 			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("99c4ebb4bc4754985fcc28677badf90f525aa231b1db0fe75659f11b86dc20e8"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("2e8159583ac825acada763358290cfbea919a33873a926cab84f4f1a67ecf111"))
 			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
 			hash, err := packageHash.DependencyBuildImage(a)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(hash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+
+			Expect(hash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))

 			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
 			Expect(assertionA.Hash.PackageHash).To(Equal(expectedPackageHash))
 			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
 			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
-			Expect(assertionB.Hash.PackageHash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+
+			Expect(assertionB.Hash.PackageHash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))
 			hashB, err := packageHash.DependencyBuildImage(b)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(hashB).To(Equal("2668e418eab6861404834ad617713e39b8e58f68016a1fbfcc9384efdd037376"))
+
+			Expect(hashB).To(Equal("315075265aeb2e3c04c5428d31911f53c194ec9fa3db1421e8478f44b1e0def8"))
 		})
 	})

@@ -112,33 +115,35 @@ var _ = Describe("ImageHashTree", func() {
 			Expect(err).ToNot(HaveOccurred())

 			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).ToNot(Equal(expectedPackageHash))
-			sourceHash := "1d91b13d0246fa000085a1071c63397d21546300b17f69493f22315a64b717d4"
+			sourceHash := "66ec001fe72052d0e605ca96f607ae39ea4f8b53f0b7f762e377622d9c654de3"
 			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(sourceHash))
 			Expect(packageHash.SourceHash).To(Equal(sourceHash))
+
 			Expect(packageHash.SourceHash).ToNot(Equal(expectedPackageHash))

-			Expect(packageHash.BuilderImageHash).To(Equal("builder-03ee108a7c56b17ee568ace0800dd16d"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-ffc02fd8aaa916d0e17249885b3226b1"))

 			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("7677da23b2cc866c2d07aa4a58fbf703340f2f78c0efbb1ba9faf8979f250c87"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("b9c0286ebf6d28be831926ec7da9cb3cda6b489722d656aefc363ebd7173f937"))
 			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
 			hash, err := packageHash.DependencyBuildImage(a)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(hash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+
+			Expect(hash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))

 			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())

-			Expect(assertionA.Hash.PackageHash).To(Equal("1d91b13d0246fa000085a1071c63397d21546300b17f69493f22315a64b717d4"))
+			Expect(assertionA.Hash.PackageHash).To(Equal("66ec001fe72052d0e605ca96f607ae39ea4f8b53f0b7f762e377622d9c654de3"))
 			Expect(assertionA.Hash.PackageHash).ToNot(Equal(expectedPackageHash))

 			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
 			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())

-			Expect(assertionB.Hash.PackageHash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+			Expect(assertionB.Hash.PackageHash).To(Equal("74c6c833730e9ebd1d9fc669278152b5b58ec7ecb28fdae56658665616076adf"))
 			hashB, err := packageHash.DependencyBuildImage(b)
 			Expect(err).ToNot(HaveOccurred())

-			Expect(hashB).To(Equal("2668e418eab6861404834ad617713e39b8e58f68016a1fbfcc9384efdd037376"))
+			Expect(hashB).To(Equal("315075265aeb2e3c04c5428d31911f53c194ec9fa3db1421e8478f44b1e0def8"))
 		})
 	})

--- a/pkg/compiler/types/spec/spec.go
+++ b/pkg/compiler/types/spec/spec.go
@@ -26,6 +26,7 @@ import (
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/otiai10/copy"
+	dirhash "golang.org/x/mod/sumdb/dirhash"
 	yaml "gopkg.in/yaml.v2"
 )

@@ -115,39 +116,42 @@ type LuetCompilationSpec struct {

 	Copy []CopyField `json:"copy"`

-	Join pkg.DefaultPackages `json:"join"`
+	Join                pkg.DefaultPackages `json:"join"`
+	RequiresFinalImages bool                `json:"requires_final_images" yaml:"requires_final_images"`
 }

 // Signature is a portion of the spec that yields a signature for the hash
 type Signature struct {
-	Image      string
-	Steps      []string
-	PackageDir string
-	Prelude    []string
-	Seed       string
-	Env        []string
-	Retrieve   []string
-	Unpack     bool
-	Includes   []string
-	Excludes   []string
-	Copy       []CopyField
-	Join       pkg.DefaultPackages
+	Image               string
+	Steps               []string
+	PackageDir          string
+	Prelude             []string
+	Seed                string
+	Env                 []string
+	Retrieve            []string
+	Unpack              bool
+	Includes            []string
+	Excludes            []string
+	Copy                []CopyField
+	Join                pkg.DefaultPackages
+	RequiresFinalImages bool
 }

 func (cs *LuetCompilationSpec) signature() Signature {
 	return Signature{
-		Image:      cs.Image,
-		Steps:      cs.Steps,
-		PackageDir: cs.PackageDir,
-		Prelude:    cs.Prelude,
-		Seed:       cs.Seed,
-		Env:        cs.Env,
-		Retrieve:   cs.Retrieve,
-		Unpack:     cs.Unpack,
-		Includes:   cs.Includes,
-		Excludes:   cs.Excludes,
-		Copy:       cs.Copy,
-		Join:       cs.Join,
+		Image:               cs.Image,
+		Steps:               cs.Steps,
+		PackageDir:          cs.PackageDir,
+		Prelude:             cs.Prelude,
+		Seed:                cs.Seed,
+		Env:                 cs.Env,
+		Retrieve:            cs.Retrieve,
+		Unpack:              cs.Unpack,
+		Includes:            cs.Includes,
+		Excludes:            cs.Excludes,
+		Copy:                cs.Copy,
+		Join:                cs.Join,
+		RequiresFinalImages: cs.RequiresFinalImages,
 	}
 }

@@ -242,7 +246,7 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 }

 func (cs *LuetCompilationSpec) EmptyPackage() bool {
-	return len(cs.BuildSteps()) == 0 && len(cs.GetPreBuildSteps()) == 0 && !cs.UnpackedPackage()
+	return len(cs.BuildSteps()) == 0 && !cs.UnpackedPackage()
 }

 func (cs *LuetCompilationSpec) UnpackedPackage() bool {
@@ -266,7 +270,11 @@ func (cs *LuetCompilationSpec) Hash() (string, error) {
 	// build a signature, we want to be part of the hash only the fields that are relevant for build purposes
 	signature := cs.signature()
 	h, err := hashstructure.Hash(signature, hashstructure.FormatV2, nil)
-	return fmt.Sprint(h), err
+	sum, err := dirhash.HashDir(cs.Package.Path, "", dirhash.DefaultHash)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(h, sum), err
 }

 func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
--- a/pkg/helpers/slice.go
+++ b/pkg/helpers/slice.go
@@ -0,0 +1,10 @@
+package helpers
+
+func Contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}
--- a/pkg/installer/installer.go
+++ b/pkg/installer/installer.go
@@ -27,6 +27,7 @@ import (
 	"github.com/mudler/luet/pkg/bus"
 	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	"github.com/mudler/luet/pkg/helpers/match"
 	. "github.com/mudler/luet/pkg/logger"
@@ -755,12 +756,63 @@ func (l *LuetInstaller) getFinalizers(allRepos pkg.PackageDatabase, solution sol
 	return toFinalize, nil
 }

+func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, s *System) error {
+	Info("Checking for file conflicts..")
+	defer s.Clean() // Release memory
+
+	filesToInstall := []string{}
+	for _, m := range toInstall {
+		a, err := l.downloadPackage(m)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed downloading package")
+		}
+		files, err := a.FileList()
+		if err != nil && !l.Options.Force {
+			return errors.Wrapf(err, "Could not get filelist for %s", a.CompileSpec.Package.HumanReadableString())
+		}
+
+		for _, f := range files {
+			if helpers.Contains(filesToInstall, f) {
+				return fmt.Errorf(
+					"file conflict between packages to be installed",
+				)
+			}
+
+			exists, p, err := s.ExistsPackageFile(f)
+			if err != nil {
+				return errors.Wrap(err, "failed checking into system db")
+			}
+			if exists {
+				return fmt.Errorf(
+					"file conflict between '%s' and '%s' ( file: %s )",
+					p.HumanReadableString(),
+					m.Package.HumanReadableString(),
+					f,
+				)
+			}
+		}
+		filesToInstall = append(filesToInstall, files...)
+	}
+
+	return nil
+}
+
 func (l *LuetInstaller) install(o Option, syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
-	// Install packages into rootfs in parallel.
+
+	// Download packages in parallel first
 	if err := l.download(syncedRepos, toInstall); err != nil {
 		return errors.Wrap(err, "Downloading packages")
 	}

+	// Check file conflicts
+	if err := l.checkFileconflicts(toInstall, s); err != nil {
+		if !l.Options.Force {
+			return errors.Wrap(err, "file conflict found")
+		} else {
+			Warning("file conflict found", err.Error())
+		}
+	}
+
 	if l.Options.DownloadOnly {
 		return nil
 	}
--- a/pkg/installer/system.go
+++ b/pkg/installer/system.go
@@ -1,6 +1,8 @@
 package installer

 import (
+	"sync"
+
 	"github.com/hashicorp/go-multierror"
 	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
@@ -10,8 +12,10 @@ import (
 )

 type System struct {
-	Database pkg.PackageDatabase
-	Target   string
+	Database  pkg.PackageDatabase
+	Target    string
+	fileIndex map[string]pkg.Package
+	sync.Mutex
 }

 func (s *System) World() (pkg.Packages, error) {
@@ -52,3 +56,38 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
 	}
 	return errs
 }
+
+func (s *System) buildFileIndex() {
+	s.Lock()
+	defer s.Unlock()
+	// Check if cache is empty or if it got modified
+	if s.fileIndex == nil { //|| len(s.Database.GetPackages()) != len(s.fileIndex) {
+		s.fileIndex = make(map[string]pkg.Package)
+		for _, p := range s.Database.World() {
+			files, _ := s.Database.GetPackageFiles(p)
+			for _, f := range files {
+				s.fileIndex[f] = p
+			}
+		}
+	}
+}
+
+func (s *System) Clean() {
+	s.Lock()
+	defer s.Unlock()
+	s.fileIndex = nil
+}
+
+func (s *System) ExistsPackageFile(file string) (bool, pkg.Package, error) {
+	Debug("Checking if file ", file, "belongs to any package")
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	if p, exists := s.fileIndex[file]; exists {
+		Debug(file, "belongs already to", p.HumanReadableString())
+
+		return exists, p, nil
+	}
+	Debug(file, "doesn't belong to any package")
+	return false, nil, nil
+}
--- a/pkg/installer/system_test.go
+++ b/pkg/installer/system_test.go
@@ -0,0 +1,70 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer_test
+
+import (
+
+	//	. "github.com/mudler/luet/pkg/installer"
+
+	. "github.com/mudler/luet/pkg/installer"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("System", func() {
+	Context("Files", func() {
+		var s *System
+		var db pkg.PackageDatabase
+		var a, b *pkg.DefaultPackage
+
+		BeforeEach(func() {
+			db = pkg.NewInMemoryDatabase(false)
+			s = &System{Database: db}
+
+			a = &pkg.DefaultPackage{Name: "test", Version: "1", Category: "t"}
+
+			db.CreatePackage(a)
+			db.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.GetFingerPrint(), Files: []string{"foo", "f"}})
+
+			b = &pkg.DefaultPackage{Name: "test2", Version: "1", Category: "t"}
+
+			db.CreatePackage(b)
+			db.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: b.GetFingerPrint(), Files: []string{"barz", "f"}})
+		})
+
+		It("detects when are already shipped by other packages", func() {
+			r, p, err := s.ExistsPackageFile("foo")
+			Expect(r).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(Equal(a))
+			r, p, err = s.ExistsPackageFile("baz")
+			Expect(r).To(BeFalse())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(BeNil())
+
+			r, p, err = s.ExistsPackageFile("f")
+			Expect(r).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(Equal(b))
+			r, p, err = s.ExistsPackageFile("barz")
+			Expect(r).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p).To(Equal(b))
+		})
+	})
+})
--- a/tests/fixtures/fileconflicts/conflict1/build.yaml
+++ b/tests/fixtures/fileconflicts/conflict1/build.yaml
@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+  - mkdir /foo
+steps:
+  - echo conflict > /foo/test1
+package_dir: /foo
--- a/tests/fixtures/fileconflicts/conflict1/collection.yaml
+++ b/tests/fixtures/fileconflicts/conflict1/collection.yaml
@@ -0,0 +1,7 @@
+packages:
+- category: "test1"
+  name: "conflict"
+  version: "1.0"
+- category: "test2"
+  name: "conflict"
+  version: "1.0"
--- a/tests/integration/12_config_protect.sh
+++ b/tests/integration/12_config_protect.sh
@@ -92,7 +92,7 @@ testUnInstall() {
    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
    # TODO: we need remove it or not??
    assertTrue 'config protect created' "[ -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
-    assertTrue 'config protect maintains the protected files' "[ -e '$tmpdir/testrootfs/etc/a/conf' ]"
+    assertTrue 'config protect maintains the protected files' "[ ! -e '$tmpdir/testrootfs/etc/a/conf' ]"
 }


--- a/tests/integration/12_config_protect_samefile.sh
+++ b/tests/integration/12_config_protect_samefile.sh
@@ -87,7 +87,7 @@ testInstall() {


 testUnInstall() {
-    luet uninstall -y --full --config $tmpdir/luet.yaml test/a
+    luet uninstall -y --full --keep-protected-files --config $tmpdir/luet.yaml test/a
    installst=$?
    assertEquals 'uninstall test successfully' "$installst" "0"
    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
--- a/tests/integration/13_config_protect_annotation.sh
+++ b/tests/integration/13_config_protect_annotation.sh
@@ -86,7 +86,7 @@ testInstall() {


 testUnInstall() {
-    luet uninstall -y --full --config $tmpdir/luet.yaml test/a
+    luet uninstall -y --full --keep-protected-files --config $tmpdir/luet.yaml test/a
    installst=$?
    assertEquals 'uninstall test successfully' "$installst" "0"
    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
--- a/tests/integration/28_nobuildtreedocker.sh
+++ b/tests/integration/28_nobuildtreedocker.sh
@@ -51,9 +51,9 @@ testBuild() {
    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
    assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
    assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available remotely for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:ac34beb3fc2752bab54549db095dd6994d7531b88e1ff7f902d01ae80fdd030d"
-    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available remotely for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:d8b8881d50ae83646728bb28ad678fc14e4e003e4b0d0a66f8e6167c6116e024"
-    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2aaccd929ebe5683f95c70c6ec8d68f240ceea8633f3350b94904ad73da5fd47"
+    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available remotely for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:7a46e5c8893b574d9f984eaade75b0e68cc891bb6d01860ba91a62f9e4de2b62"
+    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available remotely for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:4277934ece4de17856b87294301cdfcdf3c6f956e682ff64697be67178b8a4b1"
+    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:132c097844b4e809efa8ae234452893b0214a1c860371e21564d89655ab10a56"
 }

 testRepo() {
--- a/tests/integration/29_nobuildtreeinheritvalues.sh
+++ b/tests/integration/29_nobuildtreeinheritvalues.sh
@@ -56,8 +56,8 @@ EOF
    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
    assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
    assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertNotContains 'Does NOT use the upstream cache without specifying it' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2aaccd929ebe5683f95c70c6ec8d68f240ceea8633f3350b94904ad73da5fd47"
-    assertContains 'Does generate a new hash as values changed build.yaml for test/interpolated-1.0+2 package image' "$build_output" "Building image luet/cache:c34b533cf76886c332fe9b2d3208f04265360a465a90c996cb4fcdaf959dee36 done"
+    assertNotContains 'Does NOT use the upstream cache without specifying it' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:132c097844b4e809efa8ae234452893b0214a1c860371e21564d89655ab10a56"
+    assertContains 'Does generate a new hash as values changed build.yaml for test/interpolated-1.0+2 package image' "$build_output" "Building image luet/cache:a7edf5c9ab219e406b64b2692ce08d56a8bcd212a43bb70df3737f13a008dfd4 done"
 }

 testRepo() {
--- a/tests/integration/29_nobuildtreeinheritvalues_noignored.sh
+++ b/tests/integration/29_nobuildtreeinheritvalues_noignored.sh
@@ -56,7 +56,7 @@ EOF
    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
    assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
    assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Downloading image quay.io/mocaccinoos/integration-test-cache:4db24406e8db30a3310a1cf8c4d4e19597745e6d41b189dc51a73ac4a50cc9e6"
+    assertContains 'Does use the upstream cache without specifying it (test/c-1.0)' "$build_output" "quay.io/mocaccinoos/integration-test-cache:a786d3fd29d0b8bdfe5f304c8bf8be909d5c764cd7059c0e63294a8bff17f3ef"
 }

 testRepo() {
--- a/tests/integration/32_fileconflicts.sh
+++ b/tests/integration/32_fileconflicts.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/fileconflicts" --destination $tmpdir/testbuild --compression gzip --all
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test1-1.0.package.tar.gz' ]"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test2-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/fileconflicts" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install -y --config $tmpdir/luet.yaml test1/conflict test2/conflict
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test failed' "$installst" "1"
+    #assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testReInstall() {
+    luet install -y --config $tmpdir/luet.yaml test1/conflict
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "0"
+    luet install -y --config $tmpdir/luet.yaml test2/conflict
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "1"
+    luet install -y --force --config $tmpdir/luet.yaml test2/conflict
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+
--- a/tests/integration/33_reinstall.sh
+++ b/tests/integration/33_reinstall.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/fileconflicts" --destination $tmpdir/testbuild --compression gzip --all
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test1-1.0.package.tar.gz' ]"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test2-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/fileconflicts" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testReInstall() {
+    luet install -y --config $tmpdir/luet.yaml test1/conflict
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "0"
+    luet reinstall -y --config $tmpdir/luet.yaml test1/conflict
+    installst=$?
+    assertEquals 'reinstall test succeeded' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+
--- a/vendor/golang.org/x/mod/semver/semver.go
+++ b/vendor/golang.org/x/mod/semver/semver.go
@@ -138,6 +138,9 @@ func Compare(v, w string) int {

 // Max canonicalizes its arguments and then returns the version string
 // that compares greater.
+//
+// Deprecated: use Compare instead. In most cases, returning a canonicalized
+// version is not expected or desired.
 func Max(v, w string) string {
 	v = Canonical(v)
 	w = Canonical(w)
--- a/vendor/golang.org/x/mod/sumdb/dirhash/hash.go
+++ b/vendor/golang.org/x/mod/sumdb/dirhash/hash.go
@@ -0,0 +1,132 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package dirhash defines hashes over directory trees.
+// These hashes are recorded in go.sum files and in the Go checksum database,
+// to allow verifying that a newly-downloaded module has the expected content.
+package dirhash
+
+import (
+	"archive/zip"
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// DefaultHash is the default hash function used in new go.sum entries.
+var DefaultHash Hash = Hash1
+
+// A Hash is a directory hash function.
+// It accepts a list of files along with a function that opens the content of each file.
+// It opens, reads, hashes, and closes each file and returns the overall directory hash.
+type Hash func(files []string, open func(string) (io.ReadCloser, error)) (string, error)
+
+// Hash1 is the "h1:" directory hash function, using SHA-256.
+//
+// Hash1 is "h1:" followed by the base64-encoded SHA-256 hash of a summary
+// prepared as if by the Unix command:
+//
+//	find . -type f | sort | sha256sum
+//
+// More precisely, the hashed summary contains a single line for each file in the list,
+// ordered by sort.Strings applied to the file names, where each line consists of
+// the hexadecimal SHA-256 hash of the file content,
+// two spaces (U+0020), the file name, and a newline (U+000A).
+//
+// File names with newlines (U+000A) are disallowed.
+func Hash1(files []string, open func(string) (io.ReadCloser, error)) (string, error) {
+	h := sha256.New()
+	files = append([]string(nil), files...)
+	sort.Strings(files)
+	for _, file := range files {
+		if strings.Contains(file, "\n") {
+			return "", errors.New("dirhash: filenames with newlines are not supported")
+		}
+		r, err := open(file)
+		if err != nil {
+			return "", err
+		}
+		hf := sha256.New()
+		_, err = io.Copy(hf, r)
+		r.Close()
+		if err != nil {
+			return "", err
+		}
+		fmt.Fprintf(h, "%x  %s\n", hf.Sum(nil), file)
+	}
+	return "h1:" + base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
+}
+
+// HashDir returns the hash of the local file system directory dir,
+// replacing the directory name itself with prefix in the file names
+// used in the hash function.
+func HashDir(dir, prefix string, hash Hash) (string, error) {
+	files, err := DirFiles(dir, prefix)
+	if err != nil {
+		return "", err
+	}
+	osOpen := func(name string) (io.ReadCloser, error) {
+		return os.Open(filepath.Join(dir, strings.TrimPrefix(name, prefix)))
+	}
+	return hash(files, osOpen)
+}
+
+// DirFiles returns the list of files in the tree rooted at dir,
+// replacing the directory name dir with prefix in each name.
+// The resulting names always use forward slashes.
+func DirFiles(dir, prefix string) ([]string, error) {
+	var files []string
+	dir = filepath.Clean(dir)
+	err := filepath.Walk(dir, func(file string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		rel := file
+		if dir != "." {
+			rel = file[len(dir)+1:]
+		}
+		f := filepath.Join(prefix, rel)
+		files = append(files, filepath.ToSlash(f))
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return files, nil
+}
+
+// HashZip returns the hash of the file content in the named zip file.
+// Only the file names and their contents are included in the hash:
+// the exact zip file format encoding, compression method,
+// per-file modification times, and other metadata are ignored.
+func HashZip(zipfile string, hash Hash) (string, error) {
+	z, err := zip.OpenReader(zipfile)
+	if err != nil {
+		return "", err
+	}
+	defer z.Close()
+	var files []string
+	zfiles := make(map[string]*zip.File)
+	for _, file := range z.File {
+		files = append(files, file.Name)
+		zfiles[file.Name] = file
+	}
+	zipOpen := func(name string) (io.ReadCloser, error) {
+		f := zfiles[name]
+		if f == nil {
+			return nil, fmt.Errorf("file %q not found in zip", name) // should never happen
+		}
+		return f.Open()
+	}
+	return hash(files, zipOpen)
+}
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -690,8 +690,10 @@ golang.org/x/crypto/ssh/terminal
 # golang.org/x/lint v0.0.0-20200302205851-738671d3881b
 golang.org/x/lint
 golang.org/x/lint/golint
-# golang.org/x/mod v0.3.0
+# golang.org/x/mod v0.4.2
+## explicit
 golang.org/x/mod/semver
+golang.org/x/mod/sumdb/dirhash
 # golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0
 golang.org/x/net/context
 golang.org/x/net/context/ctxhttp
Author	SHA1	Message	Date
Ettore Di Giacinto	db8bf2b85e	Tag 0.17.2	2021-07-30 13:33:29 +02:00
Ettore Di Giacinto	5eb586ddb0	Speedup conflict check by caching in memory files	2021-07-30 12:40:55 +02:00
Ettore Di Giacinto	f9747cdf87	Tag 0.17.1	2021-07-29 15:07:32 +02:00
Ettore Di Giacinto	becac7d853	Add tests	2021-07-29 11:03:02 +02:00
Ettore Di Giacinto	5aa5bffb48	Add reinstall command Fixes #51	2021-07-29 11:03:02 +02:00
Ettore Di Giacinto	9aa3159787	Check for file conflicts before install Fixes #88	2021-07-29 11:03:02 +02:00
Ettore Di Giacinto	9cb6e65bb6	Don't display banner on cleanup	2021-07-29 11:03:02 +02:00
Ettore Di Giacinto	92b243d7aa	Add git-chglog template and config	2021-07-12 10:26:22 +02:00
Ettore Di Giacinto	29ec19a8a1	Tag 0.17.0	2021-07-11 11:55:35 +02:00
Ettore Di Giacinto	440e07c418	Display version banner only on specific commands Avoid cumbersome text on areas that can be parsed by machines Signed-off-by: Ettore Di Giacinto <mudler@sabayon.org>	2021-07-11 11:00:59 +02:00
Ettore Di Giacinto	acf74f5896	config_protect: don't protect files during uninstall by default Otherwise during uninstall we would retain the files which are protected. We introduced a specific flag for it to pass during uninstall, but for now we choose semplicity and expected default first. Signed-off-by: Ettore Di Giacinto <mudler@sabayon.org>	2021-07-11 10:40:36 +02:00
Ettore Di Giacinto	1ee1894ffa	ci: split integration/unit tests	2021-07-09 17:15:17 +02:00
Ettore Di Giacinto	c8573f9535	Adapt hash tests Signed-off-by: Ettore Di Giacinto <mudler@sabayon.org>	2021-07-09 17:15:17 +02:00
Ettore Di Giacinto	76b70ebeb4	Add dirhash to package signatures While this breaks current hashing, it ties also the spec content to the hash, in this way if we change something in the spec folder, it breaks the hashing for the package. Signed-off-by: Ettore Di Giacinto <mudler@sabayon.org>	2021-07-09 12:37:38 +02:00
Ettore Di Giacinto	2efb17a06c	Add specific field to resolve final images This commit starts deprecation of `join` keyword in favor of `requires_final_images` as boolean in the compilation spec. The change is driven by two reasons: syntax and guarantee unique hashes. - the hashtree when computing a hash it analizes the requires field of each spec, ignoring the join field - the join field doesn't add much value. Having it separate suggests that a spec can contain both `requires` and `join`, but that's not actually true. We just act differently on the same list. Signed-off-by: Ettore Di Giacinto <mudler@sabayon.org>	2021-07-09 12:37:38 +02:00
Ettore Di Giacinto	64ab3711ca	Display version and license on each run Signed-off-by: Ettore Di Giacinto <mudler@sabayon.org>	2021-07-09 12:25:55 +02:00
Ettore Di Giacinto	eb5d7ba35b	Merge branch 'master' into develop	2021-07-09 12:25:04 +02:00
Daniele Rondina	b6b91cfd7a	compiler: Avoid generation of delta if there are only prelude steps (#232 )	2021-07-09 12:23:32 +02:00
Daniele Rondina	4d8a9a544b	cmd/database/create: Avoid creation of the same package multiple times (#222 )	2021-06-25 13:09:32 +02:00