🆕 Tag 0.21.2

Fixes #274
Fixes #212

.github/workflows/pr.yml

@@ -2,7 +2,7 @@
 on: pull_request
 name: Build and Test
 jobs:
-  tests-integration:
+  tests-integration-img:
     strategy:
       matrix:
         go-version: [1.16.x]
@@ -24,6 +24,25 @@ jobs:
             sudo chmod a+x "/usr/bin/img"
     - name: Tests with Img backend
       run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
+
+  tests-integration:
+    strategy:
+      matrix:
+        go-version: [1.16.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
     - name: Tests
       run: sudo -E env "PATH=$PATH" make test-integration
   tests-unit:

.github/workflows/push.yml

@@ -4,8 +4,8 @@ concurrency:
 
 name: Build on push
 jobs:
-  tests-integration:
-    name: Integration tests
+  tests-integration-img:
+    name: Integration tests with img
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
@@ -14,8 +14,6 @@ jobs:
           go-version: 1.16.x
       - name: Checkout code
         uses: actions/checkout@v2
-      - name: setup-docker
-        uses: docker-practice/actions-setup-docker@0.0.1
       - name: Login to quay
         run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
       - name: Install deps
@@ -30,6 +28,21 @@ jobs:
             sudo -E env "PATH=$PATH" \
                     env "LUET_BACKEND=img" \
                     make test-integration
+  tests-integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
       - name: Tests
         run: |
               sudo -E \
@@ -49,8 +62,6 @@ jobs:
           go-version: 1.16.x
       - name: Checkout code
         uses: actions/checkout@v2
-      - name: setup-docker
-        uses: docker-practice/actions-setup-docker@0.0.1
       - name: Login to quay
         run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
       - name: Install deps

.github/workflows/release.yml

@@ -7,7 +7,7 @@ concurrency:
 
 name: Test and Release on tag
 jobs:
-  tests-integration:
+  tests-integration-img:
     name: Integration tests
     runs-on: ubuntu-latest
     steps:
@@ -33,6 +33,23 @@ jobs:
           sudo -E env "PATH=$PATH" \
                   env "LUET_BACKEND=img" \
                   make test-integration
+  tests-integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+          sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
       - name: Tests
         run: |
           sudo -E \
@@ -75,7 +92,7 @@ jobs:
   release:
     name: Release
     runs-on: ubuntu-latest
-    needs: [ "tests-integration","tests-unit" ]
+    needs: [ "tests-integration-img", "tests-integration","tests-unit" ]
     steps:
       - name: Install Go
         uses: actions/setup-go@v2

CONTRIBUTING.md

@@ -17,7 +17,7 @@ Join us in [slack](https://luet.slack.com/join/shared_invite/enQtOTQxMjcyNDQ0MDU
 ## All Code Changes Happen Through Pull Requests
 Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests:
 
-1. Fork the repo you want to contribute to and create your branch from `develop`.
+1. Fork the repo you want to contribute to and create your branch from `master`.
 2. If you've added code that should be tested, add tests.
 3. If you've changed APIs, update the [documentation](https://github.com/Luet-lab/docs).
 4. Ensure the test suite passes.

README.md

@@ -22,7 +22,7 @@ It is written entirely in Golang and where used as package manager, it can run i
 ## In a glance
 
 - Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.
-- It builds, installs, uninstalls and perform upgrades on machines
+- It builds from containers, but installs, uninstalls and perform upgrades on machines
 - Installer doesn't depend on anything ( 0 dep installer !), statically built
 - You can install it aside also with your current distro package manager, and start building and distributing your packages
 - [Support for packages as "layers"](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/#building-strategies)
@@ -30,6 +30,7 @@ It is written entirely in Golang and where used as package manager, it can run i
 - Support for [collections](https://luet-lab.github.io/docs/docs/concepts/packages/collections/) and [templated package definitions](https://luet-lab.github.io/docs/docs/concepts/packages/templates/)
 - [Can be extended with Plugins and Extensions](https://luet-lab.github.io/docs/docs/concepts/plugins-and-extensions/)
 - [Can build packages in Kubernetes (experimental)](https://github.com/mudler/luet-k8s)
+- Uses containerd/go-containerregistry to manipulate images - works also daemonless with the img backend
 
 ## Install
 

cmd/build.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/mudler/luet/pkg/compiler/types/compression"
 	"github.com/mudler/luet/pkg/compiler/types/options"
+	fileHelpers "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
@@ -109,6 +110,9 @@ Build packages specifying multiple definition trees:
 		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
 		full, _ := cmd.Flags().GetBool("full")
 		rebuild, _ := cmd.Flags().GetBool("rebuild")
+		pushFinalImages, _ := cmd.Flags().GetBool("push-final-images")
+		pushFinalImagesRepository, _ := cmd.Flags().GetString("push-final-images-repository")
+		pushFinalImagesForce, _ := cmd.Flags().GetBool("push-final-images-force")
 
 		var results Results
 		backendArgs := viper.GetStringSlice("backend-args")
@@ -144,6 +148,11 @@ Build packages specifying multiple definition trees:
 
 		util.DefaultContext.Info("Building in", dst)
 
+		if !fileHelpers.Exists(dst) {
+			os.MkdirAll(dst, 0600)
+			util.DefaultContext.Debug("Creating destination folder", dst)
+		}
+
 		opts := util.SetSolverConfig(util.DefaultContext)
 		pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
@@ -153,8 +162,7 @@ Build packages specifying multiple definition trees:
 
 		opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
 
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
-			options.NoDeps(nodeps),
+		compileropts := []options.Option{options.NoDeps(nodeps),
 			options.WithBackendType(backendType),
 			options.PushImages(push),
 			options.WithBuildValues(values),
@@ -171,8 +179,21 @@ Build packages specifying multiple definition trees:
 			options.WithContext(util.DefaultContext),
 			options.BackendArgs(backendArgs),
 			options.Concurrency(concurrency),
-			options.WithCompressionType(compression.Implementation(compressionType)),
-		)
+			options.WithCompressionType(compression.Implementation(compressionType))}
+
+		if pushFinalImages {
+			compileropts = append(compileropts, options.EnablePushFinalImages)
+			if pushFinalImagesForce {
+				compileropts = append(compileropts, options.ForcePushFinalImages)
+			}
+			if pushFinalImagesRepository != "" {
+				compileropts = append(compileropts, options.WithFinalRepository(pushFinalImagesRepository))
+			} else if imageRepository != "" {
+				compileropts = append(compileropts, options.WithFinalRepository(imageRepository))
+			}
+		}
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), compileropts...)
 
 		if full {
 			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
@@ -296,6 +317,11 @@ func init() {
 	buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
+
+	buildCmd.Flags().Bool("push-final-images", false, "Push final images while building")
+	buildCmd.Flags().Bool("push-final-images-force", false, "Override existing images")
+	buildCmd.Flags().String("push-final-images-repository", "", "Repository where to push final images to")
+
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
 	buildCmd.Flags().StringSlice("values", []string{}, "Build values file to interpolate with each package")
 	buildCmd.Flags().StringSliceP("backend-args", "a", []string{}, "Backend args")

cmd/cleanup.go

@@ -47,9 +47,7 @@ var cleanupCmd = &cobra.Command{
 
 			for _, file := range files {
 
-				if util.DefaultContext.Config.GetGeneral().Debug {
-					util.DefaultContext.Info("Removing ", file.Name())
-				}
+				util.DefaultContext.Debug("Removing ", file.Name())
 
 				err := os.RemoveAll(
 					filepath.Join(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))

cmd/create-repo.go

@@ -100,6 +100,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		helpers.CheckErr(err)
 		force := viper.GetBool("force-push")
 		imagePush := viper.GetBool("push-images")
+		snapshotID, _ := cmd.Flags().GetString("snapshot-id")
 
 		opts := []installer.RepositoryOption{
 			installer.WithSource(viper.GetString("packages")),
@@ -163,7 +164,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		if metaName != "" {
 			metaFile.SetFileName(metaName)
 		}
-
+		repo.SetSnapshotID(snapshotID)
 		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
@@ -197,6 +198,7 @@ func init() {
 	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
 	createrepoCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
+	createrepoCmd.Flags().String("snapshot-id", "", "Unique ID to use when creating repository snapshots")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

cmd/oscheck.go

@@ -0,0 +1,140 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	installer "github.com/mudler/luet/pkg/installer"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/mudler/luet/cmd/util"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var osCheckCmd = &cobra.Command{
+	Use:   "oscheck",
+	Short: "Checks packages integrity",
+	Long: `List packages that are installed in the system which files are missing in the system.
+
+	$ luet oscheck
+	
+To reinstall packages in the list:
+	
+	$ luet oscheck --reinstall
+`,
+	Aliases: []string{"i"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+
+		force := viper.GetBool("force")
+		onlydeps := viper.GetBool("onlydeps")
+		yes := viper.GetBool("yes")
+
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+		util.SetSystemConfig(util.DefaultContext)
+
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		packs := system.OSCheck()
+		if !util.DefaultContext.Config.General.Quiet {
+			if len(packs) == 0 {
+				util.DefaultContext.Success("All good!")
+				os.Exit(0)
+			} else {
+				util.DefaultContext.Info("Following packages are missing files or are incomplete:")
+				for _, p := range packs {
+					util.DefaultContext.Info(p.HumanReadableString())
+				}
+			}
+		} else {
+			var s []string
+			for _, p := range packs {
+				s = append(s, p.HumanReadableString())
+			}
+			fmt.Println(strings.Join(s, " "))
+		}
+
+		reinstall, _ := cmd.Flags().GetBool("reinstall")
+		if reinstall {
+
+			// Strip version for reinstall
+			toInstall := pkg.Packages{}
+			for _, p := range packs {
+				new := p.Clone()
+				new.SetVersion(">=0")
+				toInstall = append(toInstall, new)
+			}
+
+			util.SetSolverConfig(util.DefaultContext)
+
+			util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+			util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+
+			// Load config protect configs
+			util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+				Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+				SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+				NoDeps:                      true,
+				Force:                       force,
+				OnlyDeps:                    onlydeps,
+				PreserveSystemEssentialData: true,
+				Ask:                         !yes,
+				DownloadOnly:                downloadOnly,
+				Context:                     util.DefaultContext,
+				PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			})
+
+			err := inst.Swap(packs, toInstall, system)
+			if err != nil {
+				util.DefaultContext.Fatal("Error: " + err.Error())
+			}
+		}
+	},
+}
+
+func init() {
+	osCheckCmd.Flags().String("system-dbpath", "", "System db path")
+	osCheckCmd.Flags().String("system-target", "", "System rootpath")
+	osCheckCmd.Flags().String("system-engine", "", "System DB engine")
+	osCheckCmd.Flags().Bool("reinstall", false, "reinstall")
+
+	osCheckCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	osCheckCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	osCheckCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	osCheckCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	osCheckCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	osCheckCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	osCheckCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	osCheckCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	osCheckCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(osCheckCmd)
+}

cmd/reinstall.go

@@ -52,18 +52,10 @@ var reinstallCmd = &cobra.Command{
 		yes := viper.GetBool("yes")
 
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+		installed, _ := cmd.Flags().GetBool("installed")
 
 		util.SetSystemConfig(util.DefaultContext)
 
-		for _, a := range args {
-			pack, err := helpers.ParsePackageStr(a)
-			if err != nil {
-				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
-			}
-			toUninstall = append(toUninstall, pack)
-			toAdd = append(toAdd, pack)
-		}
-
 		util.SetSolverConfig(util.DefaultContext)
 
 		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
@@ -87,6 +79,25 @@ var reinstallCmd = &cobra.Command{
 		})
 
 		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+
+		if installed {
+			for _, p := range system.Database.World() {
+				toUninstall = append(toUninstall, p)
+				c := p.Clone()
+				c.SetVersion(">=0")
+				toAdd = append(toAdd, c)
+			}
+		} else {
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
+				}
+				toUninstall = append(toUninstall, pack)
+				toAdd = append(toAdd, pack)
+			}
+		}
+
 		err := inst.Swap(toUninstall, toAdd, system)
 		if err != nil {
 			util.DefaultContext.Fatal("Error: " + err.Error())
@@ -107,6 +118,7 @@ func init() {
 	reinstallCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
 	reinstallCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
 	reinstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	reinstallCmd.Flags().Bool("installed", false, "Reinstall installed packages")
 	reinstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 	reinstallCmd.Flags().Bool("download-only", false, "Download only")
 

cmd/repo/update.go

@@ -1,5 +1,6 @@
 // Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
 //                  Daniele Rondina <geaaru@sabayonlinux.org>
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -24,7 +25,7 @@ import (
 )
 
 func NewRepoUpdateCommand() *cobra.Command {
-	var ans = &cobra.Command{
+	var repoUpdate = &cobra.Command{
 		Use:   "update [repo1] [repo2] [OPTIONS]",
 		Short: "Update a specific cached repository or all cached repositories.",
 		Example: `
@@ -72,8 +73,8 @@ $> luet repo update repo1 repo2
 		},
 	}
 
-	ans.Flags().BoolP("ignore-errors", "i", false, "Ignore errors on sync repositories.")
-	ans.Flags().BoolP("force", "f", false, "Force resync.")
+	repoUpdate.Flags().BoolP("ignore-errors", "i", false, "Ignore errors on sync repositories.")
+	repoUpdate.Flags().BoolP("force", "f", true, "Force resync.")
 
-	return ans
+	return repoUpdate
 }

cmd/root.go

@@ -20,7 +20,7 @@ import (
 	"os"
 
 	"github.com/mudler/luet/cmd/util"
-	bus "github.com/mudler/luet/pkg/bus"
+	bus "github.com/mudler/luet/pkg/api/core/bus"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -30,7 +30,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.19.1"
+	LuetCLIVersion = "0.21.2"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -99,11 +99,11 @@ To build a package, from a tree definition:
 
 		plugin := viper.GetStringSlice("plugin")
 
-		bus.Manager.Initialize(plugin...)
+		bus.Manager.Initialize(util.DefaultContext, plugin...)
 		if len(bus.Manager.Plugins) != 0 {
 			util.DefaultContext.Info(":lollipop:Enabled plugins:")
 			for _, p := range bus.Manager.Plugins {
-				util.DefaultContext.Info("\t:arrow_right:", p.Name)
+				util.DefaultContext.Info(fmt.Sprintf("\t:arrow_right: %s (at %s)", p.Name, p.Executable))
 			}
 		}
 	},

cmd/search.go

@@ -334,7 +334,7 @@ Search can also return results in the terminal in different ways: as terminal ou
 
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
-			util.DefaultContext.Config.GetLogging().SetLogLevel("error")
+			util.DefaultContext.Config.GetLogging().SetLogLevel(types.FatalLevel)
 		}
 
 		l := &util.ListWriter{}
@@ -353,12 +353,6 @@ Search can also return results in the terminal in different ways: as terminal ou
 			results = searchLocally(args[0], l, t, searchWithLabel, searchWithLabelMatch, revdeps, hidden)
 		}
 
-		if tableMode {
-			t.Render()
-		} else {
-			l.Render()
-		}
-
 		y, err := yaml.Marshal(results)
 		if err != nil {
 			fmt.Printf("err: %v\n", err)
@@ -374,8 +368,17 @@ Search can also return results in the terminal in different ways: as terminal ou
 				return
 			}
 			fmt.Println(string(j2))
+		default:
+			if tableMode {
+				t.Render()
+			} else if util.DefaultContext.Config.General.Quiet {
+				for _, tt := range results.Packages {
+					fmt.Printf("%s/%s-%s\n", tt.Category, tt.Name, tt.Version)
+				}
+			} else {
+				l.Render()
+			}
 		}
-
 	},
 }
 

cmd/tree/images.go

@@ -102,7 +102,12 @@ func NewTreeImageCommand() *cobra.Command {
 			}
 
 			ht := compiler.NewHashTree(reciper.GetDatabase())
-			hashtree, err := ht.Query(luetCompiler, spec)
+
+			copy, err := compiler.CompilerFinalImages(luetCompiler)
+			if err != nil {
+				util.DefaultContext.Fatal("Error: " + err.Error())
+			}
+			hashtree, err := ht.Query(copy, spec)
 			if err != nil {
 				util.DefaultContext.Fatal("Error: " + err.Error())
 			}

cmd/upgrade.go

@@ -43,6 +43,8 @@ var upgradeCmd = &cobra.Command{
 		universe, _ := cmd.Flags().GetBool("universe")
 		clean, _ := cmd.Flags().GetBool("clean")
 		sync, _ := cmd.Flags().GetBool("sync")
+		osCheck, _ := cmd.Flags().GetBool("oscheck")
+
 		yes := viper.GetBool("yes")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
@@ -67,6 +69,7 @@ var upgradeCmd = &cobra.Command{
 			UpgradeNewRevisions:         sync,
 			PreserveSystemEssentialData: true,
 			Ask:                         !yes,
+			AutoOSCheck:                 osCheck,
 			DownloadOnly:                downloadOnly,
 			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
 			Context:                     util.DefaultContext,
@@ -97,6 +100,7 @@ func init() {
 	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	upgradeCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 	upgradeCmd.Flags().Bool("download-only", false, "Download only")
+	upgradeCmd.Flags().Bool("oscheck", false, "Perform automatically oschecks after upgrades")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

cmd/util.go

@@ -19,9 +19,14 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/go-units"
+	"github.com/mudler/luet/pkg/api/core/image"
+	luettypes "github.com/mudler/luet/pkg/api/core/types"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	"github.com/pkg/errors"
 
 	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/helpers/docker"
@@ -29,6 +34,55 @@ import (
 	"github.com/spf13/cobra"
 )
 
+func pack(ctx *luettypes.Context, p, dst, imageName, arch, OS string) error {
+
+	tempimage, err := ctx.Config.GetSystem().TempFile("tempimage")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for "+p)
+	}
+	defer os.RemoveAll(tempimage.Name()) // clean up
+
+	if err := image.CreateTar(p, tempimage.Name(), imageName, arch, OS); err != nil {
+		return errors.Wrap(err, "could not create image from tar")
+	}
+
+	return fileHelper.CopyFile(tempimage.Name(), dst)
+}
+
+func NewPackCommand() *cobra.Command {
+
+	c := &cobra.Command{
+		Use:   "pack image src.tar dst.tar",
+		Short: "Pack a standard tar archive as a container image",
+		Long: `Pack creates a tar which can be loaded as an image from a standard flat tar archive, for e.g. with docker load. 
+It doesn't need the docker daemon to run, and allows to override default os/arch:
+		
+	luet util pack --os arm64 image:tag src.tar dst.tar
+`,
+		Args: cobra.MinimumNArgs(3),
+		Run: func(cmd *cobra.Command, args []string) {
+
+			image := args[0]
+			src := args[1]
+			dst := args[2]
+
+			arch, _ := cmd.Flags().GetString("arch")
+			os, _ := cmd.Flags().GetString("os")
+
+			err := pack(util.DefaultContext, src, dst, image, arch, os)
+			if err != nil {
+				util.DefaultContext.Fatal(err.Error())
+			}
+			util.DefaultContext.Info("Image packed as", image)
+		},
+	}
+
+	c.Flags().String("arch", runtime.GOARCH, "Image architecture")
+	c.Flags().String("os", runtime.GOOS, "Image OS")
+
+	return c
+}
+
 func NewUnpackCommand() *cobra.Command {
 
 	c := &cobra.Command{
@@ -62,11 +116,6 @@ func NewUnpackCommand() *cobra.Command {
 			identity, _ := cmd.Flags().GetString("auth-identity-token")
 			registryToken, _ := cmd.Flags().GetString("auth-registry-token")
 
-			temp, err := util.DefaultContext.Config.GetSystem().TempDir("contentstore")
-			if err != nil {
-				util.DefaultContext.Fatal("Cannot create a tempdir", err.Error())
-			}
-
 			util.DefaultContext.Info("Downloading", image, "to", destination)
 			auth := &types.AuthConfig{
 				Username:      user,
@@ -77,7 +126,7 @@ func NewUnpackCommand() *cobra.Command {
 				RegistryToken: registryToken,
 			}
 
-			info, err := docker.DownloadAndExtractDockerImage(temp, image, destination, auth, verify)
+			info, err := docker.DownloadAndExtractDockerImage(util.DefaultContext, image, destination, auth, verify)
 			if err != nil {
 				util.DefaultContext.Error(err.Error())
 				os.Exit(1)
@@ -107,5 +156,6 @@ func init() {
 
 	utilGroup.AddCommand(
 		NewUnpackCommand(),
+		NewPackCommand(),
 	)
 }

cmd/util/cli.go

@@ -156,11 +156,15 @@ func DisplayVersionBanner(c *types.Context, banner func(), version func() string
 		}
 	}
 	if display {
-		banner()
-		pterm.DefaultCenter.Print(version())
-		for _, l := range license {
-			pterm.DefaultCenter.Print(l)
-
+		if c.Config.General.Quiet {
+			pterm.Info.Printf("Luet %s\n", version())
+			pterm.Info.Println(strings.Join(license, "\n"))
+		} else {
+			banner()
+			pterm.DefaultCenter.Print(version())
+			for _, l := range license {
+				pterm.DefaultCenter.Print(l)
+			}
 		}
 	}
 }

cmd/util/config.go

@@ -120,8 +120,10 @@ func setDefaults(viper *viper.Viper) {
 
 	viper.SetDefault("general.concurrency", runtime.NumCPU())
 	viper.SetDefault("general.debug", false)
+	viper.SetDefault("general.quiet", false)
 	viper.SetDefault("general.show_build_output", false)
 	viper.SetDefault("general.fatal_warnings", false)
+	viper.SetDefault("general.http_timeout", 360)
 
 	u, err := user.Current()
 	// os/user doesn't work in from scratch environments
@@ -158,7 +160,8 @@ func InitViper(ctx *types.Context, RootCmd *cobra.Command) {
 	cobra.OnInitialize(initConfig)
 	pflags := RootCmd.PersistentFlags()
 	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
-	pflags.BoolP("debug", "d", false, "verbose output")
+	pflags.BoolP("debug", "d", false, "debug output")
+	pflags.BoolP("quiet", "q", false, "quiet output")
 	pflags.Bool("fatal", false, "Enables Warnings to exit")
 	pflags.Bool("enable-logfile", false, "Enable log to file")
 	pflags.Bool("no-spinner", false, "Disable spinner.")
@@ -178,6 +181,7 @@ func InitViper(ctx *types.Context, RootCmd *cobra.Command) {
 	}
 	pflags.Bool("same-owner", ctx.Config.GetGeneral().SameOwner, "Maintain same owner on uncompress.")
 	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
+	pflags.Int("http-timeout", ctx.Config.General.HTTPTimeout, "Default timeout for http(s) requests")
 
 	viper.BindPFlag("logging.color", pflags.Lookup("color"))
 	viper.BindPFlag("logging.enable_emoji", pflags.Lookup("emoji"))
@@ -186,9 +190,11 @@ func InitViper(ctx *types.Context, RootCmd *cobra.Command) {
 
 	viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
 	viper.BindPFlag("general.debug", pflags.Lookup("debug"))
+	viper.BindPFlag("general.quiet", pflags.Lookup("quiet"))
 	viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
 	viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
 	viper.BindPFlag("plugin", pflags.Lookup("plugin"))
+	viper.BindPFlag("general.http_timeout", pflags.Lookup("http-timeout"))
 
 	// Currently I maintain this only from cli.
 	viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))

go.mod

@@ -6,60 +6,55 @@ require (
 	github.com/DataDog/zstd v1.4.5 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
-	github.com/Microsoft/go-winio v0.5.0 // indirect
 	github.com/Sabayon/pkgs-checker v0.8.4
-	github.com/apex/log v1.9.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec
-	github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d // indirect
-	github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc
-	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/containerd/containerd v1.5.2
 	github.com/crillab/gophersat v1.3.2-0.20210701121804-72b19f5b6b38
-	github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible
+	github.com/docker/cli v20.10.7+incompatible
 	github.com/docker/distribution v2.7.1+incompatible
-	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
+	github.com/docker/docker v20.10.7+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
-	github.com/google/go-containerregistry v0.2.1
+	github.com/google/go-containerregistry v0.6.0
 	github.com/google/renameio v1.0.0
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/gookit/color v1.5.0 // indirect
+	github.com/gookit/color v1.5.0
+	github.com/gorilla/mux v1.7.4 // indirect
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/hashicorp/go-version v1.3.0
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
-	github.com/klauspost/compress v1.12.2
-	github.com/klauspost/pgzip v1.2.1
+	github.com/klauspost/compress v1.13.0
+	github.com/klauspost/pgzip v1.2.5
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.1
 	github.com/mitchellh/mapstructure v1.4.2 // indirect
+	github.com/moby/moby v20.10.9+incompatible
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
-	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
+	github.com/mudler/go-pluggable v0.0.0-20211206135551-9263b05c562e
 	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.16.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
-	github.com/opencontainers/runc v1.0.0-rc9.0.20200221051241-688cf6d43cc4 // indirect
 	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
 	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
 	github.com/pkg/errors v0.9.1
 	github.com/pterm/pterm v0.12.32-0.20211002183613-ada9ef6790c3
 	github.com/rancher-sandbox/gofilecache v0.0.0-20210330135715-becdeff5df15
-	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.8.1
@@ -71,15 +66,12 @@ require (
 	golang.org/x/mod v0.4.2
 	golang.org/x/oauth2 v0.0.0-20210810183815-faf39c7919d5 // indirect
 	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	google.golang.org/genproto v0.0.0-20210811021853-ddbe55d93216 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/ini.v1 v1.63.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0
-	gotest.tools/v3 v3.0.2 // indirect
 	helm.sh/helm/v3 v3.3.4
 
 )
-
-replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200605210607-749178b8f80d+incompatible

pkg/api/core/bus/events.go

@@ -1,8 +1,10 @@
 package bus
 
 import (
+	"fmt"
+
 	"github.com/mudler/go-pluggable"
-	"github.com/sirupsen/logrus"
+	"github.com/mudler/luet/pkg/api/core/types"
 )
 
 var (
@@ -12,6 +14,10 @@ var (
 	EventPackageInstall pluggable.EventType = "package.install"
 	// EventPackageUnInstall is the event fired when a new package is being uninstalled
 	EventPackageUnInstall pluggable.EventType = "package.uninstall"
+	// EventPreUpgrade is the event fired before an upgrade is attempted
+	EventPreUpgrade pluggable.EventType = "package.pre.upgrade"
+	// EventPostUpgrade is the event fired after an upgrade is done
+	EventPostUpgrade pluggable.EventType = "package.post.upgrade"
 
 	// Package build
 
@@ -61,6 +67,8 @@ var Manager *Bus = &Bus{
 			EventPackageInstall,
 			EventPackageUnInstall,
 			EventPackagePreBuild,
+			EventPreUpgrade,
+			EventPostUpgrade,
 			EventPackagePreBuildArtifact,
 			EventPackagePostBuildArtifact,
 			EventPackagePostBuild,
@@ -82,15 +90,12 @@ type Bus struct {
 	*pluggable.Manager
 }
 
-func (b *Bus) Initialize(plugin ...string) {
+func (b *Bus) Initialize(ctx *types.Context, plugin ...string) {
 	b.Manager.Load(plugin...).Register()
 
 	for _, e := range b.Manager.Events {
 		b.Manager.Response(e, func(p *pluggable.Plugin, r *pluggable.EventResponse) {
-			if r.Errored() {
-				logrus.Fatal("Plugin", p.Name, "at", p.Executable, "Error", r.Error)
-			}
-			logrus.Debug(
+			ctx.Debug(
 				"plugin_event",
 				"received from",
 				p.Name,
@@ -98,6 +103,16 @@ func (b *Bus) Initialize(plugin ...string) {
 				p.Executable,
 				r,
 			)
+			if r.Errored() {
+				err := fmt.Sprintf("Plugin %s at %s had an error: %s", p.Name, p.Executable, r.Error)
+				ctx.Fatal(err)
+			} else {
+				if r.State != "" {
+					message := fmt.Sprintf(":lollipop: Plugin %s at %s succeded, state reported:", p.Name, p.Executable)
+					ctx.Success(message)
+					ctx.Info(r.State)
+				}
+			}
 		})
 	}
 }

pkg/api/core/config/config_protect.go

@@ -53,18 +53,10 @@ func NewConfigProtect(annotationDir string) *ConfigProtect {
 	}
 	return &ConfigProtect{
 		AnnotationDir: annotationDir,
-		MapProtected:  make(map[string]bool, 0),
+		MapProtected:  make(map[string]bool),
 	}
 }
 
-func (c *ConfigProtect) AddAnnotationDir(d string) {
-	c.AnnotationDir = d
-}
-
-func (c *ConfigProtect) GetAnnotationDir() string {
-	return c.AnnotationDir
-}
-
 func (c *ConfigProtect) Map(files []string, protected []ConfigProtectConfFile) {
 
 	for _, file := range files {
@@ -105,7 +97,7 @@ func (c *ConfigProtect) Protected(file string) bool {
 func (c *ConfigProtect) GetProtectFiles(withSlash bool) []string {
 	ans := []string{}
 
-	for key, _ := range c.MapProtected {
+	for key := range c.MapProtected {
 		if withSlash {
 			ans = append(ans, key)
 		} else {

pkg/api/core/image/cache.go

@@ -0,0 +1,168 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"encoding/json"
+	"os"
+	"strings"
+
+	"github.com/peterbourgon/diskv"
+)
+
+// Cache represents a key-value store which is capable to upgrade to disk when it
+// reaches a pre-defined threshold.
+type Cache struct {
+	store                      *diskv.Diskv
+	memory                     map[string]string
+	dir                        string
+	onDisk                     bool
+	maxmemorySize, maxItemSize int
+}
+
+// New creates a new key-value cache
+// the cache acts in memory as long as the maxItemsize is not reached.
+// Once the threshold is met the cache is offloaded to disk automatically,
+// with a buffer of maxmemorySize into memory.
+func NewCache(path string, maxmemorySize, maxItemsize int) *Cache {
+	disk := diskv.New(diskv.Options{
+		BasePath:     path,
+		CacheSizeMax: uint64(maxmemorySize), // 500MB
+	})
+
+	return &Cache{
+		memory:        make(map[string]string),
+		store:         disk,
+		dir:           path,
+		maxmemorySize: maxmemorySize,
+		maxItemSize:   maxItemsize,
+	}
+}
+
+// This is needed as the disk cache is merely stored as separate files
+// thus we don't want to conflict file names with the path separator.
+// XXX: This is inconvenient as while we are looping result we can't rely
+// anymore originally to the key name.
+// We don't do any hashing to avoid any performance impact
+func cleanKey(s string) string {
+	return strings.ReplaceAll(s, string(os.PathSeparator), "_")
+}
+
+// Count returns the items in the cache.
+// If it's a disk cache might be an expensive call.
+func (c *Cache) Count() int {
+	if !c.onDisk {
+		return len(c.memory)
+	}
+
+	count := 0
+	for range c.store.Keys(nil) {
+		count++
+	}
+	return count
+}
+
+// Get attempts to retrieve a value for a key
+func (c *Cache) Get(key string) (value string, found bool) {
+
+	if !c.onDisk {
+		v, ok := c.memory[key]
+		return v, ok
+	}
+	v, err := c.store.Read(cleanKey(key))
+	if err == nil {
+		found = true
+	}
+	value = string(v)
+	return
+}
+
+func (c *Cache) flushToDisk() {
+	for k, v := range c.memory {
+		c.store.Write(cleanKey(k), []byte(v))
+	}
+	c.memory = make(map[string]string)
+	c.onDisk = true
+}
+
+// Set updates or inserts a new value
+func (c *Cache) Set(key, value string) error {
+
+	if !c.onDisk && c.Count() >= c.maxItemSize && c.maxItemSize != 0 {
+		c.flushToDisk()
+	}
+
+	if c.onDisk {
+		return c.store.Write(cleanKey(key), []byte(value))
+	}
+
+	c.memory[key] = value
+
+	return nil
+}
+
+// SetValue updates or inserts a new value by marshalling it into JSON.
+func (c *Cache) SetValue(key string, value interface{}) error {
+	dat, err := json.Marshal(value)
+	if err != nil {
+		return err
+	}
+	return c.Set(cleanKey(key), string(dat))
+}
+
+// CacheResult represent the key value result when
+// iterating over the cache
+type CacheResult struct {
+	key, value string
+}
+
+// Value returns the underlying value
+func (c CacheResult) Value() string {
+	return c.value
+}
+
+// Key returns the cache result key
+func (c CacheResult) Key() string {
+	return c.key
+}
+
+// Unmarshal the result into the interface. Use it to retrieve data
+// set with SetValue
+func (c CacheResult) Unmarshal(i interface{}) error {
+	return json.Unmarshal([]byte(c.Value()), i)
+}
+
+// Iterates over cache by key
+func (c *Cache) All(fn func(CacheResult)) {
+	if !c.onDisk {
+		for k, v := range c.memory {
+			fn(CacheResult{key: k, value: v})
+		}
+		return
+	}
+
+	for key := range c.store.Keys(nil) {
+		val, _ := c.store.Read(key)
+		fn(CacheResult{key: key, value: string(val)})
+	}
+}
+
+// Clean the cache
+func (c *Cache) Clean() {
+	c.memory = make(map[string]string)
+	c.onDisk = false
+	os.RemoveAll(c.dir)
+}

pkg/api/core/image/cache_test.go

@@ -0,0 +1,98 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"path/filepath"
+
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Cache", func() {
+
+	ctx := types.NewContext()
+	Context("used as k/v store", func() {
+
+		cache := &Cache{}
+		var dir string
+
+		BeforeEach(func() {
+			ctx = types.NewContext()
+			var err error
+			dir, err = ctx.Config.System.TempDir("foo")
+			Expect(err).ToNot(HaveOccurred())
+			cache = NewCache(dir, 10*1024*1024, 1) // 10MB Cache when upgrading to files. Max volatile memory of 1 row.
+		})
+
+		AfterEach(func() {
+			cache.Clean()
+		})
+
+		It("does handle automatically memory upgrade", func() {
+			cache.Set("foo", "bar")
+			v, found := cache.Get("foo")
+			Expect(found).To(BeTrue())
+			Expect(v).To(Equal("bar"))
+			Expect(file.Exists(filepath.Join(dir, "foo"))).To(BeFalse())
+			cache.Set("baz", "bar")
+			Expect(file.Exists(filepath.Join(dir, "foo"))).To(BeTrue())
+			Expect(file.Exists(filepath.Join(dir, "baz"))).To(BeTrue())
+			v, found = cache.Get("foo")
+			Expect(found).To(BeTrue())
+			Expect(v).To(Equal("bar"))
+
+			Expect(cache.Count()).To(Equal(2))
+		})
+
+		It("does CRUD", func() {
+			cache.Set("foo", "bar")
+
+			v, found := cache.Get("foo")
+			Expect(found).To(BeTrue())
+			Expect(v).To(Equal("bar"))
+
+			hit := false
+			cache.All(func(c CacheResult) {
+				hit = true
+				Expect(c.Key()).To(Equal("foo"))
+				Expect(c.Value()).To(Equal("bar"))
+			})
+			Expect(hit).To(BeTrue())
+
+		})
+
+		It("Unmarshals values", func() {
+			type testStruct struct {
+				Test string
+			}
+
+			cache.SetValue("foo", &testStruct{Test: "baz"})
+
+			n := &testStruct{}
+
+			cache.All(func(cr CacheResult) {
+				err := cr.Unmarshal(n)
+				Expect(err).ToNot(HaveOccurred())
+
+			})
+			Expect(n.Test).To(Equal("baz"))
+		})
+	})
+})

pkg/api/core/image/create.go

@@ -0,0 +1,97 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"io"
+	"os"
+
+	containerdCompression "github.com/containerd/containerd/archive/compression"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/pkg/errors"
+)
+
+func imageFromTar(imagename, architecture, OS string, opener func() (io.ReadCloser, error)) (name.Reference, v1.Image, error) {
+	newRef, err := name.ParseReference(imagename)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	layer, err := tarball.LayerFromOpener(opener)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	baseImage := empty.Image
+	cfg, err := baseImage.ConfigFile()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfg.Architecture = architecture
+	cfg.OS = OS
+
+	baseImage, err = mutate.ConfigFile(baseImage, cfg)
+	if err != nil {
+		return nil, nil, err
+	}
+	img, err := mutate.Append(baseImage, mutate.Addendum{
+		Layer: layer,
+		History: v1.History{
+			CreatedBy: "luet",
+			Comment:   "Custom image",
+		},
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return newRef, img, nil
+}
+
+// CreateTar a imagetarball from a standard tarball
+func CreateTar(srctar, dstimageTar, imagename, architecture, OS string) error {
+
+	dstFile, err := os.Create(dstimageTar)
+	if err != nil {
+		return errors.Wrap(err, "Cannot create "+dstimageTar)
+	}
+	defer dstFile.Close()
+
+	newRef, img, err := imageFromTar(imagename, architecture, OS, func() (io.ReadCloser, error) {
+		f, err := os.Open(srctar)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot open "+srctar)
+		}
+		decompressed, err := containerdCompression.DecompressStream(f)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot open "+srctar)
+		}
+
+		return decompressed, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// NOTE: We might also stream that back to the daemon with daemon.Write(tag, img)
+	return tarball.Write(newRef, img, dstFile)
+
+}

pkg/api/core/image/create_test.go

@@ -0,0 +1,80 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
+	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Create", func() {
+	Context("Creates an OCI image from a standard tar", func() {
+		It("creates an image which is loadable", func() {
+			ctx := types.NewContext()
+
+			dst, err := ctx.Config.System.TempFile("dst")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(dst.Name())
+			srcTar, err := ctx.Config.System.TempFile("srcTar")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(srcTar.Name())
+
+			b := backend.NewSimpleDockerBackend(ctx)
+
+			b.DownloadImage(backend.Options{ImageName: "alpine"})
+			img, err := b.ImageReference("alpine", false)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, dir, err := Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			defer os.RemoveAll(dir)
+
+			Expect(file.Touch(filepath.Join(dir, "test"))).ToNot(HaveOccurred())
+			Expect(file.Exists(filepath.Join(dir, "bin"))).To(BeTrue())
+
+			a := artifact.NewPackageArtifact(srcTar.Name())
+			a.Compress(dir, 1)
+
+			// Unfortunately there is no other easy way to test this
+			err = CreateTar(srcTar.Name(), dst.Name(), "testimage", runtime.GOARCH, runtime.GOOS)
+			Expect(err).ToNot(HaveOccurred())
+
+			b.LoadImage(dst.Name())
+
+			Expect(b.ImageExists("testimage")).To(BeTrue())
+
+			img, err = b.ImageReference("testimage", false)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, dir, err = Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			defer os.RemoveAll(dir)
+			Expect(file.Exists(filepath.Join(dir, "bin"))).To(BeTrue())
+			Expect(file.Exists(filepath.Join(dir, "test"))).To(BeTrue())
+		})
+	})
+})

pkg/api/core/image/delta.go

@@ -0,0 +1,111 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"archive/tar"
+	"io"
+	"regexp"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+)
+
+func compileRegexes(regexes []string) []*regexp.Regexp {
+	var result []*regexp.Regexp
+	for _, i := range regexes {
+		r, e := regexp.Compile(i)
+		if e != nil {
+			continue
+		}
+		result = append(result, r)
+	}
+	return result
+}
+
+type ImageDiffNode struct {
+	Name string `json:"Name"`
+	Size int    `json:"Size"`
+}
+type ImageDiff struct {
+	Additions []ImageDiffNode `json:"Adds"`
+	Deletions []ImageDiffNode `json:"Dels"`
+	Changes   []ImageDiffNode `json:"Mods"`
+}
+
+func Delta(srcimg, dstimg v1.Image) (res ImageDiff, err error) {
+	srcReader := mutate.Extract(srcimg)
+	defer srcReader.Close()
+
+	dstReader := mutate.Extract(dstimg)
+	defer dstReader.Close()
+
+	filesSrc, filesDst := map[string]int64{}, map[string]int64{}
+
+	srcTar := tar.NewReader(srcReader)
+	dstTar := tar.NewReader(dstReader)
+
+	for {
+		var hdr *tar.Header
+		hdr, err = srcTar.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return
+		}
+		filesSrc[hdr.Name] = hdr.Size
+	}
+
+	for {
+		var hdr *tar.Header
+		hdr, err = dstTar.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return
+		}
+		filesDst[hdr.Name] = hdr.Size
+	}
+	err = nil
+
+	for f, size := range filesDst {
+		if size2, exist := filesSrc[f]; exist && size2 != size {
+			res.Changes = append(res.Changes, ImageDiffNode{
+				Name: f,
+				Size: int(size),
+			})
+		} else if !exist {
+			res.Additions = append(res.Additions, ImageDiffNode{
+				Name: f,
+				Size: int(size),
+			})
+		}
+	}
+	for f, size := range filesSrc {
+		if _, exist := filesDst[f]; !exist {
+			res.Deletions = append(res.Deletions, ImageDiffNode{
+				Name: f,
+				Size: int(size),
+			})
+		}
+	}
+
+	return
+}

pkg/api/core/image/delta_test.go

@@ -0,0 +1,134 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	daemon "github.com/google/go-containerregistry/pkg/v1/daemon"
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Delta", func() {
+	Context("Generates deltas of images", func() {
+		It("computes delta", func() {
+			ref, err := name.ParseReference("alpine")
+			Expect(err).ToNot(HaveOccurred())
+
+			img, err := daemon.Image(ref)
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := Delta(img, img)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers.Changes)).To(Equal(0))
+			Expect(len(layers.Additions)).To(Equal(0))
+			Expect(len(layers.Deletions)).To(Equal(0))
+		})
+
+		Context("ExtractDeltaFiles", func() {
+			ctx := types.NewContext()
+			var tmpfile *os.File
+			var ref, ref2 name.Reference
+			var img, img2 v1.Image
+			var err error
+
+			ref, _ = name.ParseReference("alpine")
+			ref2, _ = name.ParseReference("golang:alpine")
+			img, _ = daemon.Image(ref)
+			img2, _ = daemon.Image(ref2)
+
+			BeforeEach(func() {
+				ctx = types.NewContext()
+
+				tmpfile, err = ioutil.TempFile("", "delta")
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpfile.Name()) // clean up
+			})
+
+			It("Extract all deltas", func() {
+
+				f, err := ExtractDeltaAdditionsFiles(ctx, img, []string{}, []string{})
+				Expect(err).ToNot(HaveOccurred())
+
+				_, tmpdir, err := Extract(
+					ctx,
+					img2,
+					f,
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+				Expect(file.Exists(filepath.Join(tmpdir, "home"))).To(BeFalse())
+				Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeTrue())
+			})
+
+			It("Extract deltas and excludes /usr/local/go", func() {
+				f, err := ExtractDeltaAdditionsFiles(ctx, img, []string{}, []string{"usr/local/go"})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(err).ToNot(HaveOccurred())
+				_, tmpdir, err := Extract(
+					ctx,
+					img2,
+					f,
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeFalse())
+			})
+
+			It("Extract deltas and excludes /usr/local/go/bin, but includes /usr/local/go", func() {
+				f, err := ExtractDeltaAdditionsFiles(ctx, img, []string{"usr/local/go"}, []string{"usr/local/go/bin"})
+				Expect(err).ToNot(HaveOccurred())
+
+				_, tmpdir, err := Extract(
+					ctx,
+					img2,
+					f,
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeFalse())
+			})
+
+			It("Extract deltas and includes /usr/local/go", func() {
+				f, err := ExtractDeltaAdditionsFiles(ctx, img, []string{"usr/local/go"}, []string{})
+				Expect(err).ToNot(HaveOccurred())
+				_, tmpdir, err := Extract(
+					ctx,
+					img2,
+					f,
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeFalse())
+			})
+		})
+	})
+})

pkg/api/core/image/extract.go

@@ -0,0 +1,227 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"archive/tar"
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	containerdarchive "github.com/containerd/containerd/archive"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/pkg/errors"
+)
+
+// ExtractDeltaAdditionsFromImages is a filter that takes two images
+// an includes and an excludes list. It computes the delta between the images
+// considering the added files only, and applies a filter on them based on the regexes
+// in the lists.
+func ExtractDeltaAdditionsFiles(
+	ctx *types.Context,
+	srcimg v1.Image,
+	includes []string, excludes []string,
+) (func(h *tar.Header) (bool, error), error) {
+
+	includeRegexp := compileRegexes(includes)
+	excludeRegexp := compileRegexes(excludes)
+
+	srcfilesd, err := ctx.Config.System.TempDir("srcfiles")
+	if err != nil {
+		return nil, err
+	}
+	filesSrc := NewCache(srcfilesd, 50*1024*1024, 10000)
+
+	srcReader := mutate.Extract(srcimg)
+	defer srcReader.Close()
+
+	srcTar := tar.NewReader(srcReader)
+
+	for {
+		var hdr *tar.Header
+		hdr, err := srcTar.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			filesSrc.Set(filepath.Dir(hdr.Name), "")
+		default:
+			filesSrc.Set(hdr.Name, "")
+		}
+	}
+
+	return func(h *tar.Header) (bool, error) {
+
+		fileName := filepath.Join(string(os.PathSeparator), h.Name)
+		_, exists := filesSrc.Get(h.Name)
+		if exists {
+			return false, nil
+		}
+
+		switch {
+		case len(includes) == 0 && len(excludes) != 0:
+			for _, i := range excludeRegexp {
+				if i.MatchString(filepath.Join(string(os.PathSeparator), h.Name)) &&
+					fileName == filepath.Join(string(os.PathSeparator), h.Name) {
+					return false, nil
+				}
+			}
+			ctx.Debug("Adding name", fileName)
+
+			return true, nil
+		case len(includes) > 0 && len(excludes) == 0:
+			for _, i := range includeRegexp {
+				if i.MatchString(filepath.Join(string(os.PathSeparator), h.Name)) && fileName == filepath.Join(string(os.PathSeparator), h.Name) {
+					ctx.Debug("Adding name", fileName)
+
+					return true, nil
+				}
+			}
+			return false, nil
+		case len(includes) != 0 && len(excludes) != 0:
+			for _, i := range includeRegexp {
+				if i.MatchString(filepath.Join(string(os.PathSeparator), h.Name)) && fileName == filepath.Join(string(os.PathSeparator), h.Name) {
+					for _, e := range excludeRegexp {
+						if e.MatchString(fileName) {
+							return false, nil
+						}
+					}
+					ctx.Debug("Adding name", fileName)
+
+					return true, nil
+				}
+			}
+
+			return false, nil
+		default:
+			ctx.Debug("Adding name", fileName)
+			return true, nil
+		}
+
+	}, nil
+}
+
+// ExtractFiles returns a filter that extracts files from the given path (if not empty)
+// It then filters files by an include and exclude list.
+// The list can be regexes
+func ExtractFiles(
+	ctx *types.Context,
+	prefixPath string,
+	includes []string, excludes []string,
+) func(h *tar.Header) (bool, error) {
+	includeRegexp := compileRegexes(includes)
+	excludeRegexp := compileRegexes(excludes)
+
+	return func(h *tar.Header) (bool, error) {
+
+		fileName := filepath.Join(string(os.PathSeparator), h.Name)
+		switch {
+		case len(includes) == 0 && len(excludes) != 0:
+			for _, i := range excludeRegexp {
+				if i.MatchString(filepath.Join(prefixPath, fileName)) {
+					return false, nil
+				}
+			}
+			if prefixPath != "" {
+				return strings.HasPrefix(fileName, prefixPath), nil
+			}
+			ctx.Debug("Adding name", fileName)
+			return true, nil
+
+		case len(includes) > 0 && len(excludes) == 0:
+			for _, i := range includeRegexp {
+				if i.MatchString(filepath.Join(prefixPath, fileName)) {
+					if prefixPath != "" {
+						return strings.HasPrefix(fileName, prefixPath), nil
+					}
+					ctx.Debug("Adding name", fileName)
+
+					return true, nil
+				}
+			}
+			return false, nil
+		case len(includes) != 0 && len(excludes) != 0:
+			for _, i := range includeRegexp {
+				if i.MatchString(filepath.Join(prefixPath, fileName)) {
+					for _, e := range excludeRegexp {
+						if e.MatchString(filepath.Join(prefixPath, fileName)) {
+							return false, nil
+						}
+					}
+					if prefixPath != "" {
+						return strings.HasPrefix(fileName, prefixPath), nil
+					}
+					ctx.Debug("Adding name", fileName)
+
+					return true, nil
+				}
+			}
+			return false, nil
+		default:
+			if prefixPath != "" {
+				return strings.HasPrefix(fileName, prefixPath), nil
+			}
+
+			return true, nil
+		}
+	}
+}
+
+// ExtractReader perform the extracting action over the io.ReadCloser
+// it extracts the files over output. Accepts a filter as an option
+// and additional containerd Options
+func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+	defer reader.Close()
+
+	// If no filter is specified, grab all.
+	if filter == nil {
+		filter = func(h *tar.Header) (bool, error) { return true, nil }
+	}
+
+	opts = append(opts, containerdarchive.WithFilter(filter))
+
+	// Handle the extraction
+	c, err := containerdarchive.Apply(context.Background(), output, reader, opts...)
+	if err != nil {
+		return 0, "", err
+	}
+
+	return c, output, nil
+}
+
+// Extract is just syntax sugar around ExtractReader. It extracts an image into a dir
+func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
+	if err != nil {
+		return 0, "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, filter, opts...)
+}
+
+// ExtractTo is just syntax sugar around ExtractReader
+func ExtractTo(ctx *types.Context, img v1.Image, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+	return ExtractReader(ctx, mutate.Extract(img), output, filter, opts...)
+}

pkg/api/core/image/extract_test.go

@@ -0,0 +1,111 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	daemon "github.com/google/go-containerregistry/pkg/v1/daemon"
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Extract", func() {
+
+	Context("extract files from images", func() {
+		Context("ExtractFiles", func() {
+			ctx := types.NewContext()
+			var tmpfile *os.File
+			var ref name.Reference
+			var img v1.Image
+			var err error
+
+			BeforeEach(func() {
+				ctx = types.NewContext()
+
+				tmpfile, err = ioutil.TempFile("", "extract")
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpfile.Name()) // clean up
+
+				ref, err = name.ParseReference("alpine")
+				Expect(err).ToNot(HaveOccurred())
+
+				img, err = daemon.Image(ref)
+				Expect(err).ToNot(HaveOccurred())
+			})
+
+			It("Extract all files", func() {
+				_, tmpdir, err := Extract(
+					ctx,
+					img,
+					ExtractFiles(ctx, "", []string{}, []string{}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "bin"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeTrue())
+			})
+
+			It("Extract specific dir", func() {
+				_, tmpdir, err := Extract(
+					ctx,
+					img,
+					ExtractFiles(ctx, "/usr", []string{}, []string{}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "sbin"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "bin"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
+			})
+
+			It("Extract a dir with includes/excludes", func() {
+				_, tmpdir, err := Extract(
+					ctx,
+					img,
+					ExtractFiles(ctx, "/usr", []string{"bin"}, []string{"sbin"}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "bin"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "sbin"))).To(BeFalse())
+			})
+
+			It("Extract with includes/excludes", func() {
+				_, tmpdir, err := Extract(
+					ctx,
+					img,
+					ExtractFiles(ctx, "", []string{"/usr|/usr/bin"}, []string{"^/bin"}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "bin"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
+			})
+		})
+	})
+})

pkg/api/core/image/mutator_suite_test.go

@@ -1,4 +1,4 @@
-// Copyright © 2021 Ettore Di Giacinto <mudler@sabayon.org>
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -13,18 +13,22 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package docker_test
+package image_test
 
 import (
-	"github.com/mudler/luet/pkg/helpers/docker"
+	"testing"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("StripInvalidStringsFromImage", func() {
-	Context("Image names", func() {
-		It("strips invalid chars", func() {
-			Expect(docker.StripInvalidStringsFromImage("foo+bar")).To(Equal("foo-bar"))
-		})
-	})
-})
+func TestMutator(t *testing.T) {
+	RegisterFailHandler(Fail)
+	b := backend.NewSimpleDockerBackend(types.NewContext())
+	b.DownloadImage(backend.Options{ImageName: "alpine"})
+	b.DownloadImage(backend.Options{ImageName: "golang:alpine"})
+
+	RunSpecs(t, "Mutator API Suite")
+}

pkg/api/core/types/artifact/artifact.go

@@ -27,18 +27,21 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
+	"runtime"
 
+	"github.com/docker/docker/pkg/pools"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	zstd "github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
 	"strings"
-	"sync"
 
+	containerdCompression "github.com/containerd/containerd/archive/compression"
+	bus "github.com/mudler/luet/pkg/api/core/bus"
 	config "github.com/mudler/luet/pkg/api/core/config"
+	"github.com/mudler/luet/pkg/api/core/image"
 	types "github.com/mudler/luet/pkg/api/core/types"
-	bus "github.com/mudler/luet/pkg/bus"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
@@ -67,6 +70,22 @@ type PackageArtifact struct {
 	Runtime           *pkg.DefaultPackage               `json:"runtime,omitempty"`
 }
 
+func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
+	_, tmpdiffs, err := image.Extract(ctx, img, filter)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	a := NewPackageArtifact(output)
+	a.CompressionType = t
+	err = a.Compress(tmpdiffs, 1)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while creating package archive")
+	}
+	return a, nil
+}
+
 func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
 	copy := *p
 	return &copy
@@ -156,12 +175,6 @@ func (a *PackageArtifact) GetFileName() string {
 	return path.Base(a.Path)
 }
 
-func (a *PackageArtifact) genDockerfile() string {
-	return `
-FROM scratch
-COPY . /`
-}
-
 // CreateArtifactForFile creates a new artifact from the given file
 func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
 	if _, err := os.Stat(s); os.IsNotExist(err) {
@@ -193,52 +206,26 @@ func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageAr
 
 type ImageBuilder interface {
 	BuildImage(backend.Options) error
+	LoadImage(path string) error
 }
 
 // GenerateFinalImage takes an artifact and builds a Docker image with its content
-func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) {
-	builderOpts := backend.Options{}
-	archive, err := ctx.Config.GetSystem().TempDir("archive")
+func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) error {
+
+	tempimage, err := ctx.Config.GetSystem().TempFile("tempimage")
 	if err != nil {
-		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+		return errors.Wrap(err, "error met while creating tempdir for "+a.Path)
 	}
-	defer os.RemoveAll(archive) // clean up
+	defer os.RemoveAll(tempimage.Name()) // clean up
 
-	uncompressedFiles := filepath.Join(archive, "files")
-	dockerFile := filepath.Join(archive, "Dockerfile")
-
-	if err := os.MkdirAll(uncompressedFiles, os.ModePerm); err != nil {
-		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	if err := image.CreateTar(a.Path, tempimage.Name(), imageName, runtime.GOARCH, runtime.GOOS); err != nil {
+		return errors.Wrap(err, "could not create image from tar")
 	}
 
-	if err := a.Unpack(ctx, uncompressedFiles, keepPerms); err != nil {
-		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
+	if err := b.LoadImage(tempimage.Name()); err != nil {
+		return errors.Wrap(err, "while loading image")
 	}
-
-	empty, err := fileHelper.DirectoryIsEmpty(uncompressedFiles)
-	if err != nil {
-		return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles)
-	}
-
-	// See https://github.com/moby/moby/issues/38039.
-	// We can't generate FROM scratch empty images. Docker will refuse to export them
-	// workaround: Inject a .virtual empty file
-	if empty {
-		fileHelper.Touch(filepath.Join(uncompressedFiles, ".virtual"))
-	}
-
-	data := a.genDockerfile()
-	if err := ioutil.WriteFile(dockerFile, []byte(data), 0644); err != nil {
-		return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path)
-	}
-
-	builderOpts = backend.Options{
-		ImageName:      imageName,
-		SourcePath:     archive,
-		DockerFileName: dockerFile,
-		Context:        uncompressedFiles,
-	}
-	return builderOpts, b.BuildImage(builderOpts)
+	return nil
 }
 
 // Compress is responsible to archive and compress to the artifact Path.
@@ -369,6 +356,78 @@ func hashFileContent(path string) (string, error) {
 	return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil
 }
 
+func replaceFileTarWrapper(dst string, inputTarStream io.ReadCloser, mods []string, fn func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)) io.ReadCloser {
+	pipeReader, pipeWriter := io.Pipe()
+
+	go func() {
+		tarReader := tar.NewReader(inputTarStream)
+		tarWriter := tar.NewWriter(pipeWriter)
+		defer inputTarStream.Close()
+		defer tarWriter.Close()
+
+		modify := func(name string, original *tar.Header, tarReader io.Reader) error {
+			header, data, err := fn(dst, name, original, tarReader)
+			switch {
+			case err != nil:
+				return err
+			case header == nil:
+				return nil
+			}
+
+			if header.Name == "" {
+				header.Name = name
+			}
+			header.Size = int64(len(data))
+			if err := tarWriter.WriteHeader(header); err != nil {
+				return err
+			}
+			if len(data) != 0 {
+				if _, err := tarWriter.Write(data); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+		//	var remaining []string
+		var err error
+		var originalHeader *tar.Header
+		for {
+			originalHeader, err = tarReader.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				pipeWriter.CloseWithError(err)
+				return
+			}
+
+			if !helpers.Contains(mods, originalHeader.Name) {
+				// No modifiers for this file, copy the header and data
+				if err := tarWriter.WriteHeader(originalHeader); err != nil {
+					pipeWriter.CloseWithError(err)
+					return
+				}
+				if _, err := pools.Copy(tarWriter, tarReader); err != nil {
+					pipeWriter.CloseWithError(err)
+					return
+				}
+				continue
+			}
+
+			if err := modify(originalHeader.Name, originalHeader, tarReader); err != nil {
+				pipeWriter.CloseWithError(err)
+				return
+			}
+		}
+
+		// Apply the modifiers that haven't matched any files in the archive
+
+		pipeWriter.Close()
+
+	}()
+	return pipeReader
+}
+
 func tarModifierWrapperFunc(ctx *types.Context) func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
 	return func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
 		// If the destination path already exists I rename target file name with postfix.
@@ -404,11 +463,15 @@ func tarModifierWrapperFunc(ctx *types.Context) func(dst, path string, header *t
 				}
 			}
 
-			ctx.Debug("Existing file hash: ", existingHash, "Tar file hashsum: ", tarHash)
+			ctx.Debug(destPath, "- existing file hash: ", existingHash, "Tar file hashsum: ", tarHash)
+			if fileHelper.Exists(destPath) {
+				ctx.Debug(destPath, "already exists")
+			}
 			// We want to protect file only if the hash of the files are differing OR the file size are
 			differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
 			// Check if exists
 			if fileHelper.Exists(destPath) && differs {
+				ctx.Debug(destPath, "already exists and differs")
 				for i := 1; i < 1000; i++ {
 					name := filepath.Join(filepath.Join(filepath.Dir(path),
 						fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
@@ -432,8 +495,7 @@ func tarModifierWrapperFunc(ctx *types.Context) func(dst, path string, header *t
 	}
 }
 
-func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) []string {
-	ans := []string{}
+func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) (res []string) {
 	annotationDir := ""
 
 	if !ctx.Config.ConfigProtectSkip {
@@ -452,159 +514,73 @@ func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) []string {
 		cp.Map(a.Files, ctx.Config.GetConfigProtectConfFiles())
 
 		// NOTE: for unpack we need files path without initial /
-		ans = cp.GetProtectFiles(false)
+		res = cp.GetProtectFiles(false)
 	}
 
-	return ans
+	return
 }
 
 // Unpack Untar and decompress (TODO) to the given path
 func (a *PackageArtifact) Unpack(ctx *types.Context, dst string, keepPerms bool) error {
-	if !strings.HasPrefix(dst, "/") {
+
+	if !strings.HasPrefix(dst, string(os.PathSeparator)) {
 		return errors.New("destination must be an absolute path")
 	}
 
 	// Create
 	protectedFiles := a.GetProtectFiles(ctx)
 
-	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc(ctx))
+	mod := tarModifierWrapperFunc(ctx)
+	//tarModifier := helpers.NewTarModifierWrapper(dst, mod)
 
-	switch a.CompressionType {
-	case compression.Zstandard:
-		// Create the uncompressed archive
-		archive, err := os.Create(a.Path + ".uncompressed")
-		if err != nil {
-			return err
-		}
-		defer os.RemoveAll(a.Path + ".uncompressed")
-		defer archive.Close()
-
-		original, err := os.Open(a.Path)
-		if err != nil {
-			return errors.Wrap(err, "Cannot open "+a.Path)
-		}
-		defer original.Close()
-
-		bufferedReader := bufio.NewReader(original)
-
-		d, err := zstd.NewReader(bufferedReader)
-		if err != nil {
-			return err
-		}
-		defer d.Close()
-
-		_, err = io.Copy(archive, d)
-		if err != nil {
-			return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
-		}
-
-		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
-			ctx.Config.GetGeneral().SameOwner, protectedFiles, tarModifier)
-		if err != nil {
-			return err
-		}
-		return nil
-	case compression.GZip:
-		// Create the uncompressed archive
-		archive, err := os.Create(a.Path + ".uncompressed")
-		if err != nil {
-			return err
-		}
-		defer os.RemoveAll(a.Path + ".uncompressed")
-		defer archive.Close()
-
-		original, err := os.Open(a.Path)
-		if err != nil {
-			return errors.Wrap(err, "Cannot open "+a.Path)
-		}
-		defer original.Close()
-
-		bufferedReader := bufio.NewReader(original)
-		r, err := gzip.NewReader(bufferedReader)
-		if err != nil {
-			return err
-		}
-		defer r.Close()
-
-		_, err = io.Copy(archive, r)
-		if err != nil {
-			return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
-		}
-
-		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
-			ctx.Config.GetGeneral().SameOwner, protectedFiles, tarModifier)
-		if err != nil {
-			return err
-		}
-		return nil
-	// Defaults to tar only (covers when "none" is supplied)
-	default:
-		return helpers.UntarProtect(a.Path, dst, ctx.Config.GetGeneral().SameOwner,
-			protectedFiles, tarModifier)
+	archiveFile, err := os.Open(a.Path)
+	if err != nil {
+		return errors.Wrap(err, "Cannot open "+a.Path)
 	}
-	return errors.New("Compression type must be supplied")
+	defer archiveFile.Close()
+
+	decompressed, err := containerdCompression.DecompressStream(archiveFile)
+	if err != nil {
+		return errors.Wrap(err, "Cannot open "+a.Path)
+	}
+
+	replacerArchive := replaceFileTarWrapper(dst, decompressed, protectedFiles, mod)
+
+	// or with filter?
+	// func(header *tar.Header) (bool, error) {
+	// 	if helpers.Contains(protectedFiles, header.Name) {
+	// 		newHead, _, err := mod(dst, header.Name, header, decompressed)
+	// 		if err != nil {
+	// 			return false, err
+	// 		}
+	// 		header.Name = newHead.Name
+	// 		// Override target path
+	// 		//target = filepath.Join(dest, header.Name)
+	// 	}
+	// 	//	tarModifier.Modifier()
+	// 	return true, nil
+	// },
+	_, _, err = image.ExtractReader(ctx, replacerArchive, dst, nil)
+	return err
 }
 
 // FileList generates the list of file of a package from the local archive
 func (a *PackageArtifact) FileList() ([]string, error) {
-	var tr *tar.Reader
-	switch a.CompressionType {
-	case compression.Zstandard:
-		archive, err := os.Create(a.Path + ".uncompressed")
-		if err != nil {
-			return []string{}, err
-		}
-		defer os.RemoveAll(a.Path + ".uncompressed")
-		defer archive.Close()
-
-		original, err := os.Open(a.Path)
-		if err != nil {
-			return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
-		}
-		defer original.Close()
-
-		bufferedReader := bufio.NewReader(original)
-		r, err := zstd.NewReader(bufferedReader)
-		if err != nil {
-			return []string{}, err
-		}
-		defer r.Close()
-		tr = tar.NewReader(r)
-	case compression.GZip:
-		// Create the uncompressed archive
-		archive, err := os.Create(a.Path + ".uncompressed")
-		if err != nil {
-			return []string{}, err
-		}
-		defer os.RemoveAll(a.Path + ".uncompressed")
-		defer archive.Close()
-
-		original, err := os.Open(a.Path)
-		if err != nil {
-			return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
-		}
-		defer original.Close()
-
-		bufferedReader := bufio.NewReader(original)
-		r, err := gzip.NewReader(bufferedReader)
-		if err != nil {
-			return []string{}, err
-		}
-		defer r.Close()
-		tr = tar.NewReader(r)
-
-	// Defaults to tar only (covers when "none" is supplied)
-	default:
-		tarFile, err := os.Open(a.Path)
-		if err != nil {
-			return []string{}, errors.Wrap(err, "Could not open package archive")
-		}
-		defer tarFile.Close()
-		tr = tar.NewReader(tarFile)
-
-	}
-
 	var files []string
+
+	archiveFile, err := os.Open(a.Path)
+	if err != nil {
+		return files, errors.Wrap(err, "Cannot open "+a.Path)
+	}
+	defer archiveFile.Close()
+
+	decompressed, err := containerdCompression.DecompressStream(archiveFile)
+	if err != nil {
+		return files, errors.Wrap(err, "Cannot open "+a.Path)
+	}
+	defer decompressed.Close()
+	tr := tar.NewReader(decompressed)
+
 	// untar each segment
 	for {
 		hdr, err := tr.Next()
@@ -626,181 +602,3 @@ func (a *PackageArtifact) FileList() ([]string, error) {
 	}
 	return files, nil
 }
-
-type CopyJob struct {
-	Src, Dst string
-	Artifact string
-}
-
-func worker(ctx *types.Context, i int, wg *sync.WaitGroup, s <-chan CopyJob) {
-	defer wg.Done()
-
-	for job := range s {
-		_, err := os.Lstat(job.Dst)
-		if err != nil {
-			ctx.Debug("Copying ", job.Src)
-			if err := fileHelper.DeepCopyFile(job.Src, job.Dst); err != nil {
-				ctx.Warning("Error copying", job, err)
-			}
-		}
-	}
-}
-
-func compileRegexes(regexes []string) []*regexp.Regexp {
-	var result []*regexp.Regexp
-	for _, i := range regexes {
-		r, e := regexp.Compile(i)
-		if e != nil {
-			continue
-		}
-		result = append(result, r)
-	}
-	return result
-}
-
-type ArtifactNode struct {
-	Name string `json:"Name"`
-	Size int    `json:"Size"`
-}
-type ArtifactDiffs struct {
-	Additions []ArtifactNode `json:"Adds"`
-	Deletions []ArtifactNode `json:"Dels"`
-	Changes   []ArtifactNode `json:"Mods"`
-}
-
-type ArtifactLayer struct {
-	FromImage string        `json:"Image1"`
-	ToImage   string        `json:"Image2"`
-	Diffs     ArtifactDiffs `json:"Diff"`
-}
-
-// ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(ctx *types.Context, src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) {
-
-	archive, err := ctx.Config.GetSystem().TempDir("archive")
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for archive")
-	}
-	defer os.RemoveAll(archive) // clean up
-
-	if strings.HasSuffix(src, ".tar") {
-		rootfs, err := ctx.Config.GetSystem().TempDir("rootfs")
-		if err != nil {
-			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-		}
-		defer os.RemoveAll(rootfs) // clean up
-		err = helpers.Untar(src, rootfs, keepPerms)
-		if err != nil {
-			return nil, errors.Wrap(err, "Error met while unpacking rootfs")
-		}
-		src = rootfs
-	}
-
-	toCopy := make(chan CopyJob)
-
-	var wg = new(sync.WaitGroup)
-	for i := 0; i < concurrency; i++ {
-		wg.Add(1)
-		go worker(ctx, i, wg, toCopy)
-	}
-
-	// Handle includes in spec. If specified they filter what gets in the package
-
-	if len(includes) > 0 && len(excludes) == 0 {
-		includeRegexp := compileRegexes(includes)
-		for _, l := range layers {
-			// Consider d.Additions (and d.Changes? - warn at least) only
-		ADDS:
-			for _, a := range l.Diffs.Additions {
-				for _, i := range includeRegexp {
-					if i.MatchString(a.Name) {
-						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
-						continue ADDS
-					}
-				}
-			}
-			for _, a := range l.Diffs.Changes {
-				ctx.Debug("File ", a.Name, " changed")
-			}
-			for _, a := range l.Diffs.Deletions {
-				ctx.Debug("File ", a.Name, " deleted")
-			}
-		}
-
-	} else if len(includes) == 0 && len(excludes) != 0 {
-		excludeRegexp := compileRegexes(excludes)
-		for _, l := range layers {
-			// Consider d.Additions (and d.Changes? - warn at least) only
-		ADD:
-			for _, a := range l.Diffs.Additions {
-				for _, i := range excludeRegexp {
-					if i.MatchString(a.Name) {
-						continue ADD
-					}
-				}
-				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
-			}
-			for _, a := range l.Diffs.Changes {
-				ctx.Debug("File ", a.Name, " changed")
-			}
-			for _, a := range l.Diffs.Deletions {
-				ctx.Debug("File ", a.Name, " deleted")
-			}
-		}
-
-	} else if len(includes) != 0 && len(excludes) != 0 {
-		includeRegexp := compileRegexes(includes)
-		excludeRegexp := compileRegexes(excludes)
-
-		for _, l := range layers {
-			// Consider d.Additions (and d.Changes? - warn at least) only
-		EXCLUDES:
-			for _, a := range l.Diffs.Additions {
-				for _, i := range includeRegexp {
-					if i.MatchString(a.Name) {
-						for _, e := range excludeRegexp {
-							if e.MatchString(a.Name) {
-								continue EXCLUDES
-							}
-						}
-						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
-						continue EXCLUDES
-					}
-				}
-			}
-			for _, a := range l.Diffs.Changes {
-				ctx.Debug("File ", a.Name, " changed")
-			}
-			for _, a := range l.Diffs.Deletions {
-				ctx.Debug("File ", a.Name, " deleted")
-			}
-		}
-
-	} else {
-		// Otherwise just grab all
-		for _, l := range layers {
-			// Consider d.Additions (and d.Changes? - warn at least) only
-			for _, a := range l.Diffs.Additions {
-				ctx.Debug("File ", a.Name, " added")
-				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
-			}
-			for _, a := range l.Diffs.Changes {
-				ctx.Debug("File ", a.Name, " changed")
-			}
-			for _, a := range l.Diffs.Deletions {
-				ctx.Debug("File ", a.Name, " deleted")
-			}
-		}
-	}
-
-	close(toCopy)
-	wg.Wait()
-
-	a := NewPackageArtifact(dst)
-	a.CompressionType = t
-	err = a.Compress(archive, concurrency)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating package archive")
-	}
-	return a, nil
-}

pkg/api/core/types/artifact/artifact_test.go

@@ -20,9 +20,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types"
 	. "github.com/mudler/luet/pkg/api/core/types/artifact"
-	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
@@ -30,7 +30,6 @@ import (
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 
 	. "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
@@ -117,53 +116,7 @@ RUN echo bar > /test2`))
 			}
 			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
-			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
-			diffs, err := compiler.GenerateChanges(ctx, b, opts, opts2)
-			Expect(err).ToNot(HaveOccurred())
 
-			artifacts := []ArtifactNode{{
-				Name: "/luetbuild/LuetDockerfile",
-				Size: 175,
-			}}
-			if os.Getenv("DOCKER_BUILDKIT") == "1" {
-				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
-			}
-			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
-			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
-
-			Expect(diffs).To(Equal(
-				[]ArtifactLayer{{
-					FromImage: "luet/base",
-					ToImage:   "test",
-					Diffs: ArtifactDiffs{
-						Additions: artifacts,
-					},
-				}}))
-			err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false)
-			Expect(err).ToNot(HaveOccurred())
-
-			a, err := ExtractArtifactFromDelta(ctx, rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(fileHelper.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
-			err = helpers.Untar(a.Path, unpacked, false)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(fileHelper.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
-			Expect(fileHelper.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
-			content1, err := fileHelper.Read(filepath.Join(unpacked, "test"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(content1).To(Equal("foo\n"))
-			content2, err := fileHelper.Read(filepath.Join(unpacked, "test2"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(content2).To(Equal("bar\n"))
-
-			err = a.Hash()
-			Expect(err).ToNot(HaveOccurred())
-			err = a.Verify()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(fileHelper.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
-
-			err = a.Verify()
-			Expect(err).To(HaveOccurred())
 		})
 
 		It("Generates packages images", func() {
@@ -193,9 +146,8 @@ RUN echo bar > /test2`))
 			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false)
+			err = a.GenerateFinalImage(ctx, resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(opts.ImageName).To(Equal(resultingImage))
 
 			Expect(b.ImageExists(resultingImage)).To(BeTrue())
 
@@ -203,7 +155,14 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(result) // clean up
 
-			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
+			img, err := b.ImageReference(resultingImage, true)
+			Expect(err).ToNot(HaveOccurred())
+			_, _, err = image.ExtractTo(
+				ctx,
+				img,
+				result,
+				nil,
+			)
 			Expect(err).ToNot(HaveOccurred())
 
 			content, err := ioutil.ReadFile(filepath.Join(result, "test"))
@@ -235,9 +194,8 @@ RUN echo bar > /test2`))
 			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false)
+			err = a.GenerateFinalImage(ctx, resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(opts.ImageName).To(Equal(resultingImage))
 
 			Expect(b.ImageExists(resultingImage)).To(BeTrue())
 
@@ -245,14 +203,17 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(result) // clean up
 
-			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
+			img, err := b.ImageReference(resultingImage, false)
+			Expect(err).ToNot(HaveOccurred())
+			_, _, err = image.ExtractTo(
+				ctx,
+				img,
+				result,
+				nil,
+			)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(fileHelper.DirectoryIsEmpty(result)).To(BeFalse())
-			content, err := ioutil.ReadFile(filepath.Join(result, ".virtual"))
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(string(content)).To(Equal(""))
+			Expect(fileHelper.DirectoryIsEmpty(result)).To(BeTrue())
 		})
 
 		It("Retrieves uncompressed name", func() {

pkg/api/core/types/config.go

@@ -60,6 +60,8 @@ type LuetGeneralConfig struct {
 	Debug           bool `yaml:"debug,omitempty" mapstructure:"debug"`
 	ShowBuildOutput bool `yaml:"show_build_output,omitempty" mapstructure:"show_build_output"`
 	FatalWarns      bool `yaml:"fatal_warnings,omitempty" mapstructure:"fatal_warnings"`
+	HTTPTimeout     int  `yaml:"http_timeout,omitempty" mapstructure:"http_timeout"`
+	Quiet           bool `yaml:"quiet" mapstructure:"quiet"`
 }
 
 type LuetSolverOptions struct {
@@ -136,7 +138,7 @@ func (sc *LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
 	return dbpath
 }
 
-func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (p string) {
 	var cachepath string
 	if sc.PkgsCachePath != "" {
 		cachepath = sc.PkgsCachePath
@@ -146,11 +148,13 @@ func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
 	}
 
 	if filepath.IsAbs(cachepath) {
-		ans = cachepath
+		p = cachepath
 	} else {
-		ans = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
+		p = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
 	}
 
+	sc.PkgsCachePath = cachepath // Be consistent with the path we set
+
 	return
 }
 

pkg/api/core/types/context.go

@@ -48,11 +48,11 @@ type Context struct {
 	Config     *LuetConfig
 	IsTerminal bool
 	NoSpinner  bool
+	name       string
 
 	s           *pterm.SpinnerPrinter
 	spinnerLock *sync.Mutex
 	z           *zap.Logger
-	AreaPrinter *pterm.AreaPrinter
 	ProgressBar *pterm.ProgressbarPrinter
 }
 
@@ -73,6 +73,12 @@ func NewContext() *Context {
 	}
 }
 
+func (c *Context) WithName(name string) *Context {
+	newc := c.Copy()
+	newc.name = name
+	return newc
+}
+
 func (c *Context) Copy() *Context {
 
 	configCopy := *c.Config
@@ -113,6 +119,11 @@ func (c *Context) Init() (err error) {
 		c.NoColor()
 	}
 
+	if c.Config.General.Quiet {
+		c.NoColor()
+		pterm.DisableStyling()
+	}
+
 	c.Debug("Colors", c.Config.GetLogging().Color)
 	c.Debug("Logging level", c.Config.GetLogging().Level)
 	c.Debug("Debug mode", c.Config.GetGeneral().Debug)
@@ -288,7 +299,9 @@ func (c *Context) Msg(level LogLevel, ln bool, msg ...interface{}) {
 		switch level {
 		case WarningLevel:
 			levelMsg = pterm.LightYellow(":construction: warning" + message)
-		case InfoLevel, SuccessLevel:
+		case InfoLevel:
+			levelMsg = message
+		case SuccessLevel:
 			levelMsg = pterm.LightGreen(message)
 		case ErrorLevel:
 			levelMsg = pterm.Red(message)
@@ -305,6 +318,10 @@ func (c *Context) Msg(level LogLevel, ln bool, msg ...interface{}) {
 		levelMsg = re.ReplaceAllString(levelMsg, "")
 	}
 
+	if c.name != "" {
+		levelMsg = fmt.Sprintf("[%s] %s", c.name, levelMsg)
+	}
+
 	if c.z != nil {
 		c.log2File(level, message)
 	}

pkg/api/core/types/context_test.go

@@ -0,0 +1,125 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types_test
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/gookit/color"
+	types "github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func captureStdout(f func(w io.Writer)) string {
+	originalStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+	color.SetOutput(w)
+	f(w)
+
+	_ = w.Close()
+	out, _ := ioutil.ReadAll(r)
+	os.Stdout = originalStdout
+	color.SetOutput(os.Stdout)
+
+	_ = r.Close()
+
+	return string(out)
+}
+
+var _ = Describe("Context and logging", func() {
+	ctx := types.NewContext()
+
+	BeforeEach(func() {
+		ctx = types.NewContext()
+	})
+
+	Context("LogLevel", func() {
+		It("converts it correctly to number and zaplog", func() {
+			Expect(types.ErrorLevel.ToNumber()).To(Equal(0))
+			Expect(types.InfoLevel.ToNumber()).To(Equal(2))
+			Expect(types.WarningLevel.ToNumber()).To(Equal(1))
+			Expect(types.LogLevel("foo").ToNumber()).To(Equal(3))
+			Expect(types.WarningLevel.ZapLevel().String()).To(Equal("warn"))
+			Expect(types.InfoLevel.ZapLevel().String()).To(Equal("info"))
+			Expect(types.ErrorLevel.ZapLevel().String()).To(Equal("error"))
+			Expect(types.FatalLevel.ZapLevel().String()).To(Equal("fatal"))
+			Expect(types.LogLevel("foo").ZapLevel().String()).To(Equal("debug"))
+		})
+	})
+
+	Context("Context", func() {
+		It("detect if is a terminal", func() {
+			Expect(captureStdout(func(w io.Writer) {
+				_, _, err := ctx.GetTerminalSize()
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(Equal("size not detectable"))
+				os.Stdout.Write([]byte(err.Error()))
+			})).To(ContainSubstring("size not detectable"))
+		})
+
+		It("respects loglevel", func() {
+			ctx.Config.GetGeneral().Debug = false
+			Expect(captureStdout(func(w io.Writer) {
+				ctx.Debug("")
+			})).To(Equal(""))
+
+			ctx.Config.GetGeneral().Debug = true
+			Expect(captureStdout(func(w io.Writer) {
+				ctx.Debug("foo")
+			})).To(ContainSubstring("foo"))
+		})
+
+		It("logs to file", func() {
+			ctx.NoColor()
+
+			t, err := ioutil.TempFile("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(t.Name()) // clean up
+			ctx.Config.GetLogging().EnableLogFile = true
+			ctx.Config.GetLogging().Path = t.Name()
+
+			ctx.Init()
+
+			Expect(captureStdout(func(w io.Writer) {
+				ctx.Info("foot")
+			})).To(And(ContainSubstring("INFO"), ContainSubstring("foot")))
+
+			Expect(captureStdout(func(w io.Writer) {
+				ctx.Success("test")
+			})).To(And(ContainSubstring("SUCCESS"), ContainSubstring("test")))
+
+			Expect(captureStdout(func(w io.Writer) {
+				ctx.Error("foobar")
+			})).To(And(ContainSubstring("ERROR"), ContainSubstring("foobar")))
+
+			Expect(captureStdout(func(w io.Writer) {
+				ctx.Warning("foowarn")
+			})).To(And(ContainSubstring("WARNING"), ContainSubstring("foowarn")))
+
+			l, err := ioutil.ReadFile(t.Name())
+			Expect(err).ToNot(HaveOccurred())
+			logs := string(l)
+			Expect(logs).To(ContainSubstring("foot"))
+			Expect(logs).To(ContainSubstring("test"))
+			Expect(logs).To(ContainSubstring("foowarn"))
+			Expect(logs).To(ContainSubstring("foobar"))
+		})
+	})
+})

pkg/compiler/backend.go

@@ -1,14 +1,8 @@
 package compiler
 
 import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/mudler/luet/pkg/api/core/types"
-	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/pkg/errors"
@@ -32,9 +26,9 @@ func NewBackend(ctx *types.Context, s string) (CompilerBackend, error) {
 type CompilerBackend interface {
 	BuildImage(backend.Options) error
 	ExportImage(backend.Options) error
+	LoadImage(string) error
 	RemoveImage(backend.Options) error
 	ImageDefinitionToTar(backend.Options) error
-	ExtractRootfs(opts backend.Options, keepPerms bool) error
 
 	CopyImage(string, string) error
 	DownloadImage(opts backend.Options) error
@@ -42,204 +36,6 @@ type CompilerBackend interface {
 	Push(opts backend.Options) error
 	ImageAvailable(string) bool
 
+	ImageReference(img1 string, ondisk bool) (v1.Image, error)
 	ImageExists(string) bool
 }
-
-// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
-// example of json return: [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-func GenerateChanges(ctx *types.Context, b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) {
-
-	res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
-
-	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-
-	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(srcRootFS) // clean up
-
-	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(dstRootFS) // clean up
-
-	srcImageExtract := backend.Options{
-		ImageName:   fromImage.ImageName,
-		Destination: srcRootFS,
-	}
-	ctx.Debug("Extracting source image", fromImage.ImageName)
-	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
-	}
-
-	dstImageExtract := backend.Options{
-		ImageName:   toImage.ImageName,
-		Destination: dstRootFS,
-	}
-	ctx.Debug("Extracting destination image", toImage.ImageName)
-	err = b.ExtractRootfs(dstImageExtract, false)
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
-	}
-
-	// Get Additions/Changes. dst -> src
-	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
-		if info.IsDir() {
-			return nil
-		}
-
-		realpath := strings.Replace(path, dstRootFS, "", -1)
-		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
-		if err == nil {
-			var sizeA, sizeB int64
-			sizeA = fileInfo.Size()
-
-			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
-				sizeB = s.Size()
-			}
-
-			if sizeA != sizeB {
-				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
-				res.Diffs.Changes = append(res.Diffs.Changes, artifact.ArtifactNode{
-					Name: filepath.Join("/", realpath),
-					Size: int(sizeB),
-				})
-			} else {
-				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
-			}
-		} else {
-			var sizeB int64
-
-			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
-				sizeB = s.Size()
-			}
-			res.Diffs.Additions = append(res.Diffs.Additions, artifact.ArtifactNode{
-				Name: filepath.Join("/", realpath),
-				Size: int(sizeB),
-			})
-
-			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
-		}
-
-		return nil
-	})
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
-	}
-
-	// Get deletions. src -> dst
-	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
-		if info.IsDir() {
-			return nil
-		}
-
-		realpath := strings.Replace(path, srcRootFS, "", -1)
-		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
-			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
-			res.Diffs.Deletions = append(res.Diffs.Deletions, artifact.ArtifactNode{
-				Name: filepath.Join("/", realpath),
-			})
-		}
-
-		return nil
-	})
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
-	}
-
-	diffs := []artifact.ArtifactLayer{res}
-
-	if ctx.Config.GetGeneral().Debug {
-		summary := ComputeArtifactLayerSummary(diffs)
-		for _, l := range summary.Layers {
-			ctx.Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
-				l.FromImage, l.ToImage,
-				l.AddFiles, l.AddSizes,
-				l.DelFiles, l.DelSizes,
-				l.ChangeFiles, l.ChangeSizes))
-		}
-	}
-
-	return diffs, nil
-}
-
-type ArtifactLayerSummary struct {
-	FromImage   string `json:"image1"`
-	ToImage     string `json:"image2"`
-	AddFiles    int    `json:"add_files"`
-	AddSizes    int64  `json:"add_sizes"`
-	DelFiles    int    `json:"del_files"`
-	DelSizes    int64  `json:"del_sizes"`
-	ChangeFiles int    `json:"change_files"`
-	ChangeSizes int64  `json:"change_sizes"`
-}
-
-type ArtifactLayersSummary struct {
-	Layers []ArtifactLayerSummary `json:"summary"`
-}
-
-func ComputeArtifactLayerSummary(diffs []artifact.ArtifactLayer) ArtifactLayersSummary {
-
-	ans := ArtifactLayersSummary{
-		Layers: make([]ArtifactLayerSummary, 0),
-	}
-
-	for _, layer := range diffs {
-		sum := ArtifactLayerSummary{
-			FromImage:   layer.FromImage,
-			ToImage:     layer.ToImage,
-			AddFiles:    0,
-			AddSizes:    0,
-			DelFiles:    0,
-			DelSizes:    0,
-			ChangeFiles: 0,
-			ChangeSizes: 0,
-		}
-		for _, a := range layer.Diffs.Additions {
-			sum.AddFiles++
-			sum.AddSizes += int64(a.Size)
-		}
-		for _, d := range layer.Diffs.Deletions {
-			sum.DelFiles++
-			sum.DelSizes += int64(d.Size)
-		}
-		for _, c := range layer.Diffs.Changes {
-			sum.ChangeFiles++
-			sum.ChangeSizes += int64(c.Size)
-		}
-		ans.Layers = append(ans.Layers, sum)
-	}
-
-	return ans
-}

pkg/compiler/backend/simpledocker.go

@@ -1,4 +1,4 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@gentoo.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -16,20 +16,17 @@
 package backend
 
 import (
-	"encoding/json"
-	"io/ioutil"
-	"os"
+	"io"
 	"os/exec"
-	"path/filepath"
-	"strings"
 
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/daemon"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	bus "github.com/mudler/luet/pkg/api/core/bus"
 	"github.com/mudler/luet/pkg/api/core/types"
-	bus "github.com/mudler/luet/pkg/bus"
-	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
-	capi "github.com/mudler/docker-companion/api"
-
-	"github.com/mudler/luet/pkg/helpers"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 
 	"github.com/pkg/errors"
 )
@@ -74,6 +71,17 @@ func (s *SimpleDocker) CopyImage(src, dst string) error {
 	return nil
 }
 
+func (s *SimpleDocker) LoadImage(path string) error {
+	s.ctx.Debug(":whale: Loading image:", path)
+	cmd := exec.Command("docker", "load", "-i", path)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed loading image: "+string(out))
+	}
+	s.ctx.Success(":whale: Loaded image:", path)
+	return nil
+}
+
 func (s *SimpleDocker) DownloadImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePull, opts)
@@ -144,6 +152,68 @@ func (s *SimpleDocker) Push(opts Options) error {
 	return nil
 }
 
+func (s *SimpleDocker) imagefromDaemon(a string) (v1.Image, error) {
+	ref, err := name.ParseReference(a)
+	if err != nil {
+		return nil, err
+	}
+	img, err := daemon.Image(ref, daemon.WithUnbufferedOpener())
+	if err != nil {
+		return nil, err
+	}
+	return img, nil
+}
+
+// TODO: Make it possible optionally to use this?
+// It might be unsafer, as it relies on the pipe.
+// imageFromCLIPipe returns a new image from a tarball by providing a reader from the docker stdout pipe.
+// See also daemon.Image implementation below for an example (which returns the tarball stream
+// from the HTTP api endpoint instead ).
+func (s *SimpleDocker) imageFromCLIPipe(a string) (v1.Image, error) {
+	return tarball.Image(func() (io.ReadCloser, error) {
+		buildarg := []string{"save", a}
+		s.ctx.Spinner()
+		defer s.ctx.SpinnerStop()
+		c := exec.Command("docker", buildarg...)
+		p, err := c.StdoutPipe()
+		if err != nil {
+			return nil, err
+		}
+		err = c.Start()
+		if err != nil {
+			return nil, err
+		}
+
+		go func() { c.Wait() }()
+		return p, nil
+	}, nil)
+}
+
+func (s *SimpleDocker) imageFromDisk(a string) (v1.Image, error) {
+	f, err := s.ctx.Config.GetSystem().TempFile("snapshot")
+	if err != nil {
+		return nil, err
+	}
+	buildarg := []string{"save", a, "-o", f.Name()}
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
+
+	out, err := exec.Command("docker", buildarg...).CombinedOutput()
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed saving image: "+string(out))
+	}
+
+	return crane.Load(f.Name())
+}
+
+func (s *SimpleDocker) ImageReference(a string, ondisk bool) (v1.Image, error) {
+	if ondisk {
+		return s.imageFromDisk(a)
+	}
+
+	return s.imagefromDaemon(a)
+}
+
 func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
@@ -179,102 +249,3 @@ func (s *SimpleDocker) ExportImage(opts Options) error {
 type ManifestEntry struct {
 	Layers []string `json:"Layers"`
 }
-
-func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
-	name := opts.ImageName
-	dst := opts.Destination
-
-	if !b.ImageExists(name) {
-		if err := b.DownloadImage(opts); err != nil {
-			return errors.Wrap(err, "failed pulling image "+name+" during extraction")
-		}
-	}
-
-	tempexport, err := ioutil.TempDir(dst, "tmprootfs")
-	if err != nil {
-		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tempexport) // clean up
-
-	imageExport := filepath.Join(tempexport, "image.tar")
-
-	b.ctx.Spinner()
-	defer b.ctx.SpinnerStop()
-
-	if err := b.ExportImage(Options{ImageName: name, Destination: imageExport}); err != nil {
-		return errors.Wrap(err, "failed while extracting rootfs for "+name)
-	}
-
-	src := imageExport
-
-	if src == "" && opts.ImageName != "" {
-		tempUnpack, err := ioutil.TempDir(dst, "tempUnpack")
-		if err != nil {
-			return errors.Wrap(err, "Error met while creating tempdir for rootfs")
-		}
-		defer os.RemoveAll(tempUnpack) // clean up
-		imageExport := filepath.Join(tempUnpack, "image.tar")
-		if err := b.ExportImage(Options{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
-			return errors.Wrap(err, "while exporting image before extraction")
-		}
-		src = imageExport
-	}
-
-	rootfs, err := ioutil.TempDir(dst, "tmprootfs")
-	if err != nil {
-		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(rootfs) // clean up
-
-	// TODO: Following as option if archive as output?
-	// archive, err := ioutil.TempDir(os.TempDir(), "archive")
-	// if err != nil {
-	// 	return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	// }
-	// defer os.RemoveAll(archive) // clean up
-
-	err = helpers.Untar(src, rootfs, keepPerms)
-	if err != nil {
-		return errors.Wrap(err, "Error met while unpacking rootfs")
-	}
-
-	manifest, err := fileHelper.Read(filepath.Join(rootfs, "manifest.json"))
-	if err != nil {
-		return errors.Wrap(err, "Error met while reading image manifest")
-	}
-
-	// Unpack all layers
-	var manifestData []ManifestEntry
-
-	if err := json.Unmarshal([]byte(manifest), &manifestData); err != nil {
-		return errors.Wrap(err, "Error met while unmarshalling manifest")
-	}
-
-	layers_sha := []string{}
-
-	for _, data := range manifestData {
-
-		for _, l := range data.Layers {
-			if strings.Contains(l, "layer.tar") {
-				layers_sha = append(layers_sha, strings.Replace(l, "/layer.tar", "", -1))
-			}
-		}
-	}
-	// TODO: Drop capi in favor of the img approach already used in pkg/installer/repository
-	export, err := capi.CreateExport(rootfs)
-	if err != nil {
-		return err
-	}
-
-	err = export.UnPackLayers(layers_sha, dst, "containerd")
-	if err != nil {
-		return err
-	}
-
-	// err = helpers.Tar(archive, dst)
-	// if err != nil {
-	// 	return nil, errors.Wrap(err, "Error met while creating package archive")
-	// }
-
-	return nil
-}

pkg/compiler/backend/simpledocker_test.go

@@ -17,8 +17,6 @@ package backend_test
 
 import (
 	"github.com/mudler/luet/pkg/api/core/types"
-	"github.com/mudler/luet/pkg/api/core/types/artifact"
-	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
@@ -107,35 +105,6 @@ RUN echo bar > /test2`))
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
-			artifacts := []artifact.ArtifactNode{{
-				Name: "/luetbuild/LuetDockerfile",
-				Size: 175,
-			}}
-			if os.Getenv("DOCKER_BUILDKIT") == "1" {
-				artifacts = append(artifacts, artifact.ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
-			}
-			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4})
-			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4})
-
-			Expect(compiler.GenerateChanges(ctx, b, opts, opts2)).To(Equal(
-				[]artifact.ArtifactLayer{{
-					FromImage: "luet/base",
-					ToImage:   "test",
-					Diffs: artifact.ArtifactDiffs{
-						Additions: artifacts,
-					},
-				}}))
-
-			opts2 = backend.Options{
-				ImageName:      "test",
-				SourcePath:     tmpdir,
-				DockerFileName: "LuetDockerfile",
-				Destination:    filepath.Join(tmpdir, "output3.tar"),
-			}
-
-			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
-			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
-			Expect(b.ImageExists(opts2.ImageName)).To(BeFalse())
 		})
 
 		It("Detects available images", func() {

pkg/compiler/backend/simpleimg.go

@@ -16,12 +16,13 @@
 package backend
 
 import (
-	"os"
 	"os/exec"
 	"strings"
 
+	"github.com/google/go-containerregistry/pkg/crane"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	bus "github.com/mudler/luet/pkg/api/core/bus"
 	"github.com/mudler/luet/pkg/api/core/types"
-	bus "github.com/mudler/luet/pkg/bus"
 
 	"github.com/pkg/errors"
 )
@@ -34,6 +35,10 @@ func NewSimpleImgBackend(ctx *types.Context) *SimpleImg {
 	return &SimpleImg{ctx: ctx}
 }
 
+func (s *SimpleImg) LoadImage(string) error {
+	return errors.New("Not supported")
+}
+
 // TODO: Missing still: labels, and build args expansion
 func (s *SimpleImg) BuildImage(opts Options) error {
 	name := opts.ImageName
@@ -70,6 +75,29 @@ func (s *SimpleImg) RemoveImage(opts Options) error {
 	return nil
 }
 
+func (s *SimpleImg) ImageReference(a string, ondisk bool) (v1.Image, error) {
+
+	f, err := s.ctx.Config.GetSystem().TempFile("snapshot")
+	if err != nil {
+		return nil, err
+	}
+	buildarg := []string{"save", a, "-o", f.Name()}
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
+
+	out, err := exec.Command("img", buildarg...).CombinedOutput()
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed saving image: "+string(out))
+	}
+
+	img, err := crane.Load(f.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	return img, nil
+}
+
 func (s *SimpleImg) DownloadImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePull, opts)
@@ -153,33 +181,6 @@ func (s *SimpleImg) ExportImage(opts Options) error {
 	return nil
 }
 
-// ExtractRootfs extracts the docker image content inside the destination
-func (s *SimpleImg) ExtractRootfs(opts Options, keepPerms bool) error {
-	name := opts.ImageName
-	path := opts.Destination
-
-	if !s.ImageExists(name) {
-		if err := s.DownloadImage(opts); err != nil {
-			return errors.Wrap(err, "failed pulling image "+name+" during extraction")
-		}
-	}
-
-	os.RemoveAll(path)
-
-	buildarg := []string{"unpack", "-o", path, name}
-	s.ctx.Debug(":tea: Extracting image " + name)
-
-	s.ctx.Spinner()
-	defer s.ctx.SpinnerStop()
-
-	out, err := exec.Command("img", buildarg...).CombinedOutput()
-	if err != nil {
-		return errors.Wrap(err, "Failed extracting image: "+string(out))
-	}
-	s.ctx.Debug(":tea: Image " + name + " extracted")
-	return nil
-}
-
 func (s *SimpleImg) Push(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePush, opts)

pkg/compiler/backend_test.go

@@ -1,79 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler_test
-
-import (
-	"github.com/mudler/luet/pkg/api/core/types"
-	. "github.com/mudler/luet/pkg/compiler"
-	. "github.com/mudler/luet/pkg/compiler/backend"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Docker image diffs", func() {
-	var b CompilerBackend
-	ctx := types.NewContext()
-	BeforeEach(func() {
-		b = NewSimpleDockerBackend(ctx)
-	})
-
-	Context("Generate diffs from docker images", func() {
-		It("Detect no changes", func() {
-			opts := Options{
-				ImageName: "alpine:latest",
-			}
-			err := b.DownloadImage(opts)
-			Expect(err).ToNot(HaveOccurred())
-
-			layers, err := GenerateChanges(ctx, b, opts, opts)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(layers)).To(Equal(1))
-			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
-			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
-			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
-		})
-
-		It("Detects additions and changed files", func() {
-			err := b.DownloadImage(Options{
-				ImageName: "quay.io/mocaccino/micro",
-			})
-			Expect(err).ToNot(HaveOccurred())
-			err = b.DownloadImage(Options{
-				ImageName: "quay.io/mocaccino/extra",
-			})
-			Expect(err).ToNot(HaveOccurred())
-
-			layers, err := GenerateChanges(ctx, b, Options{
-				ImageName: "quay.io/mocaccino/micro",
-			}, Options{
-				ImageName: "quay.io/mocaccino/extra",
-			})
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(layers)).To(Equal(1))
-
-			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
-			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
-			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
-
-			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
-			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
-			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
-
-			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
-		})
-	})
-})

pkg/compiler/compiler.go

@@ -29,9 +29,10 @@ import (
 	"sync"
 	"time"
 
+	bus "github.com/mudler/luet/pkg/api/core/bus"
+	"github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
-	bus "github.com/mudler/luet/pkg/bus"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
@@ -227,35 +228,42 @@ func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, includ
 
 func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
 
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	if !cs.Backend.ImageExists(runnerOpts.ImageName) {
+		if err := cs.Backend.DownloadImage(runnerOpts); err != nil {
+			return nil, errors.Wrap(err, "failed pulling image "+runnerOpts.ImageName+" during extraction")
+		}
+	}
+
+	img, err := cs.Backend.ImageReference(runnerOpts.ImageName, true)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not create tempdir")
+		return nil, err
+	}
+
+	_, rootfs, err := image.Extract(
+		cs.Options.Context,
+		img,
+		image.ExtractFiles(
+			cs.Options.Context,
+			p.GetPackageDir(),
+			p.GetIncludes(),
+			p.GetExcludes(),
+		),
+	)
+	if err != nil {
+		return nil, err
 	}
 	defer os.RemoveAll(rootfs) // clean up
 
-	err = cs.Backend.ExtractRootfs(backend.Options{
-		ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not extract rootfs")
+	toUnpack := rootfs
+
+	if p.PackageDir != "" {
+		toUnpack = filepath.Join(toUnpack, p.PackageDir)
 	}
 
-	if p.GetPackageDir() != "" {
-		cs.Options.Context.Info(":tophat: Packing from output dir", p.GetPackageDir())
-		rootfs = filepath.Join(rootfs, p.GetPackageDir())
-	}
-
-	if len(p.GetIncludes()) > 0 {
-		// strip from includes
-		cs.stripFromRootfs(p.GetIncludes(), rootfs, true)
-	}
-	if len(p.GetExcludes()) > 0 {
-		// strip from excludes
-		cs.stripFromRootfs(p.GetExcludes(), rootfs, false)
-	}
 	a := artifact.NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 	a.CompressionType = cs.Options.CompressionType
 
-	if err := a.Compress(rootfs, concurrency); err != nil {
+	if err := a.Compress(toUnpack, concurrency); err != nil {
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}
 
@@ -265,38 +273,65 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
 
 func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, builderOpts, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
 
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	rootfs, err := cs.Options.Context.Config.System.TempDir("rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
 	}
-	defer os.RemoveAll(rootfs) // clean up
+	defer os.RemoveAll(rootfs)
 
 	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
-	if cs.Options.PullFirst && !cs.Backend.ImageExists(builderOpts.ImageName) && cs.Backend.ImageAvailable(builderOpts.ImageName) {
-		err := cs.Backend.DownloadImage(builderOpts)
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not pull image")
+	if cs.Options.PullFirst {
+		if !cs.Backend.ImageExists(builderOpts.ImageName) {
+			err := cs.Backend.DownloadImage(builderOpts)
+			if err != nil {
+				return nil, errors.Wrap(err, "Could not pull image")
+			}
+		}
+		if !cs.Backend.ImageExists(runnerOpts.ImageName) {
+			err := cs.Backend.DownloadImage(runnerOpts)
+			if err != nil {
+				return nil, errors.Wrap(err, "Could not pull image")
+			}
 		}
 	}
 
 	cs.Options.Context.Info(pkgTag, ":hammer: Generating delta")
-	diffs, err := GenerateChanges(cs.Options.Context, cs.Backend, builderOpts, runnerOpts)
+
+	cs.Options.Context.Debug(pkgTag, ":hammer: Retrieving reference for", builderOpts.ImageName)
+
+	ref, err := cs.Backend.ImageReference(builderOpts.ImageName, true)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not generate changes from layers")
+		return nil, err
 	}
 
-	cs.Options.Context.Debug("Extracting image to grab files from delta")
-	if err := cs.Backend.ExtractRootfs(backend.Options{
-		ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions); err != nil {
-		return nil, errors.Wrap(err, "Could not extract rootfs")
-	}
-	artifact, err := artifact.ExtractArtifactFromDelta(cs.Options.Context, rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.Options.CompressionType)
+	cs.Options.Context.Debug(pkgTag, ":hammer: Retrieving reference for", runnerOpts.ImageName)
+
+	ref2, err := cs.Backend.ImageReference(runnerOpts.ImageName, true)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not generate deltas")
+		return nil, err
 	}
 
-	artifact.CompileSpec = p
-	return artifact, nil
+	cs.Options.Context.Debug(pkgTag, ":hammer: Generating filters for extraction")
+
+	filter, err := image.ExtractDeltaAdditionsFiles(cs.Options.Context, ref, p.GetIncludes(), p.GetExcludes())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed generating filter for extraction")
+	}
+
+	cs.Options.Context.Info(pkgTag, ":hammer: Extracting artifact from image", runnerOpts.ImageName)
+	a, err := artifact.ImageToArtifact(
+		cs.Options.Context,
+		ref2,
+		cs.Options.CompressionType,
+		p.Rel(fmt.Sprintf("%s%s", p.GetPackage().GetFingerPrint(), ".package.tar")),
+		filter,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	a.CompileSpec = p
+	return a, nil
 }
 
 func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
@@ -315,15 +350,11 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
 	// and we build all the images first.
 
-	err := os.MkdirAll(p.Rel("build"), os.ModePerm)
+	buildDir, err := cs.Options.Context.Config.System.TempDir("build")
 	if err != nil {
-		return builderOpts, runnerOpts, errors.Wrap(err, "Error met while creating tempdir for building")
+		return builderOpts, runnerOpts, err
 	}
-	buildDir, err := ioutil.TempDir(p.Rel("build"), "pack")
-	if err != nil {
-		return builderOpts, runnerOpts, errors.Wrap(err, "Error met while creating tempdir for building")
-	}
-	defer os.RemoveAll(buildDir) // clean up
+	defer os.RemoveAll(buildDir)
 
 	// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
 	err = fileHelper.CopyDir(p.GetPackage().GetPath(), buildDir)
@@ -340,7 +371,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	}
 
 	// First we create the builder image
-	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")); err != nil {
+	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().ImageID()+"-builder.dockerfile")); err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
 	}
 
@@ -355,21 +386,21 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	// steps in prelude are == 0 those are equivalent.
 
 	// Then we write the step image, which uses the builder one
-	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")); err != nil {
+	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().ImageID()+".dockerfile")); err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
 	}
 
 	builderOpts = backend.Options{
 		ImageName:      buildertaggedImage,
 		SourcePath:     buildDir,
-		DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
+		DockerFileName: p.GetPackage().ImageID() + "-builder.dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 		BackendArgs:    cs.Options.BackendArgs,
 	}
 	runnerOpts = backend.Options{
 		ImageName:      packageImage,
 		SourcePath:     buildDir,
-		DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
+		DockerFileName: p.GetPackage().ImageID() + ".dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 		BackendArgs:    cs.Options.BackendArgs,
 	}
@@ -427,11 +458,11 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	if p.EmptyPackage() {
 		fakePackage := p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")
 
-		rootfs, err = ioutil.TempDir(p.GetOutputPath(), "rootfs")
+		rootfs, err = cs.Options.Context.Config.System.TempDir("rootfs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not create tempdir")
 		}
-		defer os.RemoveAll(rootfs) // clean up
+		defer os.RemoveAll(rootfs)
 
 		a := artifact.NewPackageArtifact(fakePackage)
 		a.CompressionType = cs.Options.CompressionType
@@ -442,12 +473,16 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 
 		a.CompileSpec = p
 		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
-
 		err = a.WriteYAML(p.GetOutputPath())
 		if err != nil {
 			return a, errors.Wrap(err, "Failed while writing metadata file")
 		}
 		cs.Options.Context.Success(pkgTag, "   :white_check_mark: done (empty virtual package)")
+		if cs.Options.PushFinalImages {
+			if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil {
+				return nil, err
+			}
+		}
 		return a, nil
 	}
 
@@ -477,11 +512,55 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	if err != nil {
 		return a, errors.Wrap(err, "Failed while writing metadata file")
 	}
-	cs.Options.Context.Success(pkgTag, "   :white_check_mark: Done")
+	cs.Options.Context.Success(pkgTag, "   :white_check_mark: Done building")
+
+	if cs.Options.PushFinalImages {
+		if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil {
+			return nil, err
+		}
+	}
 
 	return a, nil
 }
 
+// TODO: A small readaptation of repository_docker.go pushImageFromArtifact()
+//       Move this to a common place
+func (cs *LuetCompiler) pushFinalArtifact(a *artifact.PackageArtifact, p *compilerspec.LuetCompilationSpec, keepPermissions bool) error {
+	cs.Options.Context.Info("Pushing final image for", a.CompileSpec.Package.HumanReadableString())
+	imageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, a.CompileSpec.Package.ImageID())
+
+	// First push the package image
+	if !cs.Backend.ImageAvailable(imageID) || cs.Options.PushFinalImagesForce {
+		cs.Options.Context.Info("Generating and pushing final image for", a.CompileSpec.Package.HumanReadableString(), "as", imageID)
+
+		if err := a.GenerateFinalImage(cs.Options.Context, imageID, cs.GetBackend(), true); err != nil {
+			return errors.Wrap(err, "while creating final image")
+		}
+		if err := cs.Backend.Push(backend.Options{ImageName: imageID}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", imageID)
+		}
+	}
+
+	// Then the image ID
+	metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, helpers.SanitizeImageString(a.CompileSpec.GetPackage().GetMetadataFilePath()))
+	if !cs.Backend.ImageAvailable(metadataImageID) || cs.Options.PushFinalImagesForce {
+		cs.Options.Context.Info("Generating metadata image for", a.CompileSpec.Package.HumanReadableString(), metadataImageID)
+
+		a := artifact.NewPackageArtifact(filepath.Join(p.GetOutputPath(), a.CompileSpec.GetPackage().GetMetadataFilePath()))
+		metadataArchive, err := artifact.CreateArtifactForFile(cs.Options.Context, a.Path)
+		if err != nil {
+			return errors.Wrap(err, "failed generating checksums for tree")
+		}
+		if err := metadataArchive.GenerateFinalImage(cs.Options.Context, metadataImageID, cs.Backend, keepPermissions); err != nil {
+			return errors.Wrap(err, "Failed generating metadata tree "+metadataImageID)
+		}
+		if err = cs.Backend.Push(backend.Options{ImageName: metadataImageID}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", metadataImageID)
+		}
+	}
+	return nil
+}
+
 func (cs *LuetCompiler) waitForImages(images []string) {
 	if cs.Options.PullFirst && cs.Options.Wait {
 		available, _ := oneOfImagesAvailable(images, cs.Backend)
@@ -860,11 +939,11 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 	}
 
 	// otherwise, generate it and push it aside
-	joinDir, err := ioutil.TempDir(p.GetOutputPath(), "join")
+	joinDir, err := cs.Options.Context.Config.System.TempDir("join")
 	if err != nil {
 		return errors.Wrap(err, "could not create tempdir for joining images")
 	}
-	defer os.RemoveAll(joinDir) // clean up
+	defer os.RemoveAll(joinDir)
 
 	for _, p := range fromPackages {
 		cs.Options.Context.Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
@@ -899,11 +978,11 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 		}
 	}
 
-	artifactDir, err := ioutil.TempDir(p.GetOutputPath(), "artifact")
+	artifactDir, err := cs.Options.Context.Config.System.TempDir("join")
 	if err != nil {
 		return errors.Wrap(err, "could not create tempdir for final artifact")
 	}
-	defer os.RemoveAll(joinDir) // clean up
+	defer os.RemoveAll(artifactDir)
 
 	cs.Options.Context.Info(joinTag, ":droplet: generating artifact for source image of", p.GetPackage().HumanReadableString())
 
@@ -916,14 +995,14 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 
 	joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp)
 	cs.Options.Context.Info(joinTag, ":droplet: generating image from artifact", joinImageName)
-	opts, err := a.GenerateFinalImage(cs.Options.Context, joinImageName, cs.Backend, keepPermissions)
+	err = a.GenerateFinalImage(cs.Options.Context, joinImageName, cs.Backend, keepPermissions)
 	if err != nil {
 		return errors.Wrap(err, "could not create final image")
 	}
 	if cs.Options.Push {
 		cs.Options.Context.Info(joinTag, ":droplet: pushing image from artifact", joinImageName)
-		if err = cs.Backend.Push(opts); err != nil {
-			return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
+		if err = cs.Backend.Push(backend.Options{ImageName: joinImageName}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", joinImageName)
 		}
 	}
 	cs.Options.Context.Info(joinTag, ":droplet: Consuming image", joinImageName)
@@ -974,6 +1053,36 @@ func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions
 	return nil
 }
 
+func CompilerFinalImages(cs *LuetCompiler) (*LuetCompiler, error) {
+	// When computing the hash tree, we need to take into consideration
+	// that packages that require final images have to be seen as packages without deps
+	// This is because we don't really want to calculate the deptree of them as
+	// as it is handled already when we are creating the images in resolveFinalImages().
+	c := *cs
+	copy := &c
+	memDB := pkg.NewInMemoryDatabase(false)
+	// Create a copy to avoid races
+	dbCopy := pkg.NewInMemoryDatabase(false)
+	err := cs.Database.Clone(dbCopy)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed cloning db")
+	}
+	for _, p := range dbCopy.World() {
+		copy := p.Clone()
+		spec, err := cs.FromPackage(p)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed getting compile spec for package "+p.HumanReadableString())
+		}
+		if spec.RequiresFinalImages {
+			copy.Requires([]*pkg.DefaultPackage{})
+		}
+
+		memDB.CreatePackage(copy)
+	}
+	copy.Database = memDB
+	return copy, nil
+}
+
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateFinalArtifact *bool, generateDependenciesFinalArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
 	cs.Options.Context.Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
@@ -997,8 +1106,11 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 	}
 
 	ht := NewHashTree(cs.Database)
-
-	packageHashTree, err := ht.Query(cs, p)
+	copy, err := CompilerFinalImages(cs)
+	if err != nil {
+		return nil, err
+	}
+	packageHashTree, err := ht.Query(copy, p)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed querying hashtree")
 	}
@@ -1056,6 +1168,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 	buildTarget := !cs.Options.OnlyDeps
 
 	if buildDeps {
+
 		cs.Options.Context.Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
 		for _, assertion := range dependencies { //highly dependent on the order
 			depsN++
@@ -1071,6 +1184,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
 			}
 			compileSpec.BuildOptions.PullImageRepository = append(compileSpec.BuildOptions.PullImageRepository, p.BuildOptions.PullImageRepository...)
+
 			cs.Options.Context.Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
 
 			compileSpec.SetOutputPath(p.GetOutputPath())
@@ -1227,11 +1341,12 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
 	} else {
 		bv := cs.Options.BuildValuesFile
 		if len(vals) > 0 {
-			valuesdir, err := ioutil.TempDir("", "genvalues")
+			valuesdir, err := cs.Options.Context.Config.System.TempDir("genvalues")
 			if err != nil {
 				return nil, errors.Wrap(err, "Could not create tempdir")
 			}
-			defer os.RemoveAll(valuesdir) // clean up
+			defer os.RemoveAll(valuesdir)
+
 			for _, b := range vals {
 				out, err := yaml.Marshal(b)
 				if err != nil {

pkg/compiler/compiler_test.go

@@ -16,16 +16,22 @@
 package compiler_test
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strings"
 
+	helpers "github.com/mudler/luet/tests/helpers"
+
+	"github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	. "github.com/mudler/luet/pkg/compiler"
 	sd "github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/compiler/types/compression"
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
@@ -64,7 +70,7 @@ var _ = Describe("Compiler", func() {
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -104,7 +110,7 @@ var _ = Describe("Compiler", func() {
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("result"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("bina/busybox"))).To(BeTrue())
@@ -134,7 +140,7 @@ var _ = Describe("Compiler", func() {
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(spec.Rel("newc"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
@@ -169,7 +175,7 @@ var _ = Describe("Compiler", func() {
 			Expect(errs).To(BeNil())
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 		})
@@ -232,7 +238,7 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
@@ -282,12 +288,12 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			for _, artifact := range artifacts2 {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			Expect(fileHelper.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
@@ -318,10 +324,9 @@ var _ = Describe("Compiler", func() {
 			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
-
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -355,7 +360,7 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -390,7 +395,7 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -424,7 +429,7 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(fileHelper.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
@@ -457,7 +462,7 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(fileHelper.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
@@ -490,7 +495,7 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 			Expect(fileHelper.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -528,9 +533,9 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.NewPackageArtifact(spec.Rel("extra-layer-0.1.package.tar")).Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("extra-layer"))).To(BeTrue())
 
@@ -569,9 +574,9 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.NewPackageArtifact(spec.Rel("c-test-1.0.package.tar")).Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("d"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("dd"))).To(BeTrue())
@@ -612,9 +617,9 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.NewPackageArtifact(spec.Rel("c-test-1.0.package.tar")).Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("d"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("dd"))).To(BeTrue())
@@ -652,9 +657,9 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.NewPackageArtifact(spec.Rel("extra-layer-0.1.package.tar")).Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("extra-layer"))).To(BeTrue())
 
@@ -714,9 +719,9 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts[0].Dependencies)).To(Equal(6))
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
-			Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.NewPackageArtifact(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar")).Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
@@ -749,7 +754,7 @@ var _ = Describe("Compiler", func() {
 
 			for _, artifact := range artifacts {
 				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+				Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			}
 
 			// A deps on B, so A artifacts are here:
@@ -804,13 +809,13 @@ var _ = Describe("Compiler", func() {
 
 			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
-			for _, artifact := range artifacts {
-				Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-				Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			for _, a := range artifacts {
+				Expect(fileHelper.Exists(a.Path)).To(BeTrue())
+				Expect(a.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
-				for _, d := range artifact.Dependencies {
+				for _, d := range a.Dependencies {
 					Expect(fileHelper.Exists(d.Path)).To(BeTrue())
-					Expect(helpers.Untar(d.Path, tmpdir, false)).ToNot(HaveOccurred())
+					Expect(artifact.NewPackageArtifact(d.Path).Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 				}
 			}
 
@@ -848,10 +853,72 @@ var _ = Describe("Compiler", func() {
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
-			Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(artifact.NewPackageArtifact(filepath.Join(tmpdir, "runtime-layer-0.1.package.tar")).Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
+
+		It("Pushes final images along", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			randString := strings.ToLower(helpers.String(10))
+			imageName := fmt.Sprintf("ttl.sh/%s", randString)
+			b := sd.NewSimpleDockerBackend(ctx)
+
+			err := generalRecipe.Load("../../tests/fixtures/packagelayers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(b, generalRecipe.GetDatabase(),
+				options.EnablePushFinalImages, options.ForcePushFinalImages, options.WithFinalRepository(imageName))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "build", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(2))
+			//Expect(len(artifacts[0].Dependencies)).To(Equal(1))
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.ImageID()))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.GetMetadataFilePath()))).To(BeTrue())
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.ImageID()))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.GetMetadataFilePath()))).To(BeTrue())
+
+			img, err := b.ImageReference(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.ImageID()), true)
+			Expect(err).ToNot(HaveOccurred())
+			_, path, err := image.Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(path) // clean up
+
+			Expect(fileHelper.Exists(filepath.Join(path, "bin/busybox"))).To(BeTrue())
+
+			img, err = b.ImageReference(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.GetMetadataFilePath()), true)
+			Expect(err).ToNot(HaveOccurred())
+			_, path, err = image.Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(path) // clean up
+
+			meta := filepath.Join(path, artifacts[1].Runtime.GetMetadataFilePath())
+			Expect(fileHelper.Exists(meta)).To(BeTrue())
+
+			d, err := ioutil.ReadFile(meta)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(d)).To(ContainSubstring(artifacts[1].CompileSpec.GetPackage().GetName()))
+		})
 	})
 
 	Context("Packages which conents are a package folder", func() {
@@ -896,11 +963,11 @@ var _ = Describe("Compiler", func() {
 			Expect(len(artifacts)).To(Equal(2))
 			Expect(len(artifacts[0].Dependencies)).To(Equal(0))
 
-			Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.NewPackageArtifact(filepath.Join(tmpdir, "dironly-test-1.0.package.tar")).Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(spec.Rel("test1"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test2"))).To(BeTrue())
 
-			Expect(helpers.Untar(spec2.Rel("dironly_filter-test-1.0.package.tar"), tmpdir2, false)).ToNot(HaveOccurred())
+			Expect(artifact.NewPackageArtifact(filepath.Join(tmpdir2, "dironly_filter-test-1.0.package.tar")).Unpack(ctx, tmpdir2, false)).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(spec2.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
 			Expect(fileHelper.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())

pkg/compiler/types/options/compiler_options.go

@@ -47,6 +47,12 @@ type Compiler struct {
 	// TemplatesFolder. should default to tree/templates
 	TemplatesFolder []string
 
+	// Tells wether to push final container images after building
+	PushFinalImages      bool
+	PushFinalImagesForce bool
+	// Image repository to push to
+	PushFinalImagesRepository string
+
 	Context *types.Context
 }
 
@@ -85,6 +91,25 @@ func WithOptions(opt *Compiler) func(cfg *Compiler) error {
 	}
 }
 
+// WithFinalRepository Sets the final repository where to push
+// images of built artifacts
+func WithFinalRepository(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PushFinalImagesRepository = r
+		return nil
+	}
+}
+
+func EnablePushFinalImages(cfg *Compiler) error {
+	cfg.PushFinalImages = true
+	return nil
+}
+
+func ForcePushFinalImages(cfg *Compiler) error {
+	cfg.PushFinalImagesForce = true
+	return nil
+}
+
 func WithBackendType(r string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		cfg.BackendType = r
@@ -113,6 +138,8 @@ func WithPullRepositories(r []string) func(cfg *Compiler) error {
 	}
 }
 
+// WithPushRepository Sets the image reference where to push
+// cache images
 func WithPushRepository(r string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		if len(cfg.PullImageRepository) == 0 {

pkg/helpers/archive.go

@@ -16,14 +16,10 @@
 package helpers
 
 import (
-	"archive/tar"
-	"bytes"
 	"io"
 	"os"
-	"path/filepath"
 
-	"github.com/docker/docker/pkg/archive"
-	"github.com/pkg/errors"
+	"github.com/moby/moby/pkg/archive"
 )
 
 func Tar(src, dest string) error {
@@ -50,168 +46,3 @@ func Tar(src, dest string) error {
 	}
 	return err
 }
-
-type TarModifierWrapperFunc func(path, dst string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
-type TarModifierWrapper struct {
-	DestinationPath string
-	Modifier        TarModifierWrapperFunc
-}
-
-func NewTarModifierWrapper(dst string, modifier TarModifierWrapperFunc) *TarModifierWrapper {
-	return &TarModifierWrapper{
-		DestinationPath: dst,
-		Modifier:        modifier,
-	}
-}
-
-func (m *TarModifierWrapper) GetModifier() archive.TarModifierFunc {
-	return func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
-		return m.Modifier(m.DestinationPath, path, header, content)
-	}
-}
-
-func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modifier *TarModifierWrapper) error {
-	var ans error
-
-	if len(protectedFiles) <= 0 {
-		return Untar(src, dst, sameOwner)
-	}
-
-	// POST: we have files to protect. I create a ReplaceFileTarWrapper
-	in, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer in.Close()
-
-	// Create modifier map
-	mods := make(map[string]archive.TarModifierFunc)
-	for _, file := range protectedFiles {
-		mods[file] = modifier.GetModifier()
-	}
-
-	if sameOwner {
-		// we do have root permissions, so we can extract keeping the same permissions.
-		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
-
-		opts := &archive.TarOptions{
-			NoLchown:        false,
-			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
-			ContinueOnError: true,
-		}
-
-		ans = archive.Untar(replacerArchive, dst, opts)
-	} else {
-		ans = unTarIgnoreOwner(dst, in, mods)
-	}
-
-	return ans
-}
-
-func unTarIgnoreOwner(dest string, in io.ReadCloser, mods map[string]archive.TarModifierFunc) error {
-	tr := tar.NewReader(in)
-	for {
-		header, err := tr.Next()
-
-		var data []byte
-		var headerReplaced = false
-
-		switch {
-		case err == io.EOF:
-			goto tarEof
-		case err != nil:
-			return err
-		case header == nil:
-			continue
-		}
-
-		// the target location where the dir/file should be created
-		target := filepath.Join(dest, header.Name)
-		if mods != nil {
-			modifier, ok := mods[header.Name]
-			if ok {
-				header, data, err = modifier(header.Name, header, tr)
-				if err != nil {
-					return errors.Wrap(err, "running modifier wrapper")
-				}
-
-				// Override target path
-				target = filepath.Join(dest, header.Name)
-				headerReplaced = true
-			}
-
-		}
-
-		// Check the file type
-		switch header.Typeflag {
-
-		// if its a dir and it doesn't exist create it
-		case tar.TypeDir:
-			if _, err := os.Stat(target); err != nil {
-				if err := os.MkdirAll(target, 0755); err != nil {
-					return err
-				}
-			}
-
-			// handle creation of file
-		case tar.TypeReg:
-
-			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
-			if err != nil {
-				return errors.Wrap(err, "creating destination")
-			}
-
-			// copy over contents
-			if headerReplaced {
-				_, err = io.Copy(f, bytes.NewReader(data))
-			} else {
-				_, err = io.Copy(f, tr)
-			}
-			if err != nil {
-				return err
-			}
-
-			// manually close here after each file operation; defering would cause each
-			// file close to wait until all operations have completed.
-			f.Close()
-
-		case tar.TypeSymlink:
-			source := header.Linkname
-			err := os.Symlink(source, target)
-			if err != nil {
-				return err
-			}
-		}
-	}
-tarEof:
-
-	return nil
-}
-
-// Untar just a wrapper around the docker functions
-func Untar(src, dest string, sameOwner bool) (err error) {
-
-	in, err := os.Open(src)
-	if err != nil {
-		return errors.Wrap(err, "while opening "+src+" for untar ")
-	}
-	defer in.Close()
-
-	if sameOwner {
-		opts := &archive.TarOptions{
-			NoLchown:        false,
-			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
-			ContinueOnError: true,
-		}
-
-		err = archive.Untar(in, dest, opts)
-	} else {
-		err = unTarIgnoreOwner(dest, in, nil)
-	}
-
-	if err != nil {
-		err = errors.Wrap(err, "while untarring "+src+" into "+dest)
-	}
-
-	return
-}

pkg/helpers/archive_test.go

@@ -1,136 +0,0 @@
-// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
-//                       Daniele Rondina <geaaru@sabayonlinux.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package helpers_test
-
-import (
-	"archive/tar"
-	"bytes"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	fileHelper "github.com/mudler/luet/pkg/helpers/file"
-
-	"github.com/docker/docker/pkg/archive"
-	. "github.com/mudler/luet/pkg/helpers"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-// Code from moby/moby pkg/archive/archive_test
-func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
-	fileData := []byte("fooo")
-	for n := 0; n < numberOfFiles; n++ {
-		fileName := fmt.Sprintf("file-%d", n)
-		if err := ioutil.WriteFile(filepath.Join(targetPath, fileName), fileData, 0700); err != nil {
-			return 0, err
-		}
-		if makeLinks {
-			if err := os.Link(filepath.Join(targetPath, fileName), filepath.Join(targetPath, fileName+"-link")); err != nil {
-				return 0, err
-			}
-		}
-	}
-	totalSize := numberOfFiles * len(fileData)
-	return totalSize, nil
-}
-
-func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
-	// If the destination path already exists I rename target file name with postfix.
-	var basePath string
-
-	// Read data. TODO: We need change archive callback to permit to return a Reader
-	buffer := bytes.Buffer{}
-	if content != nil {
-		if _, err := buffer.ReadFrom(content); err != nil {
-			return nil, nil, err
-		}
-	}
-
-	if header != nil {
-
-		switch header.Typeflag {
-		case tar.TypeReg:
-			basePath = filepath.Base(path)
-		default:
-			// Nothing to do. I return original reader
-			return header, buffer.Bytes(), nil
-		}
-
-		if basePath == "file-0" {
-			name := filepath.Join(filepath.Join(filepath.Dir(path), fmt.Sprintf("._cfg%04d_%s", 1, basePath)))
-			return &tar.Header{
-				Mode:       header.Mode,
-				Typeflag:   header.Typeflag,
-				PAXRecords: header.PAXRecords,
-				Name:       name,
-			}, buffer.Bytes(), nil
-		} else if basePath == "file-1" {
-			return header, []byte("newcontent"), nil
-		}
-
-		// else file not present
-	}
-
-	return header, buffer.Bytes(), nil
-}
-
-var _ = Describe("Helpers Archive", func() {
-	Context("Untar Protect", func() {
-
-		It("Detect existing and not-existing files", func() {
-
-			archiveSourceDir, err := ioutil.TempDir("", "archive-source")
-			Expect(err).ToNot(HaveOccurred())
-			defer os.RemoveAll(archiveSourceDir)
-
-			_, err = prepareUntarSourceDirectory(10, archiveSourceDir, false)
-			Expect(err).ToNot(HaveOccurred())
-
-			targetDir, err := ioutil.TempDir("", "archive-target")
-			Expect(err).ToNot(HaveOccurred())
-			//	defer os.RemoveAll(targetDir)
-
-			sourceArchive, err := archive.TarWithOptions(archiveSourceDir, &archive.TarOptions{})
-			Expect(err).ToNot(HaveOccurred())
-			defer sourceArchive.Close()
-
-			tarModifier := NewTarModifierWrapper(targetDir, tarModifierWrapperFunc)
-			mods := make(map[string]archive.TarModifierFunc)
-			mods["file-0"] = tarModifier.GetModifier()
-			mods["file-1"] = tarModifier.GetModifier()
-			mods["file-9999"] = tarModifier.GetModifier()
-
-			replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
-			//replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
-			opts := &archive.TarOptions{
-				// NOTE: NoLchown boolean is used for chmod of the symlink
-				// Probably it's needed set this always to true.
-				NoLchown:        true,
-				ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
-				ContinueOnError: true,
-			}
-
-			err = archive.Untar(replacerArchive, targetDir, opts)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(fileHelper.Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
-		})
-	})
-})

pkg/helpers/docker/docker.go

@@ -18,25 +18,22 @@ package docker
 import (
 	"context"
 	"encoding/hex"
-	"fmt"
 	"os"
-	"strings"
 
 	"github.com/containerd/containerd/images"
+	luetimages "github.com/mudler/luet/pkg/api/core/image"
+	luettypes "github.com/mudler/luet/pkg/api/core/types"
+
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
-	continerdarchive "github.com/containerd/containerd/archive"
 	"github.com/docker/cli/cli/trust"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/registry"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
-	v1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
-	"github.com/mudler/luet/pkg/bus"
+	"github.com/mudler/luet/pkg/api/core/bus"
 	"github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -129,44 +126,8 @@ type UnpackEventData struct {
 	Dest  string
 }
 
-// UnarchiveLayers extract layers with archive.Untar from docker instead of containerd
-func UnarchiveLayers(temp string, img v1.Image, image, dest string, auth *types.AuthConfig, verify bool) (int64, error) {
-	layers, err := img.Layers()
-	if err != nil {
-		return 0, fmt.Errorf("reading layers from '%s' image failed: %v", image, err)
-	}
-	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
-
-	var size int64
-	for _, l := range layers {
-		s, err := l.Size()
-		if err != nil {
-			return 0, fmt.Errorf("reading layer size from '%s' image failed: %v", image, err)
-		}
-		size += s
-
-		layerReader, err := l.Uncompressed()
-		if err != nil {
-			return 0, fmt.Errorf("reading uncompressed layer from '%s' image failed: %v", image, err)
-		}
-		defer layerReader.Close()
-
-		// Unpack the tarfile to the rootfs path.
-		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
-		if err := archive.Untar(layerReader, dest, &archive.TarOptions{
-			NoLchown:        false,
-			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
-		}); err != nil {
-			return 0, fmt.Errorf("extracting '%s' image to directory %s failed: %v", image, dest, err)
-		}
-	}
-	bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
-
-	return size, nil
-}
-
 // DownloadAndExtractDockerImage extracts a container image natively. It supports privileged/unprivileged mode
-func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*images.Image, error) {
+func DownloadAndExtractDockerImage(ctx *luettypes.Context, image, dest string, auth *types.AuthConfig, verify bool) (*images.Image, error) {
 	if verify {
 		img, err := verifyImage(image, auth)
 		if err != nil {
@@ -206,13 +167,15 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
 		return nil, err
 	}
 
-	reader := mutate.Extract(img)
-	defer reader.Close()
-	defer os.RemoveAll(temp)
-
 	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
 
-	c, err := continerdarchive.Apply(context.TODO(), dest, reader)
+	var c int64
+	c, _, err = luetimages.ExtractTo(
+		ctx,
+		img,
+		dest,
+		nil,
+	)
 	if err != nil {
 		return nil, err
 	}
@@ -229,7 +192,3 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
 		},
 	}, nil
 }
-
-func StripInvalidStringsFromImage(s string) string {
-	return strings.ReplaceAll(s, "+", "-")
-}

pkg/helpers/references.go

@@ -17,8 +17,9 @@
 package helpers
 
 import (
-	"github.com/asaskevich/govalidator"
 	"strings"
+
+	"github.com/asaskevich/govalidator"
 )
 
 func StripRegistryFromImage(image string) string {
@@ -28,3 +29,7 @@ func StripRegistryFromImage(image string) string {
 	}
 	return image
 }
+
+func SanitizeImageString(s string) string {
+	return strings.ReplaceAll(s, "+", "-")
+}

pkg/helpers/references_test.go

@@ -23,6 +23,11 @@ import (
 )
 
 var _ = Describe("Helpers", func() {
+	Context("Image names", func() {
+		It("strips invalid chars", func() {
+			Expect(SanitizeImageString("foo+bar")).To(Equal("foo-bar"))
+		})
+	})
 	Context("StripRegistryFromImage", func() {
 		It("Strips the domain name", func() {
 			out := StripRegistryFromImage("valid.domain.org/base/image:tag")

pkg/installer/client/docker.go

@@ -28,6 +28,7 @@ import (
 
 	luetTypes "github.com/mudler/luet/pkg/api/core/types"
 	"github.com/mudler/luet/pkg/api/core/types/artifact"
+	"github.com/mudler/luet/pkg/helpers"
 
 	"github.com/mudler/luet/pkg/helpers/docker"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
@@ -63,11 +64,13 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
 	c.context.Spinner()
 	defer c.context.SpinnerStop()
 
-	resultingArtifact := a.ShallowCopy()
-	artifactName := path.Base(a.Path)
-
 	downloaded := false
 
+	resultingArtifact, err := c.CacheGet(a)
+	if err == nil {
+		return resultingArtifact, nil
+	}
+
 	// TODO:
 	// Files are in URI/packagename:version (GetPackageImageName() method)
 	// use downloadAndExtract .. and egenrate an archive to consume. Checksum should be already checked while downloading the image
@@ -75,79 +78,60 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
 	// - Check how verification is done when calling DownloadArtifact outside, similarly we need to check DownloadFile, and how verification
 	// is done in such cases (see repository.go)
 
-	// Check if file is already in cache
-	fileName, err := c.Cache.Get(a)
-	// Check if file is already in cache
-	if err == nil {
-		resultingArtifact = a
-		resultingArtifact.Path = fileName
-		resultingArtifact.Checksums = artifact.Checksums{}
-		c.context.Debug("Use artifact", artifactName, "from cache.")
-	} else {
+	// We discard checksum, that are checked while during pull and unpack by containerd
+	resultingArtifact.Checksums = artifact.Checksums{}
 
-		temp, err := c.context.Config.GetSystem().TempDir("image")
+	temp, err := c.context.Config.GetSystem().TempDir("image")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(temp)
+
+	tempArtifact, err := c.context.Config.GetSystem().TempFile("artifact")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tempArtifact.Name())
+	for _, uri := range c.RepoData.Urls {
+
+		imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID())
+		c.context.Info("Downloading image", imageName)
+
+		// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
+		info, err := docker.DownloadAndExtractDockerImage(c.context, imageName, temp, c.auth, c.RepoData.Verify)
 		if err != nil {
-			return nil, err
+			c.context.Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
+			continue
 		}
-		defer os.RemoveAll(temp)
 
-		tempArtifact, err := c.context.Config.GetSystem().TempFile("artifact")
+		c.context.Info(
+			fmt.Sprintf("Image: %s. Pulled: %s. Size: %s",
+				imageName,
+				info.Target.Digest,
+				units.BytesSize(float64(info.Target.Size)),
+			),
+		)
+		c.context.Debug("\nCompressing result ", filepath.Join(temp), "to", tempArtifact.Name())
+
+		resultingArtifact.Path = tempArtifact.Name() // First set to cache file
+		err = resultingArtifact.Compress(temp, 1)
 		if err != nil {
-			return nil, err
-		}
-		defer os.RemoveAll(tempArtifact.Name())
-		for _, uri := range c.RepoData.Urls {
-
-			imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID())
-			c.context.Info("Downloading image", imageName)
-
-			contentstore, err := c.context.Config.GetSystem().TempDir("contentstore")
-			if err != nil {
-				c.context.Warning("Cannot create contentstore", err.Error())
-				continue
-			}
-
-			// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
-			info, err := docker.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
-			if err != nil {
-				c.context.Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
-				continue
-			}
-
-			c.context.Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
-			c.context.Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size))))
-			c.context.Debug("\nCompressing result ", filepath.Join(temp), "to", tempArtifact.Name())
-
-			// We discard checksum, that are checked while during pull and unpack
-			resultingArtifact.Checksums = artifact.Checksums{}
-			resultingArtifact.Path = tempArtifact.Name() // First set to cache file
-			err = resultingArtifact.Compress(temp, 1)
-			if err != nil {
-				c.context.Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))
-				continue
-			}
-
-			_, _, err = c.Cache.Put(resultingArtifact)
-			if err != nil {
-				c.context.Error(fmt.Sprintf("Failed storing package %s from cache: %s", imageName, err.Error()))
-				continue
-			}
-
-			fileName, err := c.Cache.Get(resultingArtifact)
-			if err != nil {
-				c.context.Error(fmt.Sprintf("Failed getting package %s from cache: %s", imageName, err.Error()))
-				continue
-			}
-
-			resultingArtifact.Path = fileName // Cache is persistent. tempArtifact is not
-
-			downloaded = true
-			break
+			c.context.Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))
+			continue
 		}
 
-		if !downloaded {
-			return nil, errors.Wrap(err, "no image available from repositories")
+		_, _, err = c.Cache.Put(resultingArtifact)
+		if err != nil {
+			c.context.Error(fmt.Sprintf("Failed storing package %s from cache: %s", imageName, err.Error()))
+			continue
 		}
+		downloaded = true
+
+		return c.CacheGet(resultingArtifact)
+	}
+
+	if !downloaded {
+		return nil, errors.Wrap(err, "no image available from repositories")
 	}
 
 	return resultingArtifact, nil
@@ -156,7 +140,7 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
 func (c *DockerClient) DownloadFile(name string) (string, error) {
 	var file *os.File = nil
 	var err error
-	var temp, contentstore string
+	var temp string
 	// Files should be in URI/repository:<file>
 	ok := false
 
@@ -171,16 +155,10 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 			continue
 		}
 
-		contentstore, err = c.context.Config.GetSystem().TempDir("contentstore")
-		if err != nil {
-			c.context.Warning("Cannot create contentstore", err.Error())
-			continue
-		}
-
-		imageName := fmt.Sprintf("%s:%s", uri, docker.StripInvalidStringsFromImage(name))
+		imageName := fmt.Sprintf("%s:%s", uri, helpers.SanitizeImageString(name))
 		c.context.Info("Downloading", imageName)
 
-		info, err := docker.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
+		info, err := docker.DownloadAndExtractDockerImage(c.context, imageName, temp, c.auth, c.RepoData.Verify)
 		if err != nil {
 			c.context.Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
 			continue
@@ -204,3 +182,29 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 
 	return file.Name(), err
 }
+
+func (c *DockerClient) CacheGet(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
+	resultingArtifact := a.ShallowCopy()
+	// TODO:
+	// Files are in URI/packagename:version (GetPackageImageName() method)
+	// use downloadAndExtract .. and egenrate an archive to consume. Checksum should be already checked while downloading the image
+	// with the above functions, because Docker images already contain such metadata
+	// - Check how verification is done when calling DownloadArtifact outside, similarly we need to check DownloadFile, and how verification
+	// is done in such cases (see repository.go)
+
+	// We discard checksum, that are checked while during pull and unpack by containerd
+	resultingArtifact.Checksums = artifact.Checksums{}
+
+	// Check if file is already in cache
+	fileName, err := c.Cache.Get(resultingArtifact)
+	// Check if file is already in cache
+	if err == nil {
+		artifactName := path.Base(a.Path)
+		c.context.Debug("Use artifact", artifactName, "from cache.")
+		resultingArtifact = a
+		resultingArtifact.Path = fileName
+		resultingArtifact.Checksums = artifact.Checksums{}
+	}
+
+	return resultingArtifact, err
+}

pkg/installer/client/http.go

@@ -23,7 +23,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"strconv"
 	"time"
 
 	"github.com/mudler/luet/pkg/api/core/types"
@@ -48,20 +47,11 @@ func NewHttpClient(r RepoData, ctx *types.Context) *HttpClient {
 	}
 }
 
-func NewGrabClient() *grab.Client {
-	httpTimeout := 360
-	timeout := os.Getenv("HTTP_TIMEOUT")
-	if timeout != "" {
-		timeoutI, err := strconv.Atoi(timeout)
-		if err == nil {
-			httpTimeout = timeoutI
-		}
-	}
-
+func NewGrabClient(timeout int) *grab.Client {
 	return &grab.Client{
 		UserAgent: "grab",
 		HTTPClient: &http.Client{
-			Timeout: time.Duration(httpTimeout) * time.Second,
+			Timeout: time.Duration(timeout) * time.Second,
 			Transport: &http.Transport{
 				Proxy: http.ProxyFromEnvironment,
 			},
@@ -101,7 +91,7 @@ func (c *HttpClient) DownloadFile(p string) (string, error) {
 	}
 	defer os.RemoveAll(temp)
 
-	client := NewGrabClient()
+	client := NewGrabClient(c.context.Config.General.HTTPTimeout)
 
 	for _, uri := range c.RepoData.Urls {
 		file, err = c.context.Config.GetSystem().TempFile("HttpClient")
@@ -128,11 +118,7 @@ func (c *HttpClient) DownloadFile(p string) (string, error) {
 		// Initialize a progressbar only if we have one in the current context
 		var pb *pterm.ProgressbarPrinter
 		if c.context.ProgressBar != nil {
-			pb = pterm.DefaultProgressbar.WithTotal(int(resp.Size()))
-			if c.context.AreaPrinter != nil {
-				pb = pb.WithPrintTogether(c.context.AreaPrinter)
-			}
-			pb, _ = pb.WithTitle(filepath.Base(resp.Request.HTTPRequest.URL.RequestURI())).Start()
+			pb, _ = c.context.ProgressBar.WithTotal(int(resp.Size())).WithTitle(filepath.Base(resp.Request.HTTPRequest.URL.RequestURI())).Start()
 		}
 		// start download loop
 		t := time.NewTicker(500 * time.Millisecond)
@@ -180,32 +166,33 @@ func (c *HttpClient) DownloadFile(p string) (string, error) {
 	return file.Name(), nil
 }
 
-func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
+func (c *HttpClient) CacheGet(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	newart := a.ShallowCopy()
-	artifactName := path.Base(a.Path)
 
 	fileName, err := c.Cache.Get(a)
+
+	newart.Path = fileName
+
+	return newart, err
+}
+
+func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
+	artifactName := path.Base(a.Path)
+
+	newart, err := c.CacheGet(a)
 	// Check if file is already in cache
 	if err == nil {
-		newart.Path = fileName
-		c.context.Debug("Use artifact", artifactName, "from cache.")
-	} else {
-		d, err := c.DownloadFile(artifactName)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed downloading %s", artifactName)
-		}
-
-		defer os.RemoveAll(d)
-		newart.Path = d
-		c.Cache.Put(newart)
-
-		fileName, err := c.Cache.Get(newart)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed getting file from cache %v", newart)
-		}
-
-		newart.Path = fileName
+		return newart, nil
 	}
 
-	return newart, nil
+	d, err := c.DownloadFile(artifactName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed downloading %s", artifactName)
+	}
+
+	defer os.RemoveAll(d)
+	newart.Path = d
+	c.Cache.Put(newart)
+
+	return c.CacheGet(newart)
 }

pkg/installer/client/local.go

@@ -44,32 +44,32 @@ func (c *LocalClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.P
 	var err error
 
 	artifactName := path.Base(a.Path)
-	newart := a.ShallowCopy()
 
-	fileName, err := c.Cache.Get(a)
+	newart, err := c.CacheGet(a)
 	// Check if file is already in cache
 	if err == nil {
-		newart.Path = fileName
-		c.context.Debug("Use artifact", artifactName, "from cache.")
-	} else {
-		d, err := c.DownloadFile(artifactName)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed downloading %s", artifactName)
-		}
-		defer os.RemoveAll(d)
-
-		newart.Path = d
-		c.Cache.Put(newart)
-
-		fileName, err := c.Cache.Get(newart)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed getting file from cache %v", newart)
-		}
-
-		newart.Path = fileName
+		return newart, nil
 	}
 
-	return newart, nil
+	d, err := c.DownloadFile(artifactName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed downloading %s", artifactName)
+	}
+	defer os.RemoveAll(d)
+
+	newart.Path = d
+	c.Cache.Put(newart)
+
+	return c.CacheGet(newart)
+}
+
+func (c *LocalClient) CacheGet(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
+	newart := a.ShallowCopy()
+	fileName, err := c.Cache.Get(a)
+
+	newart.Path = fileName
+
+	return newart, err
 }
 
 func (c *LocalClient) DownloadFile(name string) (string, error) {

pkg/installer/installer.go

@@ -26,10 +26,9 @@ import (
 
 	"github.com/mudler/luet/pkg/api/core/config"
 
+	"github.com/mudler/luet/pkg/api/core/bus"
 	"github.com/mudler/luet/pkg/api/core/types"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
-	"github.com/mudler/luet/pkg/bus"
-	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	"github.com/mudler/luet/pkg/helpers/match"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -53,6 +52,7 @@ type LuetInstallerOptions struct {
 	DownloadOnly                                                   bool
 	Relaxed                                                        bool
 	PackageRepositories                                            types.LuetRepositories
+	AutoOSCheck                                                    bool
 
 	Context *types.Context
 }
@@ -267,13 +267,30 @@ func (l *LuetInstaller) swap(o Option, syncedRepos Repositories, toRemove pkg.Pa
 		return nil
 	}
 
-	ops := l.getOpsWithOptions(toRemove, match, Option{
+	ops, err := l.getOpsWithOptions(toRemove, match, Option{
 		Force:              o.Force,
 		NoDeps:             false,
 		OnlyDeps:           o.OnlyDeps,
 		RunFinalizers:      false,
 		CheckFileConflicts: false,
-	}, o, syncedRepos, packages, assertions, allRepos)
+	}, o, syncedRepos, packages, assertions, allRepos, s)
+	if err != nil {
+		return errors.Wrap(err, "failed computing installer options")
+	}
+
+	l.Options.Context.Info("Computed operations")
+
+	for _, o := range ops {
+		toUninstall := ""
+		toInstall := ""
+		for _, u := range o.Uninstall {
+			toUninstall += fmt.Sprintf(" %s", u.Package.HumanReadableString())
+		}
+		for _, u := range o.Install {
+			toInstall += fmt.Sprintf(" %s", u.Package.HumanReadableString())
+		}
+		l.Options.Context.Info(fmt.Sprintf("%s -> %s", toUninstall, toInstall))
+	}
 
 	err = l.runOps(ops, s)
 	if err != nil {
@@ -317,19 +334,20 @@ type installOperation struct {
 // installerOp is the operation that is sent to the
 // upgradeWorker's channel (todo)
 type installerOp struct {
-	Uninstall operation
-	Install   installOperation
+	Uninstall []operation
+	Install   []installOperation
 }
 
 func (l *LuetInstaller) runOps(ops []installerOp, s *System) error {
 	all := make(chan installerOp)
 
 	wg := new(sync.WaitGroup)
+	systemLock := &sync.Mutex{}
 
 	// Do the real install
 	for i := 0; i < l.Options.Concurrency; i++ {
 		wg.Add(1)
-		go l.installerOpWorker(i, wg, all, s)
+		go l.installerOpWorker(i, wg, systemLock, all, s)
 	}
 
 	for _, c := range ops {
@@ -343,40 +361,43 @@ func (l *LuetInstaller) runOps(ops []installerOp, s *System) error {
 
 // TODO: use installerOpWorker in place of all the other workers.
 // This one is general enough to read a list of operations and execute them.
-func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, c <-chan installerOp, s *System) error {
+func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, systemLock *sync.Mutex, c <-chan installerOp, s *System) error {
 	defer wg.Done()
 
 	for p := range c {
-		if p.Uninstall.Package != nil {
-			l.Options.Context.Debug("Replacing package inplace")
-			toUninstall, uninstall, err := l.generateUninstallFn(p.Uninstall.Option, s, p.Uninstall.Package)
-			if err != nil {
-				l.Options.Context.Error("Failed to generate Uninstall function for" + err.Error())
-				continue
-				//return errors.Wrap(err, "while computing uninstall")
-			}
 
+		for _, pp := range p.Uninstall {
+			l.Options.Context.Debug("Replacing package inplace")
+			toUninstall, uninstall, err := l.generateUninstallFn(pp.Option, s, pp.Package)
+			if err != nil {
+				l.Options.Context.Debug("Skipping uninstall, fail to generate uninstall function, error: " + err.Error())
+				continue
+			}
+			systemLock.Lock()
 			err = uninstall()
+			systemLock.Unlock()
+
 			if err != nil {
 				l.Options.Context.Error("Failed uninstall for ", packsToList(toUninstall))
 				continue
-				//return errors.Wrap(err, "uninstalling "+packsToList(toUninstall))
 			}
 		}
-		if p.Install.Package != nil {
-			artMatch := p.Install.Matches[p.Install.Package.GetFingerPrint()]
-			ass := p.Install.Assertions.Search(p.Install.Package.GetFingerPrint())
-			packageToInstall, _ := p.Install.Packages.Find(p.Install.Package.GetPackageName())
+		for _, pp := range p.Install {
+			artMatch := pp.Matches[pp.Package.GetFingerPrint()]
+			ass := pp.Assertions.Search(pp.Package.GetFingerPrint())
+			packageToInstall, _ := pp.Packages.Find(pp.Package.GetPackageName())
 
+			systemLock.Lock()
 			err := l.install(
-				p.Install.Option,
-				p.Install.Reposiories,
-				map[string]ArtifactMatch{p.Install.Package.GetFingerPrint(): artMatch},
+				pp.Option,
+				pp.Reposiories,
+				map[string]ArtifactMatch{pp.Package.GetFingerPrint(): artMatch},
 				pkg.Packages{packageToInstall},
 				solver.PackagesAssertions{*ass},
-				p.Install.Database,
+				pp.Database,
 				s,
 			)
+			systemLock.Unlock()
 			if err != nil {
 				l.Options.Context.Error(err)
 			}
@@ -389,35 +410,140 @@ func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, c <-chan in
 // checks wheter we can uninstall and install in place and compose installer worker ops
 func (l *LuetInstaller) getOpsWithOptions(
 	toUninstall pkg.Packages, installMatch map[string]ArtifactMatch, installOpt, uninstallOpt Option,
-	syncedRepos Repositories, toInstall pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase) []installerOp {
+	syncedRepos Repositories, toInstall pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) ([]installerOp, error) {
+
+	l.Options.Context.Debug("Computing installation order")
 	resOps := []installerOp{}
+
+	insertPackage := func(install []pkg.Package, uninstall ...pkg.Package) {
+		if len(install) == 0 && len(uninstall) == 0 {
+			return
+		}
+		uOpts := []operation{}
+		for _, u := range uninstall {
+			uOpts = append(uOpts, operation{Package: u, Option: uninstallOpt})
+		}
+		iOpts := []installOperation{}
+		for _, u := range install {
+			iOpts = append(iOpts, installOperation{
+				operation: operation{
+					Package: u,
+					Option:  installOpt,
+				},
+				Matches:     installMatch,
+				Packages:    toInstall,
+				Reposiories: syncedRepos,
+				Assertions:  solution,
+				Database:    allRepos,
+			})
+		}
+		resOps = append(resOps, installerOp{
+			Uninstall: uOpts,
+			Install:   iOpts,
+		})
+	}
+
+	removals := make(map[string]interface{})
+	additions := make(map[string]interface{})
+
+	remove := func(p pkg.Package) {
+		removals[p.GetPackageName()] = nil
+	}
+
+	add := func(p pkg.Package) {
+		additions[p.GetPackageName()] = nil
+	}
+
+	added := func(p pkg.Package) bool {
+		_, exists := additions[p.GetPackageName()]
+		return exists
+	}
+
+	removed := func(p pkg.Package) bool {
+		_, exists := removals[p.GetPackageName()]
+		return exists
+	}
+
+	findToBeInstalled := func(p pkg.Package) pkg.Package {
+		for _, m := range installMatch {
+			if m.Package.GetPackageName() == p.GetPackageName() {
+				return m.Package
+			}
+		}
+		return nil
+	}
+
+	fileIndex := s.FileIndex()
+
 	for _, match := range installMatch {
-		if pack, err := toUninstall.Find(match.Package.GetPackageName()); err == nil {
-			resOps = append(resOps, installerOp{
-				Uninstall: operation{Package: pack, Option: uninstallOpt},
-				Install: installOperation{
-					operation: operation{
-						Package: match.Package,
-						Option:  installOpt,
-					},
-					Matches:     installMatch,
-					Packages:    toInstall,
-					Reposiories: syncedRepos,
-					Assertions:  solution,
-					Database:    allRepos,
-				},
-			})
-		} else {
-			resOps = append(resOps, installerOp{
-				Install: installOperation{
-					operation:   operation{Package: match.Package, Option: installOpt},
-					Matches:     installMatch,
-					Reposiories: syncedRepos,
-					Packages:    toInstall,
-					Assertions:  solution,
-					Database:    allRepos,
-				},
-			})
+		a, err := l.getPackage(match, l.Options.Context)
+		if err != nil && !l.Options.Force {
+			return nil, errors.Wrap(err, "Failed downloading package")
+		}
+		files, err := a.FileList()
+		if err != nil && !l.Options.Force {
+			return nil, errors.Wrapf(err, "Could not get filelist for %s", a.CompileSpec.Package.HumanReadableString())
+		}
+
+		l.Options.Context.Debug(match.Package.HumanReadableString(), "files", len(files))
+
+		foundPackages := make(map[string]pkg.Package)
+	FILES:
+		for _, f := range files {
+			if p, exists := fileIndex[f]; exists {
+				if _, exists := foundPackages[p.HumanReadableString()]; exists {
+					continue FILES
+				}
+
+				_, err := toUninstall.Find(p.GetPackageName())
+				if err == nil {
+					l.Options.Context.Debug(p.HumanReadableString(), "files", f, "matches")
+
+					// Packages that is being installed have a file that
+					// is going to be removed by another package
+					foundPackages[p.HumanReadableString()] = p
+				}
+			}
+		}
+		l.Options.Context.Debug(match.Package.HumanReadableString(), "found", len(foundPackages), "packages")
+		if len(foundPackages) > 0 {
+			if pack, err := toUninstall.Find(match.Package.GetPackageName()); err == nil {
+				foundPackages[pack.HumanReadableString()] = pack
+			}
+
+			toInstall := []pkg.Package{}
+			if !added(match.Package) {
+				toInstall = append(toInstall, match.Package)
+				add(match.Package)
+			}
+			toRemove := []pkg.Package{}
+			for _, p := range foundPackages {
+				if !removed(p) {
+					toRemove = append(toRemove, p)
+					if pp := findToBeInstalled(p); pp != nil && !added(pp) {
+						toInstall = append(toInstall, pp)
+						add(pp)
+					}
+					remove(p)
+				}
+			}
+			// toInstall needs to have:
+			// equivalent of what was found in foundPackages AND
+			// was not already added to installation
+			insertPackage(toInstall, toRemove...)
+		} else if pack, err := toUninstall.Find(match.Package.GetPackageName()); err == nil {
+			if !removed(pack) {
+				toInstall := []pkg.Package{}
+				if !added(match.Package) {
+					toInstall = append(toInstall, match.Package)
+					add(match.Package)
+				}
+				insertPackage(toInstall, pack)
+				remove(pack)
+			}
+		} else if !added(match.Package) {
+			insertPackage([]pkg.Package{match.Package})
+			add(match.Package)
 		}
 	}
 
@@ -428,24 +554,24 @@ func (l *LuetInstaller) getOpsWithOptions(
 			if match.Package.GetPackageName() == p.GetPackageName() {
 				found = true
 			}
-
 		}
 		if !found {
-			resOps = append(resOps, installerOp{
-				Uninstall: operation{Package: p, Option: uninstallOpt},
-			})
+			if _, ok := removals[p.GetPackageName()]; !ok {
+				resOps = append(resOps, installerOp{
+					Uninstall: []operation{{Package: p, Option: uninstallOpt}},
+				})
+				removals[p.GetPackageName()] = nil
+			}
 		}
 	}
-	return resOps
+	return resOps, nil
 }
 
 func (l *LuetInstaller) checkAndUpgrade(r Repositories, s *System) error {
-	//	Spinner(32)
 	uninstall, toInstall, err := l.computeUpgrade(r, s)
 	if err != nil {
 		return errors.Wrap(err, "failed computing upgrade")
 	}
-	//	SpinnerStop()
 
 	if len(toInstall) == 0 && len(uninstall) == 0 {
 		l.Options.Context.Info("Nothing to upgrade")
@@ -481,7 +607,34 @@ func (l *LuetInstaller) checkAndUpgrade(r Repositories, s *System) error {
 		}
 	}
 
-	return l.swap(o, r, uninstall, toInstall, s)
+	bus.Manager.Publish(bus.EventPreUpgrade, struct{ Uninstall, Install pkg.Packages }{Uninstall: uninstall, Install: toInstall})
+
+	err = l.swap(o, r, uninstall, toInstall, s)
+
+	bus.Manager.Publish(bus.EventPostUpgrade, struct {
+		Error              error
+		Uninstall, Install pkg.Packages
+	}{Uninstall: uninstall, Install: toInstall, Error: err})
+
+	if err != nil {
+		return err
+	}
+
+	if l.Options.AutoOSCheck {
+		l.Options.Context.Info("Performing automatic oscheck")
+		packs := s.OSCheck()
+		if len(packs) > 0 {
+			p := ""
+			for _, r := range packs {
+				p += " " + r.HumanReadableString()
+			}
+			l.Options.Context.Info("Following packages requires reinstallation: " + p)
+			return l.swap(o, r, packs, packs, s)
+		}
+		l.Options.Context.Info("OSCheck done")
+	}
+
+	return err
 }
 
 func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
@@ -560,6 +713,21 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 
 func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string]ArtifactMatch) error {
 
+	// Don't attempt to download stuff that is already in cache
+	missArtifacts := false
+	for _, m := range toDownload {
+		c := m.Repository.Client(l.Options.Context)
+		_, err := c.CacheGet(m.Artifact)
+		if err != nil {
+			missArtifacts = true
+		}
+	}
+
+	if !missArtifacts {
+		l.Options.Context.Debug("Packages already in cache, skipping download")
+		return nil
+	}
+
 	// Download packages into cache in parallel.
 	all := make(chan ArtifactMatch)
 
@@ -570,26 +738,25 @@ func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string
 	// Check if the terminal is big enough to display a progress bar
 	// https://github.com/pterm/pterm/blob/4c725e56bfd9eb38e1c7b9dec187b50b93baa8bd/progressbar_printer.go#L190
 	w, _, err := ctx.GetTerminalSize()
+
+	var pb *pterm.ProgressbarPrinter
 	if ctx.IsTerminal && err == nil && w > 100 {
 		area, _ := pterm.DefaultArea.Start()
-		p, _ := pterm.DefaultProgressbar.WithPrintTogether(area).WithTotal(len(toDownload)).WithTitle("Downloading packages").Start()
-
-		ctx.AreaPrinter = area
-		ctx.ProgressBar = p
+		ctx.ProgressBar = pterm.DefaultProgressbar.WithPrintTogether(area).WithTotal(len(toDownload)).WithTitle("Downloading packages")
+		pb, _ = ctx.ProgressBar.Start()
 		defer area.Stop()
 	}
 
 	// Download
 	for i := 0; i < l.Options.Concurrency; i++ {
 		wg.Add(1)
-		go l.downloadWorker(i, wg, all, ctx)
+		go l.downloadWorker(i, wg, pb, all, ctx)
 	}
 	for _, c := range toDownload {
 		all <- c
 	}
 	close(all)
 	wg.Wait()
-
 	return nil
 }
 
@@ -778,9 +945,11 @@ func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, c
 	l.Options.Context.Info("Checking for file conflicts..")
 	defer s.Clean() // Release memory
 
-	filesToInstall := []string{}
+	filesToInstall := map[string]interface{}{}
 	for _, m := range toInstall {
-		a, err := l.downloadPackage(m, l.Options.Context)
+		l.Options.Context.Debug("Checking file conflicts for", m.Package.HumanReadableString())
+
+		a, err := l.getPackage(m, l.Options.Context)
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Failed downloading package")
 		}
@@ -790,7 +959,7 @@ func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, c
 		}
 
 		for _, f := range files {
-			if helpers.Contains(filesToInstall, f) {
+			if _, ok := filesToInstall[f]; ok {
 				return fmt.Errorf(
 					"file conflict between packages to be installed",
 				)
@@ -809,9 +978,10 @@ func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, c
 					)
 				}
 			}
+			filesToInstall[f] = nil
 		}
-		filesToInstall = append(filesToInstall, files...)
 	}
+	l.Options.Context.Info("Done checking for file conflicts..")
 
 	return nil
 }
@@ -841,11 +1011,12 @@ func (l *LuetInstaller) install(o Option, syncedRepos Repositories, toInstall ma
 	all := make(chan ArtifactMatch)
 
 	wg := new(sync.WaitGroup)
+	installLock := &sync.Mutex{}
 
 	// Do the real install
 	for i := 0; i < l.Options.Concurrency; i++ {
 		wg.Add(1)
-		go l.installerWorker(i, wg, all, s)
+		go l.installerWorker(i, wg, installLock, all, s)
 	}
 
 	for _, c := range toInstall {
@@ -875,11 +1046,11 @@ func (l *LuetInstaller) install(o Option, syncedRepos Repositories, toInstall ma
 	return s.ExecuteFinalizers(l.Options.Context, toFinalize)
 }
 
-func (l *LuetInstaller) downloadPackage(a ArtifactMatch, ctx *types.Context) (*artifact.PackageArtifact, error) {
+func (l *LuetInstaller) getPackage(a ArtifactMatch, ctx *types.Context) (artifact *artifact.PackageArtifact, err error) {
 
 	cli := a.Repository.Client(ctx)
 
-	artifact, err := cli.DownloadArtifact(a.Artifact)
+	artifact, err = cli.DownloadArtifact(a.Artifact)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error on download artifact")
 	}
@@ -893,7 +1064,7 @@ func (l *LuetInstaller) downloadPackage(a ArtifactMatch, ctx *types.Context) (*a
 
 func (l *LuetInstaller) installPackage(m ArtifactMatch, s *System) error {
 
-	a, err := l.downloadPackage(m, l.Options.Context)
+	a, err := l.getPackage(m, l.Options.Context)
 	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Failed downloading package")
 	}
@@ -913,32 +1084,34 @@ func (l *LuetInstaller) installPackage(m ArtifactMatch, s *System) error {
 	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: m.Package.GetFingerPrint(), Files: files})
 }
 
-func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, ctx *types.Context) error {
+func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, pb *pterm.ProgressbarPrinter, c <-chan ArtifactMatch, ctx *types.Context) error {
 	defer wg.Done()
 
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
-		_, err := l.downloadPackage(p, ctx)
+		_, err := l.getPackage(p, ctx)
 		if err != nil {
 			l.Options.Context.Error("Failed downloading package "+p.Package.GetName(), err.Error())
 			return errors.Wrap(err, "Failed downloading package "+p.Package.GetName())
 		} else {
 			l.Options.Context.Success(":package: Package ", p.Package.HumanReadableString(), "downloaded")
 		}
-		if ctx.ProgressBar != nil {
-			ctx.ProgressBar.Increment()
+		if pb != nil {
+			pb.Increment()
 		}
 	}
 
 	return nil
 }
 
-func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System) error {
+func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, installLock *sync.Mutex, c <-chan ArtifactMatch, s *System) error {
 	defer wg.Done()
 
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
+		installLock.Lock()
 		err := l.installPackage(p, s)
+		installLock.Unlock()
 		if err != nil && !l.Options.Force {
 			//TODO: Uninstall, rollback.
 			l.Options.Context.Fatal("Failed installing package "+p.Package.GetName(), err.Error())

pkg/installer/installer_test.go

@@ -27,7 +27,6 @@ import (
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
-	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	. "github.com/mudler/luet/pkg/installer"
@@ -94,7 +93,7 @@ var _ = Describe("Installer", func() {
 			a, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(a.Path)).To(BeTrue())
-			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(a.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -211,7 +210,7 @@ urls:
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -334,7 +333,7 @@ urls:
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -463,7 +462,7 @@ urls:
 			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -655,6 +654,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			inst := NewLuetInstaller(LuetInstallerOptions{
 				Concurrency: 1, Context: ctx,
+				Relaxed:             true,
 				PackageRepositories: types.LuetRepositories{*repo2.LuetRepository},
 			})
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
@@ -703,6 +703,254 @@ urls:
 
 		})
 
+		It("Compute the correct upgrade order", func() {
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/upgrade_complex")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec4, err := c.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec5, err := c.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+			spec4.SetOutputPath(tmpdir)
+			spec5.SetOutputPath(tmpdir)
+
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec4, spec5))
+
+			Expect(errs).To(BeEmpty())
+
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_complex")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(ctx, tmpdir, false, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+enable: true
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			inst := NewLuetInstaller(LuetInstallerOptions{
+				Concurrency: 1, Context: ctx,
+				Relaxed:             true,
+				PackageRepositories: types.LuetRepositories{*repo2.LuetRepository},
+			})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.GetPackages())).To(Equal(1))
+			p, err := system.Database.GetPackage(system.Database.GetPackages()[0])
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetName()).To(Equal("b"))
+
+			files, err := systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(files).To(Equal([]string{"test5", "test6"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			files, err = systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"})
+			Expect(files).To(Equal([]string{"test3", "test4"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test4"))).To(BeTrue())
+
+			err = inst.Upgrade(system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+
+		})
+
+		It("Compute the correct upgrade order with a package replacing multiple ones", func() {
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/upgrade_complex_multiple")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(6))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec3, err := c.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec4, err := c.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec5, err := c.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+			spec6, err := c.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+			spec4.SetOutputPath(tmpdir)
+			spec5.SetOutputPath(tmpdir)
+			spec3.SetOutputPath(tmpdir)
+			spec6.SetOutputPath(tmpdir)
+
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3, spec4, spec5, spec6))
+
+			Expect(errs).To(BeEmpty())
+
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_complex_multiple")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(ctx, tmpdir, false, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+enable: true
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			inst := NewLuetInstaller(LuetInstallerOptions{
+				Concurrency: 1, Context: ctx,
+				Relaxed:             true,
+				PackageRepositories: types.LuetRepositories{*repo2.LuetRepository},
+			})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.GetPackages())).To(Equal(1))
+			p, err := system.Database.GetPackage(system.Database.GetPackages()[0])
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetName()).To(Equal("b"))
+
+			files, err := systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(files).To(Equal([]string{"test5", "test6"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.1"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test1"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test2"))).To(BeTrue())
+
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			files, err = systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"})
+			Expect(files).To(Equal([]string{"test3", "test4"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test4"))).To(BeTrue())
+
+			err = inst.Upgrade(system)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test1"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test2"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+
+		})
+
 		It("Handles package drops", func() {
 			//repo:=NewLuetSystemRepository()
 

pkg/installer/interface.go

@@ -23,6 +23,7 @@ import (
 type Client interface {
 	DownloadArtifact(*artifact.PackageArtifact) (*artifact.PackageArtifact, error)
 	DownloadFile(string) (string, error)
+	CacheGet(*artifact.PackageArtifact) (*artifact.PackageArtifact, error)
 }
 
 type Repositories []*LuetSystemRepository

pkg/installer/repository.go

@@ -74,7 +74,7 @@ type LuetSystemRepository struct {
 	PushImages      bool                          `json:"-"`
 	ForcePush       bool                          `json:"-"`
 
-	imagePrefix string
+	imagePrefix, snapshotID string
 }
 
 type LuetSystemRepositoryMetadata struct {
@@ -379,7 +379,7 @@ func (r *LuetSystemRepository) SetPriority(n int) {
 }
 
 func (r *LuetSystemRepository) initialize(ctx *types.Context, src string) error {
-	generator, err := r.getGenerator(ctx)
+	generator, err := r.getGenerator(ctx, r.snapshotID)
 	if err != nil {
 		return errors.Wrap(err, "while constructing repository generator")
 	}
@@ -430,6 +430,11 @@ func (r *LuetSystemRepository) SetType(p string) {
 	r.LuetRepository.Type = p
 }
 
+// Sets snapshot ID
+func (r *LuetSystemRepository) SetSnapshotID(i string) {
+	r.snapshotID = i
+}
+
 func (r *LuetSystemRepository) GetVerify() bool {
 	return r.LuetRepository.Verify
 }
@@ -705,11 +710,15 @@ type RepositoryGenerator interface {
 	Initialize(string, pkg.PackageDatabase) ([]*artifact.PackageArtifact, error)
 }
 
-func (r *LuetSystemRepository) getGenerator(ctx *types.Context) (RepositoryGenerator, error) {
+func (r *LuetSystemRepository) getGenerator(ctx *types.Context, snapshotID string) (RepositoryGenerator, error) {
+	if snapshotID == "" {
+		snapshotID = time.Now().Format("20060102150405")
+	}
+
 	var rg RepositoryGenerator
 	switch r.GetType() {
 	case DiskRepositoryType, HttpRepositoryType:
-		rg = &localRepositoryGenerator{context: ctx}
+		rg = &localRepositoryGenerator{context: ctx, snapshotID: snapshotID}
 	case DockerRepositoryType:
 		rg = &dockerRepositoryGenerator{
 			b:           r.Backend,
@@ -717,6 +726,7 @@ func (r *LuetSystemRepository) getGenerator(ctx *types.Context) (RepositoryGener
 			imagePush:   r.PushImages,
 			force:       r.ForcePush,
 			context:     ctx,
+			snapshotID:  snapshotID,
 		}
 	default:
 		return nil, errors.New("invalid repository type")
@@ -726,7 +736,7 @@ func (r *LuetSystemRepository) getGenerator(ctx *types.Context) (RepositoryGener
 
 // Write writes the repository metadata to the supplied destination
 func (r *LuetSystemRepository) Write(ctx *types.Context, dst string, resetRevision, force bool) error {
-	rg, err := r.getGenerator(ctx)
+	rg, err := r.getGenerator(ctx, r.snapshotID)
 	if err != nil {
 		return err
 	}
@@ -812,6 +822,10 @@ func (r *LuetSystemRepository) SyncBuildMetadata(ctx *types.Context, path string
 
 	defer os.RemoveAll(a.Path)
 
+	if !fileHelper.Exists(filepath.Join(path, "tree")) {
+		os.MkdirAll(filepath.Join(path, "tree"), 0600)
+	}
+
 	if err := a.Unpack(ctx, filepath.Join(path, "tree"), false); err != nil {
 		return errors.Wrapf(err, "while unpacking: %s", REPOFILE_COMPILER_TREE_KEY)
 	}
@@ -842,7 +856,20 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 	var repoUpdated bool = false
 	var treefs, metafs string
 
+	repobasedir := ctx.Config.GetSystem().GetRepoDatabaseDirPath(r.GetName())
+
+	toTimeSync := false
+	dat, err := ioutil.ReadFile(filepath.Join(repobasedir, "SYNCTIME"))
+	if err == nil {
+		parsed, _ := time.Parse(time.RFC3339, string(dat))
+		if time.Now().After(parsed.Add(24 * time.Hour)) {
+			toTimeSync = true
+			ctx.Debug(r.Name, "is old, refresh is suggested")
+		}
+	}
+
 	ctx.Debug("Sync of the repository", r.Name, "in progress...")
+
 	c := r.Client(ctx)
 	if c == nil {
 		return nil, errors.New("no client could be generated from repository")
@@ -850,20 +877,29 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 
 	repositoryReferenceID := r.referenceID()
 
-	// Retrieve remote repository.yaml for retrieve revision and date
-	file, err := c.DownloadFile(repositoryReferenceID)
-	if err != nil {
-		return nil, errors.Wrap(err, "while downloading "+repositoryReferenceID)
-	}
+	var downloadedRepoMeta *LuetSystemRepository
+	var file string
+	repoFile := filepath.Join(repobasedir, repositoryReferenceID)
 
-	repobasedir := ctx.Config.GetSystem().GetRepoDatabaseDirPath(r.GetName())
-	downloadedRepoMeta, err := r.ReadSpecFile(file)
-	if err != nil {
-		return nil, err
+	_, repoExistsErr := os.Stat(repoFile)
+	if toTimeSync || force || os.IsNotExist(repoExistsErr) {
+		// Retrieve remote repository.yaml for retrieve revision and date
+		file, err = c.DownloadFile(repositoryReferenceID)
+		if err != nil {
+			return nil, errors.Wrap(err, "while downloading "+repositoryReferenceID)
+		}
+		downloadedRepoMeta, err = r.ReadSpecFile(file)
+		if err != nil {
+			return nil, err
+		}
+		defer os.RemoveAll(file)
+	} else {
+		downloadedRepoMeta, err = r.ReadSpecFile(repoFile)
+		if err != nil {
+			return nil, err
+		}
+		repoUpdated = true
 	}
-	// Remove temporary file that contains repository.yaml
-	// Example: /tmp/HttpClient236052003
-	defer os.RemoveAll(file)
 
 	if r.Cached {
 		if !force {
@@ -929,7 +965,11 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 		}
 		ctx.Debug("Decompress tree of the repository " + r.Name + "...")
 
-		err = treeFileArtifact.Unpack(ctx, treefs, true)
+		if _, err := os.Lstat(treefs); os.IsNotExist(err) {
+			os.MkdirAll(treefs, 0600)
+		}
+
+		err = treeFileArtifact.Unpack(ctx, treefs, false)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while unpacking tree")
 		}
@@ -937,7 +977,7 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 		// FIXME: It seems that tar with only one file doesn't create destination
 		//       directory. I create directory directly for now.
 		os.MkdirAll(metafs, os.ModePerm)
-		err = metaFileArtifact.Unpack(ctx, metafs, true)
+		err = metaFileArtifact.Unpack(ctx, metafs, false)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while unpacking metadata")
 		}
@@ -950,8 +990,8 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 				downloadedRepoMeta.GetRevision(),
 				time.Unix(tsec, 0).String()))
 
-	} else {
-		ctx.Info("Repository", downloadedRepoMeta.GetName(), "is already up to date.")
+		now := time.Now().Format(time.RFC3339)
+		ioutil.WriteFile(filepath.Join(repobasedir, "SYNCTIME"), []byte(now), os.ModePerm)
 	}
 
 	meta, err := NewLuetSystemRepositoryMetadata(
@@ -975,11 +1015,14 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 	// e.g. locally we can override the type (disk), or priority
 	// while remotely it could be advertized differently
 	r.fill(downloadedRepoMeta)
-	ctx.Info(
-		fmt.Sprintf(":information_source: Repository: %s Priority: %d Type: %s",
-			downloadedRepoMeta.GetName(),
-			downloadedRepoMeta.GetPriority(),
-			downloadedRepoMeta.GetType()))
+
+	if !repoUpdated {
+		ctx.Info(
+			fmt.Sprintf(":information_source: Repository: %s Priority: %d Type: %s",
+				downloadedRepoMeta.GetName(),
+				downloadedRepoMeta.GetPriority(),
+				downloadedRepoMeta.GetType()))
+	}
 	return downloadedRepoMeta, nil
 }
 

pkg/installer/repository_docker.go

@@ -24,23 +24,23 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mudler/luet/pkg/api/core/bus"
+	"github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
-	"github.com/mudler/luet/pkg/bus"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/helpers"
-	"github.com/mudler/luet/pkg/helpers/docker"
 	pkg "github.com/mudler/luet/pkg/package"
 
 	"github.com/pkg/errors"
 )
 
 type dockerRepositoryGenerator struct {
-	b                compiler.CompilerBackend
-	imagePrefix      string
-	imagePush, force bool
-	context          *types.Context
+	b                       compiler.CompilerBackend
+	imagePrefix, snapshotID string
+	imagePush, force        bool
+	context                 *types.Context
 }
 
 func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
@@ -92,8 +92,8 @@ func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDataba
 		} else {
 			l.context.Info("Generating final image", packageImage,
 				"for package ", a.CompileSpec.GetPackage().HumanReadableString())
-			if opts, err := a.GenerateFinalImage(l.context, packageImage, l.b, true); err != nil {
-				return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+			if err := a.GenerateFinalImage(l.context, packageImage, l.b, true); err != nil {
+				return errors.Wrap(err, "Failed generating metadata tree"+packageImage)
 			}
 		}
 		if l.imagePush {
@@ -125,8 +125,8 @@ func pushImage(ctx *types.Context, b compiler.CompilerBackend, image string, for
 
 func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string) error {
 	d.context.Debug("Generating image", imageTree)
-	if opts, err := a.GenerateFinalImage(d.context, imageTree, d.b, false); err != nil {
-		return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
+	if err := a.GenerateFinalImage(d.context, imageTree, d.b, false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree "+imageTree)
 	}
 	if d.imagePush {
 		if err := pushImage(d.context, d.b, imageTree, true); err != nil {
@@ -165,7 +165,7 @@ func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArt
 	if err != nil {
 		return errors.Wrap(err, "failed generating checksums for tree")
 	}
-	imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, docker.StripInvalidStringsFromImage(a.GetFileName()))
+	imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, helpers.SanitizeImageString(a.GetFileName()))
 	if checkIfExists && d.imagePush && d.b.ImageAvailable(imageTree) && !d.force {
 		d.context.Info("Image", imageTree, "already present, skipping. use --force-push to override")
 		return nil
@@ -191,13 +191,25 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 	defer os.RemoveAll(repoTemp) // clean up
 
 	if r.GetBackend().ImageAvailable(imageRepository) {
-		if err := r.GetBackend().DownloadImage(backend.Options{ImageName: imageRepository}); err != nil {
+
+		err := r.GetBackend().DownloadImage(backend.Options{ImageName: imageRepository})
+		if err != nil {
 			return errors.Wrapf(err, "while downloading '%s'", imageRepository)
 		}
-
-		if err := r.GetBackend().ExtractRootfs(backend.Options{ImageName: imageRepository, Destination: repoTemp}, false); err != nil {
+		img, err := r.GetBackend().ImageReference(imageRepository, true)
+		if err != nil {
+			return errors.Wrapf(err, "while downloading '%s'", imageRepository)
+		}
+		_, _, err = image.ExtractTo(
+			d.context,
+			img,
+			repoTemp,
+			nil,
+		)
+		if err != nil {
 			return errors.Wrapf(err, "while extracting '%s'", imageRepository)
 		}
+
 	}
 
 	repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE)
@@ -258,8 +270,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 	// Create a named snapshot and push it.
 	// It edits the metadata pointing at the repository files associated with the snapshot
 	// And copies the new files
-	id := time.Now().Format("20060102150405")
-	artifacts, snapshotRepoFile, err := r.Snapshot(id, repoTemp)
+	artifacts, snapshotRepoFile, err := r.Snapshot(d.snapshotID, repoTemp)
 	if err != nil {
 		return errors.Wrap(err, "while creating snapshot")
 	}

pkg/installer/repository_local.go

@@ -28,11 +28,14 @@ import (
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	pkg "github.com/mudler/luet/pkg/package"
 
-	"github.com/mudler/luet/pkg/bus"
+	"github.com/mudler/luet/pkg/api/core/bus"
 	"github.com/pkg/errors"
 )
 
-type localRepositoryGenerator struct{ context *types.Context }
+type localRepositoryGenerator struct {
+	context    *types.Context
+	snapshotID string
+}
 
 func (l *localRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
 	return buildPackageIndex(l.context, path, db)
@@ -124,7 +127,7 @@ func (g *localRepositoryGenerator) Generate(r *LuetSystemRepository, dst string,
 	// Create named snapshot.
 	// It edits the metadata pointing at the repository files associated with the snapshot
 	// And copies the new files
-	if _, _, err := r.Snapshot(time.Now().Format("20060102150405"), dst); err != nil {
+	if _, _, err := r.Snapshot(g.snapshotID, dst); err != nil {
 		return errors.Wrap(err, "while creating snapshot")
 	}
 

pkg/installer/repository_test.go

@@ -30,7 +30,6 @@ import (
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
-	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -92,7 +91,7 @@ var _ = Describe("Repository", func() {
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -165,12 +164,12 @@ var _ = Describe("Repository", func() {
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			artifact2, err := compiler2.Compile(false, spec2)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact2.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact2.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact2.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -262,12 +261,12 @@ urls:
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			artifact2, err := compiler2.Compile(false, spec2)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact2.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact2.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact2.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -380,12 +379,12 @@ urls:
 			artifact, err := compiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			artifact2, err := compiler2.Compile(false, spec2)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(artifact2.Path)).To(BeTrue())
-			Expect(helpers.Untar(artifact2.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(artifact2.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -505,7 +504,7 @@ urls:
 			a, err := localcompiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(a.Path)).To(BeTrue())
-			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(a.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -580,7 +579,7 @@ urls:
 			a, err := localcompiler.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(a.Path)).To(BeTrue())
-			Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(a.Unpack(ctx, tmpdir, false)).ToNot(HaveOccurred())
 
 			repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/virtuals", repoImage, true, true)
 			Expect(err).ToNot(HaveOccurred())
@@ -620,11 +619,8 @@ urls:
 
 			Expect(a.Unpack(ctx, extracted, false)).ToNot(HaveOccurred())
 
-			Expect(fileHelper.DirectoryIsEmpty(extracted)).To(BeFalse())
-			content, err := ioutil.ReadFile(filepath.Join(extracted, ".virtual"))
-			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.DirectoryIsEmpty(extracted)).To(BeTrue())
 
-			Expect(string(content)).To(Equal(""))
 		})
 
 		It("Searches files", func() {

pkg/installer/system.go

@@ -1,6 +1,8 @@
 package installer
 
 import (
+	"os"
+	"path/filepath"
 	"sync"
 
 	"github.com/hashicorp/go-multierror"
@@ -12,9 +14,10 @@ import (
 )
 
 type System struct {
-	Database  pkg.PackageDatabase
-	Target    string
-	fileIndex map[string]pkg.Package
+	Database          pkg.PackageDatabase
+	Target            string
+	fileIndex         map[string]pkg.Package
+	fileIndexPackages map[string]pkg.Package
 	sync.Mutex
 }
 
@@ -22,6 +25,21 @@ func (s *System) World() (pkg.Packages, error) {
 	return s.Database.World(), nil
 }
 
+func (s *System) OSCheck() (notFound pkg.Packages) {
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	for f, p := range s.fileIndex {
+		if _, err := os.Lstat(filepath.Join(s.Target, f)); err != nil {
+			if _, err := s.Database.FindPackage(p); err == nil {
+				notFound = append(notFound, p)
+			}
+		}
+	}
+	notFound = notFound.Unique()
+	return
+}
+
 func (s *System) ExecuteFinalizers(ctx *types.Context, packs []pkg.Package) error {
 	var errs error
 	executedFinalizer := map[string]bool{}
@@ -56,16 +74,27 @@ func (s *System) ExecuteFinalizers(ctx *types.Context, packs []pkg.Package) erro
 }
 
 func (s *System) buildFileIndex() {
+	// XXX: Replace with cache
 	s.Lock()
 	defer s.Unlock()
-	// Check if cache is empty or if it got modified
-	if s.fileIndex == nil { //|| len(s.Database.GetPackages()) != len(s.fileIndex) {
+
+	if s.fileIndex == nil {
 		s.fileIndex = make(map[string]pkg.Package)
+	}
+
+	if s.fileIndexPackages == nil {
+		s.fileIndexPackages = make(map[string]pkg.Package)
+	}
+
+	// Check if cache is empty or if it got modified
+	if len(s.Database.GetPackages()) != len(s.fileIndexPackages) {
+		s.fileIndexPackages = make(map[string]pkg.Package)
 		for _, p := range s.Database.World() {
 			files, _ := s.Database.GetPackageFiles(p)
 			for _, f := range files {
 				s.fileIndex[f] = p
 			}
+			s.fileIndexPackages[p.GetPackageName()] = p
 		}
 	}
 }
@@ -76,6 +105,13 @@ func (s *System) Clean() {
 	s.fileIndex = nil
 }
 
+func (s *System) FileIndex() map[string]pkg.Package {
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	return s.fileIndex
+}
+
 func (s *System) ExistsPackageFile(file string) (bool, pkg.Package, error) {
 	s.buildFileIndex()
 	s.Lock()

pkg/installer/system_test.go

@@ -19,6 +19,10 @@ import (
 
 	//	. "github.com/mudler/luet/pkg/installer"
 
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -66,5 +70,18 @@ var _ = Describe("System", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p).To(Equal(b))
 		})
+
+		It("detect missing files", func() {
+			dir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(dir)
+			s.Target = dir
+			notfound := s.OSCheck()
+			Expect(len(notfound)).To(Equal(2))
+			ioutil.WriteFile(filepath.Join(dir, "f"), []byte{}, os.ModePerm)
+			ioutil.WriteFile(filepath.Join(dir, "foo"), []byte{}, os.ModePerm)
+			notfound = s.OSCheck()
+			Expect(len(notfound)).To(Equal(1))
+		})
 	})
 })

pkg/package/package.go

@@ -27,9 +27,9 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
-	"github.com/mudler/luet/pkg/helpers/docker"
 	"github.com/mudler/luet/pkg/helpers/match"
 	version "github.com/mudler/luet/pkg/versioner"
 
@@ -350,7 +350,7 @@ func (p *DefaultPackage) GetPackageName() string {
 }
 
 func (p *DefaultPackage) ImageID() string {
-	return docker.StripInvalidStringsFromImage(p.GetFingerPrint())
+	return helpers.SanitizeImageString(p.GetFingerPrint())
 }
 
 // GetBuildTimestamp returns the package build timestamp

pkg/tree/compiler_recipe.go

@@ -174,6 +174,7 @@ func (r *CompilerRecipe) Load(path string) error {
 								filepath.Dir(currentpath))
 					}
 					pack.Requires(packbuild.GetRequires())
+
 					pack.Conflicts(packbuild.GetConflicts())
 				}
 

scripts/ginkgo.coverage.sh

@@ -23,7 +23,7 @@ coveragetxt="coverage.txt"
 
 
 generate_cover_data() {
-  ginkgo -flakeAttempts=3 -race -failFast -cover -r .
+  ginkgo -flakeAttempts=3 -failFast -race -cover -r .
   echo "" > ${coveragetxt}
   find . -type f -name "*.coverprofile" | while read -r file;  do cat "$file" >> ${coveragetxt} && mv "$file" "${coverdir}"; done
   echo "mode: $covermode" >"$profile"

tests/fixtures/caps/pkgC/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "empty"
+version: "0.1"

tests/fixtures/extra_perms/pkgA/0.1/build.yaml

@@ -0,0 +1,18 @@
+image: "alpine"
+unpack: true
+includes:
+  - /foo
+  - /foo/bar
+  - /foo/bar/suid
+  - /foo/bar/sticky
+  - /foo/bar/sgid
+steps:
+- mkdir -p /foo/bar
+- touch /foo/bar/suid
+- touch /foo/bar/sgid
+- touch /foo/bar/sticky
+- chown 100:100 /foo/bar
+- chown 101:101 /foo/bar/suid
+- chmod u+s /foo/bar/suid
+- chmod u-s,g+s /foo/bar/sgid
+- chmod +t /foo/bar/sticky

tests/fixtures/extra_perms/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "extra-perms"
+version: "0.1"

tests/fixtures/join_complex/c/c1/build.yaml

@@ -6,4 +6,8 @@ requires:
     category: "test"
     version: ">=0"
 
-requires_final_images: true
+requires_final_images: true
+
+steps:
+- | 
+   /bin/sh -c "if [ -e /usr/bin/generate.sh ]; then ls -liah /usr/bin && exit 1; fi"

tests/fixtures/join_complex/cat/b-1.1/build.yaml

@@ -6,8 +6,9 @@ requires:
 prelude:
   - echo foo > /test
   - echo bar > /test2
+  - cp -rf generate.sh /usr/bin/
 steps:
   - echo artifact5 > /newc
   - echo artifact6 > /newnewc
   - chmod +x generate.sh
-  - ./generate.sh
+  - ./generate.sh

tests/fixtures/upgrade_complex/cat/a/a-1.2/build.yaml

@@ -0,0 +1,4 @@
+image: "alpine"
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6

tests/fixtures/upgrade_complex/cat/a/a-1.2/definition.yaml

@@ -0,0 +1,4 @@
+category: "test"
+name: "a"
+version: "1.2"
+

tests/fixtures/upgrade_complex/cat/a/a/build.yaml

@@ -0,0 +1,4 @@
+image: "alpine"
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/upgrade_complex/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.1"

tests/fixtures/upgrade_complex/cat/b-1.1/build.yaml

@@ -0,0 +1,4 @@
+image: "alpine"
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/upgrade_complex/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/upgrade_complex/cat/b/build.yaml

@@ -0,0 +1,4 @@
+image: "alpine"
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6

tests/fixtures/upgrade_complex/cat/b/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/fixtures/upgrade_complex_multiple/cat/a/a-1.2/build.yaml

@@ -0,0 +1,5 @@
+image: "alpine"
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6
+  - echo artifact5 > /test2

tests/fixtures/upgrade_complex_multiple/cat/a/a-1.2/definition.yaml

@@ -0,0 +1,4 @@
+category: "test"
+name: "a"
+version: "1.2"
+

tests/fixtures/upgrade_complex_multiple/cat/a/a/build.yaml

@@ -0,0 +1,4 @@
+image: "alpine"
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/upgrade_complex_multiple/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.1"

tests/fixtures/upgrade_complex_multiple/cat/b-1.1/build.yaml

@@ -0,0 +1,4 @@
+image: "alpine"
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/upgrade_complex_multiple/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/upgrade_complex_multiple/cat/b/build.yaml

@@ -0,0 +1,4 @@
+image: "alpine"
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6

tests/fixtures/upgrade_complex_multiple/cat/b/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/fixtures/upgrade_complex_multiple/cat/c/c-1.2/build.yaml

@@ -0,0 +1,3 @@
+image: "alpine"
+steps:
+  - echo artifact5 > /test1

tests/fixtures/upgrade_complex_multiple/cat/c/c-1.2/definition.yaml

@@ -0,0 +1,4 @@
+category: "test"
+name: "c"
+version: "1.2"
+

tests/fixtures/upgrade_complex_multiple/cat/c/c/build.yaml

@@ -0,0 +1,4 @@
+image: "alpine"
+steps:
+  - echo artifact3 > /test1
+  - echo artifact4 > /test2

tests/fixtures/upgrade_complex_multiple/cat/c/c/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.1"

tests/fixtures/upgrade_integration_oscheck/c/build.yaml

@@ -0,0 +1,10 @@
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo c > /c
+  - echo c > /cd
+requires:
+- category: "test"
+  name: "a"
+  version: ">=1.0"

tests/fixtures/upgrade_integration_oscheck/c/definition.yaml

@@ -0,0 +1,9 @@
+category: "test"
+name: "c"
+version: "1.0"
+# Boom?
+
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0.1"

tests/fixtures/upgrade_integration_oscheck/cat/a/a/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/upgrade_integration_oscheck/cat/a/a/definition.yaml

@@ -0,0 +1,8 @@
+category: "test"
+name: "a"
+version: "1.1"
+requires:
+- category: "test"
+  name: "b"
+  version: ">=0.1"
+

tests/fixtures/upgrade_integration_oscheck/cat/a/a1.0/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /testaa
+  - echo artifact4 > /testaa2
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/upgrade_integration_oscheck/cat/a/a1.0/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.0"

tests/fixtures/upgrade_integration_oscheck/cat/a/alatest/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /testlatest
+  - echo artifact4 > /testlatest2
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/upgrade_integration_oscheck/cat/a/alatest/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/upgrade_integration_oscheck/cat/b-1.1/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/upgrade_integration_oscheck/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"