Tag 0.22.0

Now we can just remove the necessary files and let the installation
handle the rest

CONTRIBUTING.md

@@ -17,7 +17,7 @@ Join us in [slack](https://luet.slack.com/join/shared_invite/enQtOTQxMjcyNDQ0MDU
 ## All Code Changes Happen Through Pull Requests
 Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests:
 
-1. Fork the repo you want to contribute to and create your branch from `develop`.
+1. Fork the repo you want to contribute to and create your branch from `master`.
 2. If you've added code that should be tested, add tests.
 3. If you've changed APIs, update the [documentation](https://github.com/Luet-lab/docs).
 4. Ensure the test suite passes.

cmd/build.go

@@ -31,7 +31,6 @@ import (
 	"github.com/mudler/luet/pkg/compiler/types/options"
 	fileHelpers "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/spf13/cobra"
@@ -39,6 +38,10 @@ import (
 )
 
 var buildCmd = &cobra.Command{
+	// Skip processing output
+	Annotations: map[string]string{
+		util.CommandProcessOutput: "",
+	},
 	Use:   "build <package name> <package name> <package name> ...",
 	Short: "build a package or a tree",
 	Long: `Builds one or more packages from a tree (current directory is implied):
@@ -83,8 +86,6 @@ Build packages specifying multiple definition trees:
 		viper.BindPFlag("wait", cmd.Flags().Lookup("wait"))
 		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
 
-		util.BindSolverFlags(cmd)
-
 		viper.BindPFlag("general.show_build_output", cmd.Flags().Lookup("live-output"))
 		viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
 
@@ -93,7 +94,7 @@ Build packages specifying multiple definition trees:
 
 		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("destination")
-		concurrency := util.DefaultContext.Config.GetGeneral().Concurrency
+		concurrency := util.DefaultContext.Config.General.Concurrency
 		backendType := viper.GetString("backend")
 		privileged := viper.GetBool("privileged")
 		revdeps := viper.GetBool("revdeps")
@@ -110,14 +111,13 @@ Build packages specifying multiple definition trees:
 		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
 		full, _ := cmd.Flags().GetBool("full")
 		rebuild, _ := cmd.Flags().GetBool("rebuild")
+		pushFinalImages, _ := cmd.Flags().GetBool("push-final-images")
+		pushFinalImagesRepository, _ := cmd.Flags().GetString("push-final-images-repository")
+		pushFinalImagesForce, _ := cmd.Flags().GetBool("push-final-images-force")
 
 		var results Results
 		backendArgs := viper.GetStringSlice("backend-args")
-
 		out, _ := cmd.Flags().GetString("output")
-		if out != "terminal" {
-			util.DefaultContext.Config.GetLogging().SetLogLevel("error")
-		}
 		pretend, _ := cmd.Flags().GetBool("pretend")
 		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
 
@@ -150,17 +150,12 @@ Build packages specifying multiple definition trees:
 			util.DefaultContext.Debug("Creating destination folder", dst)
 		}
 
-		opts := util.SetSolverConfig(util.DefaultContext)
+		opts := util.DefaultContext.GetConfig().Solver
 		pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
-		util.DefaultContext.Config.GetGeneral().ShowBuildOutput = viper.GetBool("general.show_build_output")
-
 		util.DefaultContext.Debug("Solver", opts.CompactString())
 
-		opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
-
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
-			options.NoDeps(nodeps),
+		compileropts := []options.Option{options.NoDeps(nodeps),
 			options.WithBackendType(backendType),
 			options.PushImages(push),
 			options.WithBuildValues(values),
@@ -168,7 +163,7 @@ Build packages specifying multiple definition trees:
 			options.WithPushRepository(imageRepository),
 			options.Rebuild(rebuild),
 			options.WithTemplateFolder(util.TemplateFolders(util.DefaultContext, fromRepo, treePaths)),
-			options.WithSolverOptions(*opts),
+			options.WithSolverOptions(opts),
 			options.Wait(wait),
 			options.OnlyTarget(onlyTarget),
 			options.PullFirst(pull),
@@ -177,8 +172,21 @@ Build packages specifying multiple definition trees:
 			options.WithContext(util.DefaultContext),
 			options.BackendArgs(backendArgs),
 			options.Concurrency(concurrency),
-			options.WithCompressionType(compression.Implementation(compressionType)),
-		)
+			options.WithCompressionType(compression.Implementation(compressionType))}
+
+		if pushFinalImages {
+			compileropts = append(compileropts, options.EnablePushFinalImages)
+			if pushFinalImagesForce {
+				compileropts = append(compileropts, options.ForcePushFinalImages)
+			}
+			if pushFinalImagesRepository != "" {
+				compileropts = append(compileropts, options.WithFinalRepository(pushFinalImagesRepository))
+			} else if imageRepository != "" {
+				compileropts = append(compileropts, options.WithFinalRepository(imageRepository))
+			}
+		}
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), compileropts...)
 
 		if full {
 			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
@@ -302,6 +310,11 @@ func init() {
 	buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
+
+	buildCmd.Flags().Bool("push-final-images", false, "Push final images while building")
+	buildCmd.Flags().Bool("push-final-images-force", false, "Override existing images")
+	buildCmd.Flags().String("push-final-images-repository", "", "Repository where to push final images to")
+
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
 	buildCmd.Flags().StringSlice("values", []string{}, "Build values file to interpolate with each package")
 	buildCmd.Flags().StringSliceP("backend-args", "a", []string{}, "Backend args")
@@ -321,7 +334,7 @@ func init() {
 	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
-	buildCmd.Flags().Bool("live-output", util.DefaultContext.Config.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
+	buildCmd.Flags().Bool("live-output", true, "Enable live output of the build phase.")
 	buildCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
 	buildCmd.Flags().Bool("rebuild", false, "To combine with --pull. Allows to rebuild the target package even if an image is available, against a local values file")
 	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")

cmd/cleanup.go

@@ -31,16 +31,13 @@ var cleanupCmd = &cobra.Command{
 	Use:   "cleanup",
 	Short: "Clean packages cache.",
 	Long:  `remove downloaded packages tarballs and clean cache directory`,
-	PreRun: func(cmd *cobra.Command, args []string) {
-		util.BindSystemFlags(cmd)
-	},
+
 	Run: func(cmd *cobra.Command, args []string) {
 		var cleaned int = 0
-		util.SetSystemConfig(util.DefaultContext)
 		// Check if cache dir exists
-		if fileHelper.Exists(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath()) {
+		if fileHelper.Exists(util.DefaultContext.Config.System.PkgsCachePath) {
 
-			files, err := ioutil.ReadDir(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath())
+			files, err := ioutil.ReadDir(util.DefaultContext.Config.System.PkgsCachePath)
 			if err != nil {
 				util.DefaultContext.Fatal("Error on read cachedir ", err.Error())
 			}
@@ -50,7 +47,7 @@ var cleanupCmd = &cobra.Command{
 				util.DefaultContext.Debug("Removing ", file.Name())
 
 				err := os.RemoveAll(
-					filepath.Join(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
+					filepath.Join(util.DefaultContext.Config.System.PkgsCachePath, file.Name()))
 				if err != nil {
 					util.DefaultContext.Fatal("Error on removing", file.Name())
 				}
@@ -58,14 +55,11 @@ var cleanupCmd = &cobra.Command{
 			}
 		}
 
-		util.DefaultContext.Info(fmt.Sprintf("Cleaned: %d files from %s", cleaned, util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath()))
+		util.DefaultContext.Info(fmt.Sprintf("Cleaned: %d files from %s", cleaned, util.DefaultContext.Config.System.PkgsCachePath))
 
 	},
 }
 
 func init() {
-	cleanupCmd.Flags().String("system-dbpath", "", "System db path")
-	cleanupCmd.Flags().String("system-target", "", "System rootpath")
-	cleanupCmd.Flags().String("system-engine", "", "System DB engine")
 	RootCmd.AddCommand(cleanupCmd)
 }

cmd/create-repo.go

@@ -100,6 +100,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		helpers.CheckErr(err)
 		force := viper.GetBool("force-push")
 		imagePush := viper.GetBool("push-images")
+		snapshotID, _ := cmd.Flags().GetString("snapshot-id")
 
 		opts := []installer.RepositoryOption{
 			installer.WithSource(viper.GetString("packages")),
@@ -163,7 +164,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		if metaName != "" {
 			metaFile.SetFileName(metaName)
 		}
-
+		repo.SetSnapshotID(snapshotID)
 		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
@@ -197,6 +198,7 @@ func init() {
 	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
 	createrepoCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
+	createrepoCmd.Flags().String("snapshot-id", "", "Unique ID to use when creating repository snapshots")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

cmd/database/create.go

@@ -27,7 +27,7 @@ import (
 )
 
 func NewDatabaseCreateCommand() *cobra.Command {
-	var ans = &cobra.Command{
+	return &cobra.Command{
 		Use:   "create <artifact_metadata1.yaml> <artifact_metadata1.yaml>",
 		Short: "Insert a package in the system DB",
 		Long: `Inserts a package in the system database:
@@ -42,12 +42,8 @@ The yaml must contain the package definition, and the file list at least.
 
 For reference, inspect a "metadata.yaml" file generated while running "luet build"`,
 		Args: cobra.OnlyValidArgs,
-		PreRun: func(cmd *cobra.Command, args []string) {
-			util.BindSystemFlags(cmd)
-		},
 		Run: func(cmd *cobra.Command, args []string) {
 
-			util.SetSystemConfig(util.DefaultContext)
 			systemDB := util.DefaultContext.Config.GetSystemDB()
 
 			for _, a := range args {
@@ -81,9 +77,4 @@ For reference, inspect a "metadata.yaml" file generated while running "luet buil
 		},
 	}
 
-	ans.Flags().String("system-dbpath", "", "System db path")
-	ans.Flags().String("system-target", "", "System rootpath")
-	ans.Flags().String("system-engine", "", "System DB engine")
-
-	return ans
 }

cmd/database/get.go

@@ -36,12 +36,9 @@ func NewDatabaseGetCommand() *cobra.Command {
 To return also files:
 		$ luet database get --files system/foo`,
 		Args: cobra.OnlyValidArgs,
-		PreRun: func(cmd *cobra.Command, args []string) {
-			util.BindSystemFlags(cmd)
-		},
+
 		Run: func(cmd *cobra.Command, args []string) {
 			showFiles, _ := cmd.Flags().GetBool("files")
-			util.SetSystemConfig(util.DefaultContext)
 
 			systemDB := util.DefaultContext.Config.GetSystemDB()
 
@@ -77,9 +74,6 @@ To return also files:
 		},
 	}
 	c.Flags().Bool("files", false, "Show package files.")
-	c.Flags().String("system-dbpath", "", "System db path")
-	c.Flags().String("system-target", "", "System rootpath")
-	c.Flags().String("system-engine", "", "System DB engine")
 
 	return c
 }

cmd/database/remove.go

@@ -23,7 +23,7 @@ import (
 )
 
 func NewDatabaseRemoveCommand() *cobra.Command {
-	var ans = &cobra.Command{
+	return &cobra.Command{
 		Use:   "remove [package1] [package2] ...",
 		Short: "Remove a package from the system DB (forcefully - you normally don't want to do that)",
 		Long: `Removes a package in the system database without actually uninstalling it:
@@ -33,11 +33,8 @@ func NewDatabaseRemoveCommand() *cobra.Command {
 This commands takes multiple packages as arguments and prunes their entries from the system database.
 `,
 		Args: cobra.OnlyValidArgs,
-		PreRun: func(cmd *cobra.Command, args []string) {
-			util.BindSystemFlags(cmd)
-		},
+
 		Run: func(cmd *cobra.Command, args []string) {
-			util.SetSystemConfig(util.DefaultContext)
 
 			systemDB := util.DefaultContext.Config.GetSystemDB()
 
@@ -58,9 +55,5 @@ This commands takes multiple packages as arguments and prunes their entries from
 
 		},
 	}
-	ans.Flags().String("system-dbpath", "", "System db path")
-	ans.Flags().String("system-target", "", "System rootpath")
-	ans.Flags().String("system-engine", "", "System DB engine")
 
-	return ans
 }

cmd/install.go

@@ -15,9 +15,7 @@
 package cmd
 
 import (
-	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
-	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/cmd/util"
@@ -48,8 +46,6 @@ To force install a package:
 `,
 	Aliases: []string{"i"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		util.BindSystemFlags(cmd)
-		util.BindSolverFlags(cmd)
 		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
@@ -71,28 +67,13 @@ To force install a package:
 		onlydeps := viper.GetBool("onlydeps")
 		yes := viper.GetBool("yes")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
-		finalizerEnvs, _ := cmd.Flags().GetStringArray("finalizer-env")
 		relax, _ := cmd.Flags().GetBool("relax")
 
-		util.SetSystemConfig(util.DefaultContext)
-		util.SetSolverConfig(util.DefaultContext)
-
-		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
-
-		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
-
-		// Load config protect configs
-		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
-
-		// Load finalizer runtime environments
-		err := util.SetCliFinalizerEnvs(util.DefaultContext, finalizerEnvs)
-		if err != nil {
-			util.DefaultContext.Fatal(err.Error())
-		}
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.Solver.CompactString())
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
-			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			Concurrency:                 util.DefaultContext.Config.General.Concurrency,
+			SolverOptions:               util.DefaultContext.Config.Solver,
 			NoDeps:                      nodeps,
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
@@ -104,8 +85,11 @@ To force install a package:
 			Context:                     util.DefaultContext,
 		})
 
-		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
-		err = inst.Install(toInstall, system)
+		system := &installer.System{
+			Database: util.DefaultContext.Config.GetSystemDB(),
+			Target:   util.DefaultContext.Config.System.Rootfs,
+		}
+		err := inst.Install(toInstall, system)
 		if err != nil {
 			util.DefaultContext.Fatal("Error: " + err.Error())
 		}
@@ -113,14 +97,7 @@ To force install a package:
 }
 
 func init() {
-	installCmd.Flags().String("system-dbpath", "", "System db path")
-	installCmd.Flags().String("system-target", "", "System rootpath")
-	installCmd.Flags().String("system-engine", "", "System DB engine")
 
-	installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
-	installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
-	installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
-	installCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
 	installCmd.Flags().Bool("relax", false, "Relax installation constraints")
 

cmd/oscheck.go

@@ -0,0 +1,124 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	installer "github.com/mudler/luet/pkg/installer"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/mudler/luet/cmd/util"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var osCheckCmd = &cobra.Command{
+	Use:   "oscheck",
+	Short: "Checks packages integrity",
+	Long: `List packages that are installed in the system which files are missing in the system.
+
+	$ luet oscheck
+	
+To reinstall packages in the list:
+	
+	$ luet oscheck --reinstall
+`,
+	Aliases: []string{"i"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+
+		force := viper.GetBool("force")
+		onlydeps := viper.GetBool("onlydeps")
+		yes := viper.GetBool("yes")
+
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		system := &installer.System{
+			Database: util.DefaultContext.Config.GetSystemDB(),
+			Target:   util.DefaultContext.Config.System.Rootfs,
+		}
+		packs := system.OSCheck(util.DefaultContext)
+		if !util.DefaultContext.Config.General.Quiet {
+			if len(packs) == 0 {
+				util.DefaultContext.Success("All good!")
+				os.Exit(0)
+			} else {
+				util.DefaultContext.Info("Following packages are missing files or are incomplete:")
+				for _, p := range packs {
+					util.DefaultContext.Info(p.HumanReadableString())
+				}
+			}
+		} else {
+			var s []string
+			for _, p := range packs {
+				s = append(s, p.HumanReadableString())
+			}
+			fmt.Println(strings.Join(s, " "))
+		}
+
+		reinstall, _ := cmd.Flags().GetBool("reinstall")
+		if reinstall {
+
+			// Strip version for reinstall
+			toInstall := pkg.Packages{}
+			for _, p := range packs {
+				new := p.Clone()
+				new.SetVersion(">=0")
+				toInstall = append(toInstall, new)
+			}
+
+			util.DefaultContext.Debug("Solver", util.DefaultContext.Config.Solver.CompactString())
+
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+				Concurrency:                 util.DefaultContext.Config.General.Concurrency,
+				SolverOptions:               util.DefaultContext.Config.Solver,
+				NoDeps:                      true,
+				Force:                       force,
+				OnlyDeps:                    onlydeps,
+				PreserveSystemEssentialData: true,
+				Ask:                         !yes,
+				DownloadOnly:                downloadOnly,
+				Context:                     util.DefaultContext,
+				PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			})
+
+			err := inst.Swap(packs, toInstall, system)
+			if err != nil {
+				util.DefaultContext.Fatal("Error: " + err.Error())
+			}
+		}
+	},
+}
+
+func init() {
+
+	osCheckCmd.Flags().Bool("reinstall", false, "reinstall")
+
+	osCheckCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	osCheckCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	osCheckCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	osCheckCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(osCheckCmd)
+}

cmd/pack.go

@@ -52,7 +52,7 @@ Afterwards, you can use the content generated and associate it with a tree and a
 
 		dst := viper.GetString("destination")
 		compressionType := viper.GetString("compression")
-		concurrency := util.DefaultContext.Config.GetGeneral().Concurrency
+		concurrency := util.DefaultContext.Config.General.Concurrency
 
 		if len(args) != 1 {
 			util.DefaultContext.Fatal("You must specify a package name")

cmd/reclaim.go

@@ -26,7 +26,6 @@ var reclaimCmd = &cobra.Command{
 	Use:   "reclaim",
 	Short: "Reclaim packages to Luet database from available repositories",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		util.BindSystemFlags(cmd)
 		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Long: `Reclaim tries to find association between packages in the online repositories and the system one.
@@ -36,21 +35,23 @@ var reclaimCmd = &cobra.Command{
 It scans the target file system, and if finds a match with a package available in the repositories, it marks as installed in the system database.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		util.SetSystemConfig(util.DefaultContext)
 
 		force := viper.GetBool("force")
 
-		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.Solver.CompactString())
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			Concurrency:                 util.DefaultContext.Config.General.Concurrency,
 			Force:                       force,
 			PreserveSystemEssentialData: true,
 			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
 			Context:                     util.DefaultContext,
 		})
 
-		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		system := &installer.System{
+			Database: util.DefaultContext.Config.GetSystemDB(),
+			Target:   util.DefaultContext.Config.System.Rootfs,
+		}
 		err := inst.Reclaim(system)
 		if err != nil {
 			util.DefaultContext.Fatal("Error: " + err.Error())
@@ -60,10 +61,6 @@ It scans the target file system, and if finds a match with a package available i
 
 func init() {
 
-	reclaimCmd.Flags().String("system-dbpath", "", "System db path")
-	reclaimCmd.Flags().String("system-target", "", "System rootpath")
-	reclaimCmd.Flags().String("system-engine", "", "System DB engine")
-
 	reclaimCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
 
 	RootCmd.AddCommand(reclaimCmd)

cmd/reinstall.go

@@ -15,9 +15,7 @@
 package cmd
 
 import (
-	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
-	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/cmd/util"
@@ -35,8 +33,6 @@ var reinstallCmd = &cobra.Command{
 	$ luet reinstall -y system/busybox shells/bash system/coreutils ...
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		util.BindSystemFlags(cmd)
-		util.BindSolverFlags(cmd)
 		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		viper.BindPFlag("for", cmd.Flags().Lookup("for"))
@@ -52,30 +48,13 @@ var reinstallCmd = &cobra.Command{
 		yes := viper.GetBool("yes")
 
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+		installed, _ := cmd.Flags().GetBool("installed")
 
-		util.SetSystemConfig(util.DefaultContext)
-
-		for _, a := range args {
-			pack, err := helpers.ParsePackageStr(a)
-			if err != nil {
-				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
-			}
-			toUninstall = append(toUninstall, pack)
-			toAdd = append(toAdd, pack)
-		}
-
-		util.SetSolverConfig(util.DefaultContext)
-
-		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
-
-		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
-
-		// Load config protect configs
-		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.Solver.CompactString())
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
-			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			Concurrency:                 util.DefaultContext.Config.General.Concurrency,
+			SolverOptions:               util.DefaultContext.Config.Solver,
 			NoDeps:                      true,
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
@@ -86,7 +65,26 @@ var reinstallCmd = &cobra.Command{
 			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
 		})
 
-		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.System.Rootfs}
+
+		if installed {
+			for _, p := range system.Database.World() {
+				toUninstall = append(toUninstall, p)
+				c := p.Clone()
+				c.SetVersion(">=0")
+				toAdd = append(toAdd, c)
+			}
+		} else {
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
+				}
+				toUninstall = append(toUninstall, pack)
+				toAdd = append(toAdd, pack)
+			}
+		}
+
 		err := inst.Swap(toUninstall, toAdd, system)
 		if err != nil {
 			util.DefaultContext.Fatal("Error: " + err.Error())
@@ -95,18 +93,9 @@ var reinstallCmd = &cobra.Command{
 }
 
 func init() {
-
-	reinstallCmd.Flags().String("system-dbpath", "", "System db path")
-	reinstallCmd.Flags().String("system-target", "", "System rootpath")
-	reinstallCmd.Flags().String("system-engine", "", "System DB engine")
-
-	reinstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
-	reinstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
-	reinstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
-	reinstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	reinstallCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
 	reinstallCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
-	reinstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	reinstallCmd.Flags().Bool("installed", false, "Reinstall installed packages")
 	reinstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 	reinstallCmd.Flags().Bool("download-only", false, "Download only")
 

cmd/replace.go

@@ -15,7 +15,6 @@
 package cmd
 
 import (
-	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
 	"github.com/mudler/luet/pkg/solver"
 
@@ -37,8 +36,6 @@ var replaceCmd = &cobra.Command{
 	$ luet replace -y system/busybox ... --for shells/bash --for system/coreutils ...
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		util.BindSystemFlags(cmd)
-		util.BindSolverFlags(cmd)
 		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
@@ -57,8 +54,6 @@ var replaceCmd = &cobra.Command{
 		yes := viper.GetBool("yes")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		util.SetSystemConfig(util.DefaultContext)
-		util.SetSolverConfig(util.DefaultContext)
 		for _, a := range args {
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
@@ -75,16 +70,13 @@ var replaceCmd = &cobra.Command{
 			toAdd = append(toAdd, pack)
 		}
 
-		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		util.DefaultContext.Config.Solver.Implementation = solver.SingleCoreSimple
 
-		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
-
-		// Load config protect configs
-		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.Solver.CompactString())
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
-			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			Concurrency:                 util.DefaultContext.Config.General.Concurrency,
+			SolverOptions:               util.DefaultContext.Config.Solver,
 			NoDeps:                      nodeps,
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
@@ -95,7 +87,7 @@ var replaceCmd = &cobra.Command{
 			Context:                     util.DefaultContext,
 		})
 
-		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.System.Rootfs}
 		err := inst.Swap(toUninstall, toAdd, system)
 		if err != nil {
 			util.DefaultContext.Fatal("Error: " + err.Error())
@@ -105,18 +97,9 @@ var replaceCmd = &cobra.Command{
 
 func init() {
 
-	replaceCmd.Flags().String("system-dbpath", "", "System db path")
-	replaceCmd.Flags().String("system-target", "", "System rootpath")
-	replaceCmd.Flags().String("system-engine", "", "System DB engine")
-
-	replaceCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
-	replaceCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
-	replaceCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
-	replaceCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	replaceCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
 	replaceCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
 	replaceCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
-	replaceCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	replaceCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 	replaceCmd.Flags().StringSlice("for", []string{}, "Packages that has to be installed in place of others")
 	replaceCmd.Flags().Bool("download-only", false, "Download only")

cmd/repo/list.go

@@ -68,7 +68,7 @@ func NewRepoListCommand() *cobra.Command {
 						repoText = pterm.LightYellow(repo.Urls[0])
 					}
 
-					repobasedir := util.DefaultContext.Config.GetSystem().GetRepoDatabaseDirPath(repo.Name)
+					repobasedir := util.DefaultContext.Config.System.GetRepoDatabaseDirPath(repo.Name)
 					if repo.Cached {
 
 						r := installer.NewSystemRepository(repo)

cmd/repo/update.go

@@ -1,5 +1,6 @@
 // Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
 //                  Daniele Rondina <geaaru@sabayonlinux.org>
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -24,7 +25,7 @@ import (
 )
 
 func NewRepoUpdateCommand() *cobra.Command {
-	var ans = &cobra.Command{
+	var repoUpdate = &cobra.Command{
 		Use:   "update [repo1] [repo2] [OPTIONS]",
 		Short: "Update a specific cached repository or all cached repositories.",
 		Example: `
@@ -72,8 +73,8 @@ $> luet repo update repo1 repo2
 		},
 	}
 
-	ans.Flags().BoolP("ignore-errors", "i", false, "Ignore errors on sync repositories.")
-	ans.Flags().BoolP("force", "f", false, "Force resync.")
+	repoUpdate.Flags().BoolP("ignore-errors", "i", false, "Ignore errors on sync repositories.")
+	repoUpdate.Flags().BoolP("force", "f", true, "Force resync.")
 
-	return ans
+	return repoUpdate
 }

cmd/root.go

@@ -30,7 +30,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.20.1"
+	LuetCLIVersion = "0.22.0"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -81,19 +81,15 @@ To build a package, from a tree definition:
 `,
 	Version: version(),
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
-		err := util.InitContext(util.DefaultContext)
+		ctx, err := util.InitContext(cmd)
 		if err != nil {
-			util.DefaultContext.Error("failed to load configuration:", err.Error())
+			fmt.Println("failed to load configuration:", err.Error())
+			os.Exit(1)
 		}
-		util.DisplayVersionBanner(util.DefaultContext, util.IntroScreen, version, license)
 
-		// Initialize tmpdir prefix. TODO: Move this with LoadConfig
-		// directly on sub command to ensure the creation only when it's
-		// needed.
-		err = util.DefaultContext.Config.GetSystem().InitTmpDir()
-		if err != nil {
-			util.DefaultContext.Fatal("failed on init tmp basedir:", err.Error())
-		}
+		util.DefaultContext = ctx
+
+		util.DisplayVersionBanner(util.DefaultContext, util.IntroScreen, version, license)
 
 		viper.BindPFlag("plugin", cmd.Flags().Lookup("plugin"))
 
@@ -103,13 +99,13 @@ To build a package, from a tree definition:
 		if len(bus.Manager.Plugins) != 0 {
 			util.DefaultContext.Info(":lollipop:Enabled plugins:")
 			for _, p := range bus.Manager.Plugins {
-				util.DefaultContext.Info("\t:arrow_right:", p.Name)
+				util.DefaultContext.Info(fmt.Sprintf("\t:arrow_right: %s (at %s)", p.Name, p.Executable))
 			}
 		}
 	},
 	PersistentPostRun: func(cmd *cobra.Command, args []string) {
 		// Cleanup all tmp directories used by luet
-		err := util.DefaultContext.Config.GetSystem().CleanupTmpDir()
+		err := util.DefaultContext.Clean()
 		if err != nil {
 			util.DefaultContext.Warning("failed on cleanup tmpdir:", err.Error())
 		}
@@ -129,5 +125,5 @@ func Execute() {
 }
 
 func init() {
-	util.InitViper(util.DefaultContext, RootCmd)
+	util.InitViper(RootCmd)
 }

cmd/search.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/ghodss/yaml"
 	"github.com/mudler/luet/cmd/util"
-	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/pterm/pterm"
@@ -86,7 +85,7 @@ func packageToList(l *util.ListWriter, repo string, p pkg.Package) {
 func searchLocally(term string, l *util.ListWriter, t *util.TableWriter, label, labelMatch, revdeps, hidden bool) Results {
 	var results Results
 
-	system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+	system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.System.Rootfs}
 
 	var err error
 	iMatches := pkg.Packages{}
@@ -148,8 +147,8 @@ func searchOnline(term string, l *util.ListWriter, t *util.TableWriter, label, l
 
 	inst := installer.NewLuetInstaller(
 		installer.LuetInstallerOptions{
-			Concurrency:         util.DefaultContext.Config.GetGeneral().Concurrency,
-			SolverOptions:       *util.DefaultContext.Config.GetSolverOptions(),
+			Concurrency:         util.DefaultContext.Config.General.Concurrency,
+			SolverOptions:       util.DefaultContext.Config.Solver,
 			PackageRepositories: util.DefaultContext.Config.SystemRepositories,
 			Context:             util.DefaultContext,
 		},
@@ -238,8 +237,8 @@ func searchFiles(term string, l *util.ListWriter, t *util.TableWriter) Results {
 
 	inst := installer.NewLuetInstaller(
 		installer.LuetInstallerOptions{
-			Concurrency:         util.DefaultContext.Config.GetGeneral().Concurrency,
-			SolverOptions:       *util.DefaultContext.Config.GetSolverOptions(),
+			Concurrency:         util.DefaultContext.Config.General.Concurrency,
+			SolverOptions:       util.DefaultContext.Config.Solver,
 			PackageRepositories: util.DefaultContext.Config.SystemRepositories,
 			Context:             util.DefaultContext,
 		},
@@ -272,7 +271,11 @@ func searchFiles(term string, l *util.ListWriter, t *util.TableWriter) Results {
 }
 
 var searchCmd = &cobra.Command{
-	Use:   "search <term>",
+	Use: "search <term>",
+	// Skip processing output
+	Annotations: map[string]string{
+		util.CommandProcessOutput: "",
+	},
 	Short: "Search packages",
 	Long: `Search for installed and available packages
 	
@@ -309,8 +312,7 @@ Search can also return results in the terminal in different ways: as terminal ou
 `,
 	Aliases: []string{"s"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		util.BindSystemFlags(cmd)
-		util.BindSolverFlags(cmd)
+
 		viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
@@ -329,18 +331,11 @@ Search can also return results in the terminal in different ways: as terminal ou
 		tableMode, _ := cmd.Flags().GetBool("table")
 		files, _ := cmd.Flags().GetBool("files")
 
-		util.SetSystemConfig(util.DefaultContext)
-		util.SetSolverConfig(util.DefaultContext)
-
 		out, _ := cmd.Flags().GetString("output")
-		if out != "terminal" {
-			util.DefaultContext.Config.GetLogging().SetLogLevel("error")
-		}
-
 		l := &util.ListWriter{}
 		t := &util.TableWriter{}
 		t.AppendRow(rows)
-		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.Solver.CompactString())
 
 		switch {
 		case files && installed:
@@ -353,12 +348,6 @@ Search can also return results in the terminal in different ways: as terminal ou
 			results = searchLocally(args[0], l, t, searchWithLabel, searchWithLabelMatch, revdeps, hidden)
 		}
 
-		if tableMode {
-			t.Render()
-		} else {
-			l.Render()
-		}
-
 		y, err := yaml.Marshal(results)
 		if err != nil {
 			fmt.Printf("err: %v\n", err)
@@ -374,22 +363,24 @@ Search can also return results in the terminal in different ways: as terminal ou
 				return
 			}
 			fmt.Println(string(j2))
+		default:
+			if tableMode {
+				t.Render()
+			} else if util.DefaultContext.Config.General.Quiet {
+				for _, tt := range results.Packages {
+					fmt.Printf("%s/%s-%s\n", tt.Category, tt.Name, tt.Version)
+				}
+			} else {
+				l.Render()
+			}
 		}
-
 	},
 }
 
 func init() {
-	searchCmd.Flags().String("system-dbpath", "", "System db path")
-	searchCmd.Flags().String("system-target", "", "System rootpath")
-	searchCmd.Flags().String("system-engine", "", "System DB engine")
 
 	searchCmd.Flags().Bool("installed", false, "Search between system packages")
-	searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
 	searchCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
-	searchCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
-	searchCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
-	searchCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	searchCmd.Flags().Bool("by-label", false, "Search packages through label")
 	searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex")
 	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")

cmd/tree/images.go

@@ -38,7 +38,11 @@ import (
 func NewTreeImageCommand() *cobra.Command {
 
 	var ans = &cobra.Command{
-		Use:   "images [OPTIONS]",
+		Use: "images [OPTIONS]",
+		// Skip processing output
+		Annotations: map[string]string{
+			util.CommandProcessOutput: "",
+		},
 		Short: "List of the images of a package",
 		PreRun: func(cmd *cobra.Command, args []string) {
 			t, _ := cmd.Flags().GetStringArray("tree")
@@ -60,12 +64,7 @@ func NewTreeImageCommand() *cobra.Command {
 			imageRepository := viper.GetString("image-repository")
 			pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 			values := util.ValuesFlags()
-
 			out, _ := cmd.Flags().GetString("output")
-			if out != "terminal" {
-				util.DefaultContext.Config.GetLogging().SetLogLevel("error")
-			}
-
 			reciper := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 
 			for _, t := range treePath {
@@ -76,7 +75,7 @@ func NewTreeImageCommand() *cobra.Command {
 			}
 			compilerBackend := backend.NewSimpleDockerBackend(util.DefaultContext)
 
-			opts := *util.DefaultContext.Config.GetSolverOptions()
+			opts := util.DefaultContext.Config.Solver
 			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
 			luetCompiler := compiler.NewLuetCompiler(
 				compilerBackend,
@@ -102,7 +101,12 @@ func NewTreeImageCommand() *cobra.Command {
 			}
 
 			ht := compiler.NewHashTree(reciper.GetDatabase())
-			hashtree, err := ht.Query(luetCompiler, spec)
+
+			copy, err := compiler.CompilerFinalImages(luetCompiler)
+			if err != nil {
+				util.DefaultContext.Fatal("Error: " + err.Error())
+			}
+			hashtree, err := ht.Query(copy, spec)
 			if err != nil {
 				util.DefaultContext.Fatal("Error: " + err.Error())
 			}

cmd/tree/pkglist.go

@@ -67,6 +67,10 @@ func NewTreePkglistCommand() *cobra.Command {
 	var matches []string
 
 	var ans = &cobra.Command{
+		// Skip processing output
+		Annotations: map[string]string{
+			util.CommandProcessOutput: "",
+		},
 		Use:   "pkglist [OPTIONS]",
 		Short: "List of the packages found in tree.",
 		Args:  cobra.NoArgs,
@@ -95,9 +99,6 @@ func NewTreePkglistCommand() *cobra.Command {
 			deps, _ := cmd.Flags().GetBool("deps")
 
 			out, _ := cmd.Flags().GetString("output")
-			if out != "terminal" {
-				util.DefaultContext.Config.GetLogging().SetLogLevel("error")
-			}
 
 			var reciper tree.Builder
 			if buildtime {

cmd/tree/validate.go

@@ -255,7 +255,7 @@ func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, recipe
 								r.GetCategory(), r.GetName(), r.GetVersion(),
 							))
 
-						if util.DefaultContext.Config.GetGeneral().Debug {
+						if util.DefaultContext.Config.General.Debug {
 							for idx, pa := range solution {
 								fmt.Println(fmt.Sprintf("[%9s] %s/%s-%s: solution %d: %s",
 									checkType,
@@ -426,7 +426,7 @@ func NewTreeValidateCommand() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			var reciper tree.Builder
 
-			concurrency := util.DefaultContext.Config.GetGeneral().Concurrency
+			concurrency := util.DefaultContext.Config.General.Concurrency
 
 			withSolver, _ := cmd.Flags().GetBool("with-solver")
 			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")

cmd/uninstall.go

@@ -17,7 +17,6 @@ package cmd
 import (
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/cmd/util"
-	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
@@ -32,8 +31,7 @@ var uninstallCmd = &cobra.Command{
 	Long:    `Uninstall packages`,
 	Aliases: []string{"rm", "un"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		util.BindSystemFlags(cmd)
-		util.BindSolverFlags(cmd)
+
 		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
@@ -57,21 +55,15 @@ var uninstallCmd = &cobra.Command{
 		yes := viper.GetBool("yes")
 		keepProtected, _ := cmd.Flags().GetBool("keep-protected-files")
 
-		util.SetSystemConfig(util.DefaultContext)
-		util.SetSolverConfig(util.DefaultContext)
-
 		util.DefaultContext.Config.ConfigProtectSkip = !keepProtected
 
-		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+		util.DefaultContext.Config.Solver.Implementation = solver.SingleCoreSimple
 
-		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
-
-		// Load config protect configs
-		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.Solver.CompactString())
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
-			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			Concurrency:                 util.DefaultContext.Config.General.Concurrency,
+			SolverOptions:               util.DefaultContext.Config.Solver,
 			NoDeps:                      nodeps,
 			Force:                       force,
 			FullUninstall:               full,
@@ -82,7 +74,7 @@ var uninstallCmd = &cobra.Command{
 			Context:                     util.DefaultContext,
 		})
 
-		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.System.Rootfs}
 
 		if err := inst.Uninstall(system, toRemove...); err != nil {
 			util.DefaultContext.Fatal("Error: " + err.Error())
@@ -92,14 +84,6 @@ var uninstallCmd = &cobra.Command{
 
 func init() {
 
-	uninstallCmd.Flags().String("system-dbpath", "", "System db path")
-	uninstallCmd.Flags().String("system-target", "", "System rootpath")
-	uninstallCmd.Flags().String("system-engine", "", "System DB engine")
-
-	uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
-	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
-	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
-	uninstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	uninstallCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
 	uninstallCmd.Flags().Bool("force", false, "Force uninstall")
 	uninstallCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")

cmd/upgrade.go

@@ -16,7 +16,6 @@ package cmd
 
 import (
 	"github.com/mudler/luet/cmd/util"
-	"github.com/mudler/luet/pkg/api/core/types"
 	installer "github.com/mudler/luet/pkg/installer"
 	"github.com/mudler/luet/pkg/solver"
 
@@ -29,8 +28,7 @@ var upgradeCmd = &cobra.Command{
 	Short:   "Upgrades the system",
 	Aliases: []string{"u"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		util.BindSystemFlags(cmd)
-		util.BindSolverFlags(cmd)
+
 		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
 	},
@@ -43,22 +41,18 @@ var upgradeCmd = &cobra.Command{
 		universe, _ := cmd.Flags().GetBool("universe")
 		clean, _ := cmd.Flags().GetBool("clean")
 		sync, _ := cmd.Flags().GetBool("sync")
+		osCheck, _ := cmd.Flags().GetBool("oscheck")
+
 		yes := viper.GetBool("yes")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		util.SetSystemConfig(util.DefaultContext)
-		opts := util.SetSolverConfig(util.DefaultContext)
+		util.DefaultContext.Config.Solver.Implementation = solver.SingleCoreSimple
 
-		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
-
-		util.DefaultContext.Debug("Solver", opts.CompactString())
-
-		// Load config protect configs
-		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+		util.DefaultContext.Debug("Solver", util.DefaultContext.GetConfig().Solver)
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
-			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			Concurrency:                 util.DefaultContext.Config.General.Concurrency,
+			SolverOptions:               util.DefaultContext.Config.Solver,
 			Force:                       force,
 			FullUninstall:               full,
 			NoDeps:                      nodeps,
@@ -67,12 +61,13 @@ var upgradeCmd = &cobra.Command{
 			UpgradeNewRevisions:         sync,
 			PreserveSystemEssentialData: true,
 			Ask:                         !yes,
+			AutoOSCheck:                 osCheck,
 			DownloadOnly:                downloadOnly,
 			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
 			Context:                     util.DefaultContext,
 		})
 
-		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.System.Rootfs}
 		if err := inst.Upgrade(system); err != nil {
 			util.DefaultContext.Fatal("Error: " + err.Error())
 		}
@@ -80,14 +75,6 @@ var upgradeCmd = &cobra.Command{
 }
 
 func init() {
-	upgradeCmd.Flags().String("system-dbpath", "", "System db path")
-	upgradeCmd.Flags().String("system-target", "", "System rootpath")
-	upgradeCmd.Flags().String("system-engine", "", "System DB engine")
-
-	upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
-	upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
-	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
-	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
 	upgradeCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
 	upgradeCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
@@ -97,6 +84,7 @@ func init() {
 	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	upgradeCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 	upgradeCmd.Flags().Bool("download-only", false, "Download only")
+	upgradeCmd.Flags().Bool("oscheck", false, "Perform automatically oschecks after upgrades")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

cmd/util.go

@@ -19,16 +19,70 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/go-units"
+	"github.com/mudler/luet/pkg/api/core/image"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	"github.com/pkg/errors"
 
 	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/helpers/docker"
 
 	"github.com/spf13/cobra"
 )
 
+func pack(ctx *context.Context, p, dst, imageName, arch, OS string) error {
+
+	tempimage, err := ctx.TempFile("tempimage")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for "+p)
+	}
+	defer os.RemoveAll(tempimage.Name()) // clean up
+
+	if err := image.CreateTar(p, tempimage.Name(), imageName, arch, OS); err != nil {
+		return errors.Wrap(err, "could not create image from tar")
+	}
+
+	return fileHelper.CopyFile(tempimage.Name(), dst)
+}
+
+func NewPackCommand() *cobra.Command {
+
+	c := &cobra.Command{
+		Use:   "pack image src.tar dst.tar",
+		Short: "Pack a standard tar archive as a container image",
+		Long: `Pack creates a tar which can be loaded as an image from a standard flat tar archive, for e.g. with docker load. 
+It doesn't need the docker daemon to run, and allows to override default os/arch:
+		
+	luet util pack --os arm64 image:tag src.tar dst.tar
+`,
+		Args: cobra.MinimumNArgs(3),
+		Run: func(cmd *cobra.Command, args []string) {
+
+			image := args[0]
+			src := args[1]
+			dst := args[2]
+
+			arch, _ := cmd.Flags().GetString("arch")
+			os, _ := cmd.Flags().GetString("os")
+
+			err := pack(util.DefaultContext, src, dst, image, arch, os)
+			if err != nil {
+				util.DefaultContext.Fatal(err.Error())
+			}
+			util.DefaultContext.Info("Image packed as", image)
+		},
+	}
+
+	c.Flags().String("arch", runtime.GOARCH, "Image architecture")
+	c.Flags().String("os", runtime.GOOS, "Image OS")
+
+	return c
+}
+
 func NewUnpackCommand() *cobra.Command {
 
 	c := &cobra.Command{
@@ -102,5 +156,6 @@ func init() {
 
 	utilGroup.AddCommand(
 		NewUnpackCommand(),
+		NewPackCommand(),
 	)
 }

cmd/util/cli.go

@@ -16,7 +16,6 @@
 package util
 
 import (
-	"errors"
 	"os"
 	"path/filepath"
 	"strings"
@@ -26,28 +25,14 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/api/core/types"
 	"github.com/mudler/luet/pkg/installer"
 )
 
-var DefaultContext = types.NewContext()
-
 var lockedCommands = []string{"install", "uninstall", "upgrade"}
 var bannerCommands = []string{"install", "build", "uninstall", "upgrade"}
 
-func BindSystemFlags(cmd *cobra.Command) {
-	viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-	viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-	viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-}
-
-func BindSolverFlags(cmd *cobra.Command) {
-	viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-	viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-	viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-	viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
-}
-
 func BindValuesFlags(cmd *cobra.Command) {
 	viper.BindPFlag("values", cmd.Flags().Lookup("values"))
 }
@@ -56,59 +41,14 @@ func ValuesFlags() []string {
 	return viper.GetStringSlice("values")
 }
 
-func SetSystemConfig(ctx *types.Context) {
-	dbpath := viper.GetString("system.database_path")
-	rootfs := viper.GetString("system.rootfs")
-	engine := viper.GetString("system.database_engine")
-
-	ctx.Config.System.DatabaseEngine = engine
-	ctx.Config.System.DatabasePath = dbpath
-	ctx.Config.System.SetRootFS(rootfs)
-}
-
-func SetSolverConfig(ctx *types.Context) (c *types.LuetSolverOptions) {
-	stype := viper.GetString("solver.type")
-	discount := viper.GetFloat64("solver.discount")
-	rate := viper.GetFloat64("solver.rate")
-	attempts := viper.GetInt("solver.max_attempts")
-
-	ctx.Config.GetSolverOptions().Type = stype
-	ctx.Config.GetSolverOptions().LearnRate = float32(rate)
-	ctx.Config.GetSolverOptions().Discount = float32(discount)
-	ctx.Config.GetSolverOptions().MaxAttempts = attempts
-
-	return &types.LuetSolverOptions{
-		Type:        stype,
-		LearnRate:   float32(rate),
-		Discount:    float32(discount),
-		MaxAttempts: attempts,
-	}
-}
-
-func SetCliFinalizerEnvs(ctx *types.Context, finalizerEnvs []string) error {
-	if len(finalizerEnvs) > 0 {
-		for _, v := range finalizerEnvs {
-			idx := strings.Index(v, "=")
-			if idx < 0 {
-				return errors.New("Found invalid runtime finalizer environment: " + v)
-			}
-
-			ctx.Config.SetFinalizerEnv(v[0:idx], v[idx+1:])
-		}
-
-	}
-
-	return nil
-}
-
 // TemplateFolders returns the default folders which holds shared template between packages in a given tree path
-func TemplateFolders(ctx *types.Context, fromRepo bool, treePaths []string) []string {
+func TemplateFolders(ctx *context.Context, fromRepo bool, treePaths []string) []string {
 	templateFolders := []string{}
 	for _, t := range treePaths {
 		templateFolders = append(templateFolders, filepath.Join(t, "templates"))
 	}
 	if fromRepo {
-		for _, s := range installer.SystemRepositories(ctx.Config.SystemRepositories) {
+		for _, s := range installer.SystemRepositories(ctx.GetConfig().SystemRepositories) {
 			templateFolders = append(templateFolders, filepath.Join(s.TreePath, "templates"))
 		}
 	}
@@ -126,7 +66,7 @@ func IntroScreen() {
 	pterm.DefaultCenter.Print(pterm.DefaultHeader.WithFullWidth().WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(10).Sprint("Luet - 0-deps container-based package manager"))
 }
 
-func HandleLock(c *types.Context) {
+func HandleLock(c types.Context) {
 	if os.Getenv("LUET_NOLOCK") != "true" {
 		if len(os.Args) > 1 {
 			for _, lockedCmd := range lockedCommands {
@@ -146,7 +86,7 @@ func HandleLock(c *types.Context) {
 	}
 }
 
-func DisplayVersionBanner(c *types.Context, banner func(), version func() string, license []string) {
+func DisplayVersionBanner(c *context.Context, banner func(), version func() string, license []string) {
 	display := false
 	if len(os.Args) > 1 {
 		for _, c := range bannerCommands {
@@ -156,11 +96,15 @@ func DisplayVersionBanner(c *types.Context, banner func(), version func() string
 		}
 	}
 	if display {
-		banner()
-		pterm.DefaultCenter.Print(version())
-		for _, l := range license {
-			pterm.DefaultCenter.Print(l)
-
+		if c.Config.General.Quiet {
+			pterm.Info.Printf("Luet %s\n", version())
+			pterm.Info.Println(strings.Join(license, "\n"))
+		} else {
+			banner()
+			pterm.DefaultCenter.Print(version())
+			for _, l := range license {
+				pterm.DefaultCenter.Print(l)
+			}
 		}
 	}
 }

cmd/util/config.go

@@ -16,6 +16,7 @@
 package util
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/user"
@@ -23,8 +24,15 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/ipfs/go-log/v2"
 	extensions "github.com/mudler/cobra-extensions"
+	"github.com/mudler/luet/pkg/api/core/context"
+	gc "github.com/mudler/luet/pkg/api/core/garbagecollector"
+	"github.com/mudler/luet/pkg/api/core/logger"
 	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/solver"
+	"github.com/pterm/pterm"
+	"go.uber.org/zap/zapcore"
 
 	helpers "github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
@@ -87,26 +95,117 @@ func initConfig() {
 
 }
 
+var DefaultContext *context.Context
+
 // InitContext inits the context by parsing the configurations from viper
 // this is meant to be run before each command to be able to parse any override from
 // the CLI/ENV
-func InitContext(ctx *types.Context) (err error) {
+func InitContext(cmd *cobra.Command) (ctx *context.Context, err error) {
 
-	err = viper.Unmarshal(&ctx.Config)
+	c := &types.LuetConfig{}
+	err = viper.Unmarshal(c)
 	if err != nil {
 		return
 	}
 
+	// Converts user-defined config into paths
+	// and creates the required directory on the system if necessary
+	c.Init()
+
+	finalizerEnvs, _ := cmd.Flags().GetStringArray("finalizer-env")
+	setCliFinalizerEnvs(c, finalizerEnvs)
+
+	c.Solver.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: c.General.Concurrency}
+
+	ctx = context.NewContext(
+		context.WithConfig(c),
+		context.WithGarbageCollector(gc.GarbageCollector(c.System.TmpDirBase)),
+	)
+
 	// Inits the context with the configurations loaded
 	// It reads system repositories, sets logging, and all the
 	// context which is required to perform luet actions
-	err = ctx.Init()
-	if err != nil {
-		return
+	return ctx, initContext(cmd, ctx)
+}
+
+func setCliFinalizerEnvs(c *types.LuetConfig, finalizerEnvs []string) error {
+	if len(finalizerEnvs) > 0 {
+		for _, v := range finalizerEnvs {
+			idx := strings.Index(v, "=")
+			if idx < 0 {
+				return errors.New("Found invalid runtime finalizer environment: " + v)
+			}
+
+			c.SetFinalizerEnv(v[0:idx], v[idx+1:])
+		}
 	}
 
-	// no_spinner is not mapped in our configs
-	ctx.NoSpinner = viper.GetBool("no_spinner")
+	return nil
+}
+
+const (
+	CommandProcessOutput = "command.process.output"
+)
+
+func initContext(cmd *cobra.Command, c *context.Context) (err error) {
+	if logger.IsTerminal() {
+		if !c.Config.Logging.Color {
+			pterm.DisableColor()
+		}
+	} else {
+		pterm.DisableColor()
+		c.Debug("Not a terminal, colors disabled")
+	}
+
+	if c.Config.General.Quiet {
+		pterm.DisableColor()
+		pterm.DisableStyling()
+	}
+
+	level := c.Config.Logging.Level
+	if c.Config.General.Debug {
+		level = "debug"
+	}
+
+	if _, ok := cmd.Annotations[CommandProcessOutput]; ok {
+		// Note: create-repo output is different, so we annotate in the cmd of create-repo CommandNoProcess
+		// to avoid
+		out, _ := cmd.Flags().GetString("output")
+		if out != "terminal" {
+			level = zapcore.Level(log.LevelFatal).String()
+		}
+	}
+
+	// Init logging
+	opts := []logger.LoggerOptions{
+		logger.WithLevel(level),
+	}
+
+	if c.Config.Logging.NoSpinner {
+		opts = append(opts, logger.NoSpinner)
+	}
+
+	if c.Config.Logging.EnableLogFile && c.Config.Logging.Path != "" {
+		f := "console"
+		if c.Config.Logging.JsonFormat {
+			f = "json"
+		}
+		opts = append(opts, logger.WithFileLogging(c.Config.Logging.Path, f))
+	}
+
+	if c.Config.Logging.EnableEmoji {
+		opts = append(opts, logger.EnableEmoji())
+	}
+
+	l, err := logger.New(opts...)
+
+	c.Logger = l
+
+	c.Debug("System rootfs:", c.Config.System.Rootfs)
+	c.Debug("Colors", c.Config.Logging.Color)
+	c.Debug("Logging level", c.Config.Logging.Level)
+	c.Debug("Debug mode", c.Config.General.Debug)
+
 	return
 }
 
@@ -120,6 +219,7 @@ func setDefaults(viper *viper.Viper) {
 
 	viper.SetDefault("general.concurrency", runtime.NumCPU())
 	viper.SetDefault("general.debug", false)
+	viper.SetDefault("general.quiet", false)
 	viper.SetDefault("general.show_build_output", false)
 	viper.SetDefault("general.fatal_warnings", false)
 	viper.SetDefault("general.http_timeout", 360)
@@ -155,39 +255,50 @@ func setDefaults(viper *viper.Viper) {
 
 // InitViper inits a new viper
 // this is meant to be run just once at beginning to setup the root command
-func InitViper(ctx *types.Context, RootCmd *cobra.Command) {
+func InitViper(RootCmd *cobra.Command) {
 	cobra.OnInitialize(initConfig)
 	pflags := RootCmd.PersistentFlags()
 	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
-	pflags.BoolP("debug", "d", false, "verbose output")
+	pflags.BoolP("debug", "d", false, "debug output")
+	pflags.BoolP("quiet", "q", false, "quiet output")
 	pflags.Bool("fatal", false, "Enables Warnings to exit")
 	pflags.Bool("enable-logfile", false, "Enable log to file")
 	pflags.Bool("no-spinner", false, "Disable spinner.")
-	pflags.Bool("color", ctx.Config.GetLogging().Color, "Enable/Disable color.")
-	pflags.Bool("emoji", ctx.Config.GetLogging().EnableEmoji, "Enable/Disable emoji.")
-	pflags.Bool("skip-config-protect", ctx.Config.ConfigProtectSkip,
-		"Disable config protect analysis.")
-	pflags.StringP("logfile", "l", ctx.Config.GetLogging().Path,
-		"Logfile path. Empty value disable log to file.")
+	pflags.Bool("color", true, "Enable/Disable color.")
+	pflags.Bool("emoji", true, "Enable/Disable emoji.")
+	pflags.Bool("skip-config-protect", true, "Disable config protect analysis.")
+	pflags.StringP("logfile", "l", "", "Logfile path. Empty value disable log to file.")
 	pflags.StringSlice("plugin", []string{}, "A list of runtime plugins to load")
 
-	// os/user doesn't work in from scratch environments.
-	// Check if i can retrieve user informations.
-	_, err := user.Current()
-	if err != nil {
-		ctx.Warning("failed to retrieve user identity:", err.Error())
-	}
-	pflags.Bool("same-owner", ctx.Config.GetGeneral().SameOwner, "Maintain same owner on uncompress.")
+	pflags.String("system-dbpath", "", "System db path")
+	pflags.String("system-target", "", "System rootpath")
+	pflags.String("system-engine", "", "System DB engine")
+
+	pflags.String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	pflags.Float32("solver-rate", 0.7, "Solver learning rate")
+	pflags.Float32("solver-discount", 1.0, "Solver discount rate")
+	pflags.Int("solver-attempts", 9000, "Solver maximum attempts")
+
+	pflags.Bool("same-owner", true, "Maintain same owner on uncompress.")
 	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
-	pflags.Int("http-timeout", ctx.Config.General.HTTPTimeout, "Default timeout for http(s) requests")
+	pflags.Int("http-timeout", 360, "Default timeout for http(s) requests")
+
+	viper.BindPFlag("system.database_path", pflags.Lookup("system-dbpath"))
+	viper.BindPFlag("system.rootfs", pflags.Lookup("system-target"))
+	viper.BindPFlag("system.database_engine", pflags.Lookup("system-engine"))
+	viper.BindPFlag("solver.type", pflags.Lookup("solver-type"))
+	viper.BindPFlag("solver.discount", pflags.Lookup("solver-discount"))
+	viper.BindPFlag("solver.rate", pflags.Lookup("solver-rate"))
+	viper.BindPFlag("solver.max_attempts", pflags.Lookup("solver-attempts"))
 
 	viper.BindPFlag("logging.color", pflags.Lookup("color"))
 	viper.BindPFlag("logging.enable_emoji", pflags.Lookup("emoji"))
 	viper.BindPFlag("logging.enable_logfile", pflags.Lookup("enable-logfile"))
 	viper.BindPFlag("logging.path", pflags.Lookup("logfile"))
-
+	viper.BindPFlag("logging.no_spinner", pflags.Lookup("no-spinner"))
 	viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
 	viper.BindPFlag("general.debug", pflags.Lookup("debug"))
+	viper.BindPFlag("general.quiet", pflags.Lookup("quiet"))
 	viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
 	viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
 	viper.BindPFlag("plugin", pflags.Lookup("plugin"))

go.mod

@@ -30,19 +30,21 @@ require (
 	github.com/hashicorp/go-version v1.3.0
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12
+	github.com/ipfs/go-log/v2 v2.4.0
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
 	github.com/klauspost/compress v1.13.0
 	github.com/klauspost/pgzip v1.2.5
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
+	github.com/mattn/go-isatty v0.0.14
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.1
 	github.com/mitchellh/mapstructure v1.4.2 // indirect
 	github.com/moby/moby v20.10.9+incompatible
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
-	github.com/mudler/go-pluggable v0.0.0-20211022125509-94dbf124830d
+	github.com/mudler/go-pluggable v0.0.0-20211206135551-9263b05c562e
 	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.16.0
@@ -50,6 +52,7 @@ require (
 	github.com/opencontainers/image-spec v1.0.1
 	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
 	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
 	github.com/pkg/errors v0.9.1
 	github.com/pterm/pterm v0.12.32-0.20211002183613-ada9ef6790c3
@@ -64,7 +67,7 @@ require (
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
 	golang.org/x/mod v0.4.2
 	golang.org/x/oauth2 v0.0.0-20210810183815-faf39c7919d5 // indirect
-	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654
+	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	google.golang.org/genproto v0.0.0-20210811021853-ddbe55d93216 // indirect

go.sum

@@ -537,6 +537,7 @@ github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAO
 github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8=
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/certificate-transparency-go v1.0.10-0.20180222191210-5ab67e519c93 h1:jc2UWq7CbdszqeH6qu1ougXMIUBfSy8Pbh/anURYbGI=
 github.com/google/certificate-transparency-go v1.0.10-0.20180222191210-5ab67e519c93/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg=
@@ -662,6 +663,8 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
+github.com/ipfs/go-log/v2 v2.4.0 h1:iR/2o9PGWanVJrBgIH5Ff8mPGOwpqLaPIAFqSnsdlzk=
+github.com/ipfs/go-log/v2 v2.4.0/go.mod h1:nPZnh7Cj7lwS3LpRU5Mwr2ol1c2gXIEXuF6aywqrtmo=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 h1:sHsPfNMAG70QAvKbddQ0uScZCHQoZsT5NykGRCeeeIs=
@@ -746,6 +749,8 @@ github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHef
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
@@ -809,10 +814,8 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d h1:fKh+rvwZQCA+TPzK0EMwwbqhjvRHaQ6H8AsVU1Wt+NQ=
 github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d/go.mod h1:puRUWSwyecW2V355tKncwPVPRAjQBduPsFjG0mrV/Nw=
-github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af h1:jixIxEgLSqu24eMiyzfCI+roa5IaOUhF546ePSFyHeY=
-github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
-github.com/mudler/go-pluggable v0.0.0-20211022125509-94dbf124830d h1:NKvvf/q1dWDde+yg5cMiU5EuYZ2jNuKs/9hb8xod8A0=
-github.com/mudler/go-pluggable v0.0.0-20211022125509-94dbf124830d/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
+github.com/mudler/go-pluggable v0.0.0-20211206135551-9263b05c562e h1:CZI+kJW2+WjZXLWWnVzi6NDQ6SfwSfeNqq5d1iDiwyY=
+github.com/mudler/go-pluggable v0.0.0-20211206135551-9263b05c562e/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290 h1:426hFyXMpXeqIeGJn2cGAW9ogvM2Jf+Jv23gtVPvBLM=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290/go.mod h1:uP5BBgFxq2wNWo7n1vnY5SSbgL0WDshVJrOO12tZ/lA=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
@@ -909,6 +912,7 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
 github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
+github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
 github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f h1:WyCn68lTiytVSkk7W1K9nBiSGTSRlUOdyTnSjwrIlok=
@@ -1139,15 +1143,18 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
 go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

pkg/api/core/bus/events.go

@@ -1,6 +1,8 @@
 package bus
 
 import (
+	"fmt"
+
 	"github.com/mudler/go-pluggable"
 	"github.com/mudler/luet/pkg/api/core/types"
 )
@@ -12,6 +14,10 @@ var (
 	EventPackageInstall pluggable.EventType = "package.install"
 	// EventPackageUnInstall is the event fired when a new package is being uninstalled
 	EventPackageUnInstall pluggable.EventType = "package.uninstall"
+	// EventPreUpgrade is the event fired before an upgrade is attempted
+	EventPreUpgrade pluggable.EventType = "package.pre.upgrade"
+	// EventPostUpgrade is the event fired after an upgrade is done
+	EventPostUpgrade pluggable.EventType = "package.post.upgrade"
 
 	// Package build
 
@@ -61,6 +67,8 @@ var Manager *Bus = &Bus{
 			EventPackageInstall,
 			EventPackageUnInstall,
 			EventPackagePreBuild,
+			EventPreUpgrade,
+			EventPostUpgrade,
 			EventPackagePreBuildArtifact,
 			EventPackagePostBuildArtifact,
 			EventPackagePostBuild,
@@ -82,14 +90,11 @@ type Bus struct {
 	*pluggable.Manager
 }
 
-func (b *Bus) Initialize(ctx *types.Context, plugin ...string) {
+func (b *Bus) Initialize(ctx types.Context, plugin ...string) {
 	b.Manager.Load(plugin...).Register()
 
 	for _, e := range b.Manager.Events {
 		b.Manager.Response(e, func(p *pluggable.Plugin, r *pluggable.EventResponse) {
-			if r.Errored() {
-				ctx.Fatal("Plugin", p.Name, "at", p.Executable, "Error", r.Error)
-			}
 			ctx.Debug(
 				"plugin_event",
 				"received from",
@@ -98,6 +103,16 @@ func (b *Bus) Initialize(ctx *types.Context, plugin ...string) {
 				p.Executable,
 				r,
 			)
+			if r.Errored() {
+				err := fmt.Sprintf("Plugin %s at %s had an error: %s", p.Name, p.Executable, r.Error)
+				ctx.Fatal(err)
+			} else {
+				if r.State != "" {
+					message := fmt.Sprintf(":lollipop: Plugin %s at %s succeded, state reported:", p.Name, p.Executable)
+					ctx.Success(message)
+					ctx.Info(r.State)
+				}
+			}
 		})
 	}
 }

pkg/api/core/config/config_protect_test.go

@@ -18,7 +18,7 @@ package config_test
 
 import (
 	config "github.com/mudler/luet/pkg/api/core/config"
-	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/context"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -29,7 +29,7 @@ var _ = Describe("Config", func() {
 	Context("Test config protect", func() {
 
 		It("Protect1", func() {
-			ctx := types.NewContext()
+			ctx := context.NewContext()
 			files := []string{
 				"etc/foo/my.conf",
 				"usr/bin/foo",
@@ -59,7 +59,7 @@ var _ = Describe("Config", func() {
 		})
 
 		It("Protect2", func() {
-			ctx := types.NewContext()
+			ctx := context.NewContext()
 
 			files := []string{
 				"etc/foo/my.conf",
@@ -86,7 +86,7 @@ var _ = Describe("Config", func() {
 		})
 
 		It("Protect3: Annotation dir without initial slash", func() {
-			ctx := types.NewContext()
+			ctx := context.NewContext()
 
 			files := []string{
 				"etc/foo/my.conf",

pkg/api/core/context/contest_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package context_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestContext(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Context Suite")
+}

pkg/api/core/context/context.go

@@ -0,0 +1,159 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package context
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
+	gc "github.com/mudler/luet/pkg/api/core/garbagecollector"
+	"github.com/mudler/luet/pkg/api/core/logger"
+	"github.com/mudler/luet/pkg/api/core/types"
+
+	"github.com/pkg/errors"
+)
+
+type Context struct {
+	*logger.Logger
+	context.Context
+	types.GarbageCollector
+	Config      *types.LuetConfig
+	NoSpinner   bool
+	annotations map[string]interface{}
+}
+
+// SetAnnotation sets generic annotations to hold in a context
+func (c *Context) SetAnnotation(s string, i interface{}) {
+	c.annotations[s] = i
+}
+
+// GetAnnotation gets generic annotations to hold in a context
+func (c *Context) GetAnnotation(s string) interface{} {
+	return c.annotations[s]
+}
+
+type ContextOption func(c *Context) error
+
+// WithLogger sets the logger
+func WithLogger(l *logger.Logger) ContextOption {
+	return func(c *Context) error {
+		c.Logger = l
+		return nil
+	}
+}
+
+// WithConfig sets the luet config
+func WithConfig(cc *types.LuetConfig) ContextOption {
+	return func(c *Context) error {
+		c.Config = cc
+		return nil
+	}
+}
+
+// NOTE: GC needs to be instantiated when a new context is created from system TmpDirBase
+
+// WithGarbageCollector sets the Garbage collector for the given context
+func WithGarbageCollector(l types.GarbageCollector) ContextOption {
+	return func(c *Context) error {
+		if !filepath.IsAbs(l.String()) {
+			abs, err := fileHelper.Rel2Abs(l.String())
+			if err != nil {
+				return errors.Wrap(err, "while converting relative path to absolute path")
+			}
+			l = gc.GarbageCollector(abs)
+		}
+
+		c.GarbageCollector = l
+		return nil
+	}
+}
+
+// NewContext returns a new context.
+// It accepts a Garbage collector, a config and a logger as an option
+func NewContext(opts ...ContextOption) *Context {
+	l, _ := logger.New()
+	d := &Context{
+		annotations:      make(map[string]interface{}),
+		Logger:           l,
+		GarbageCollector: gc.GarbageCollector(filepath.Join(os.TempDir(), "tmpluet")),
+		Config: &types.LuetConfig{
+			ConfigFromHost: true,
+			Logging:        types.LuetLoggingConfig{},
+			General:        types.LuetGeneralConfig{},
+			System: types.LuetSystemConfig{
+				DatabasePath:  filepath.Join("var", "db"),
+				PkgsCachePath: filepath.Join("var", "db", "packages"),
+			},
+			Solver: types.LuetSolverOptions{},
+		},
+	}
+
+	for _, o := range opts {
+		o(d)
+	}
+	return d
+}
+
+// WithLoggingContext returns a copy of the context with a contextualized logger
+func (c *Context) WithLoggingContext(name string) types.Context {
+	configCopy := *c.Config
+	configCopy.System = c.Config.System
+	configCopy.General = c.Config.General
+	configCopy.Logging = c.Config.Logging
+
+	ctx := *c
+	ctxCopy := &ctx
+	ctxCopy.Config = &configCopy
+	ctxCopy.annotations = ctx.annotations
+
+	ctxCopy.Logger, _ = c.Logger.Copy(logger.WithContext(name))
+
+	return ctxCopy
+}
+
+// Copy returns a context copy with a reset logging context
+func (c *Context) Copy() types.Context {
+	return c.WithLoggingContext("")
+}
+
+func (c *Context) Warning(mess ...interface{}) {
+	c.Logger.Warn(mess...)
+	if c.Config.General.FatalWarns {
+		os.Exit(2)
+	}
+}
+
+func (c *Context) Warn(mess ...interface{}) {
+	c.Warning(mess...)
+}
+
+func (c *Context) Warnf(t string, mess ...interface{}) {
+	c.Logger.Warnf(t, mess...)
+	if c.Config.General.FatalWarns {
+		os.Exit(2)
+	}
+}
+
+func (c *Context) Warningf(t string, mess ...interface{}) {
+	c.Warnf(t, mess...)
+}
+
+func (c *Context) GetConfig() types.LuetConfig {
+	return *c.Config
+}

pkg/api/core/garbagecollector/garbagecollector.go

@@ -0,0 +1,59 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package gc
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+type GarbageCollector string
+
+func (c GarbageCollector) String() string {
+	return string(c)
+}
+
+func (c GarbageCollector) init() error {
+	if _, err := os.Stat(string(c)); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(string(c), os.ModePerm)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c GarbageCollector) Clean() error {
+	return os.RemoveAll(string(c))
+}
+
+func (c GarbageCollector) TempDir(pattern string) (string, error) {
+	err := c.init()
+	if err != nil {
+		return "", err
+	}
+	return ioutil.TempDir(string(c), pattern)
+}
+
+func (c GarbageCollector) TempFile(s string) (*os.File, error) {
+	err := c.init()
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.TempFile(string(c), s)
+}

pkg/api/core/image/cache.go

@@ -0,0 +1,168 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"encoding/json"
+	"os"
+	"strings"
+
+	"github.com/peterbourgon/diskv"
+)
+
+// Cache represents a key-value store which is capable to upgrade to disk when it
+// reaches a pre-defined threshold.
+type Cache struct {
+	store                      *diskv.Diskv
+	memory                     map[string]string
+	dir                        string
+	onDisk                     bool
+	maxmemorySize, maxItemSize int
+}
+
+// New creates a new key-value cache
+// the cache acts in memory as long as the maxItemsize is not reached.
+// Once the threshold is met the cache is offloaded to disk automatically,
+// with a buffer of maxmemorySize into memory.
+func NewCache(path string, maxmemorySize, maxItemsize int) *Cache {
+	disk := diskv.New(diskv.Options{
+		BasePath:     path,
+		CacheSizeMax: uint64(maxmemorySize), // 500MB
+	})
+
+	return &Cache{
+		memory:        make(map[string]string),
+		store:         disk,
+		dir:           path,
+		maxmemorySize: maxmemorySize,
+		maxItemSize:   maxItemsize,
+	}
+}
+
+// This is needed as the disk cache is merely stored as separate files
+// thus we don't want to conflict file names with the path separator.
+// XXX: This is inconvenient as while we are looping result we can't rely
+// anymore originally to the key name.
+// We don't do any hashing to avoid any performance impact
+func cleanKey(s string) string {
+	return strings.ReplaceAll(s, string(os.PathSeparator), "_")
+}
+
+// Count returns the items in the cache.
+// If it's a disk cache might be an expensive call.
+func (c *Cache) Count() int {
+	if !c.onDisk {
+		return len(c.memory)
+	}
+
+	count := 0
+	for range c.store.Keys(nil) {
+		count++
+	}
+	return count
+}
+
+// Get attempts to retrieve a value for a key
+func (c *Cache) Get(key string) (value string, found bool) {
+
+	if !c.onDisk {
+		v, ok := c.memory[key]
+		return v, ok
+	}
+	v, err := c.store.Read(cleanKey(key))
+	if err == nil {
+		found = true
+	}
+	value = string(v)
+	return
+}
+
+func (c *Cache) flushToDisk() {
+	for k, v := range c.memory {
+		c.store.Write(cleanKey(k), []byte(v))
+	}
+	c.memory = make(map[string]string)
+	c.onDisk = true
+}
+
+// Set updates or inserts a new value
+func (c *Cache) Set(key, value string) error {
+
+	if !c.onDisk && c.Count() >= c.maxItemSize && c.maxItemSize != 0 {
+		c.flushToDisk()
+	}
+
+	if c.onDisk {
+		return c.store.Write(cleanKey(key), []byte(value))
+	}
+
+	c.memory[key] = value
+
+	return nil
+}
+
+// SetValue updates or inserts a new value by marshalling it into JSON.
+func (c *Cache) SetValue(key string, value interface{}) error {
+	dat, err := json.Marshal(value)
+	if err != nil {
+		return err
+	}
+	return c.Set(cleanKey(key), string(dat))
+}
+
+// CacheResult represent the key value result when
+// iterating over the cache
+type CacheResult struct {
+	key, value string
+}
+
+// Value returns the underlying value
+func (c CacheResult) Value() string {
+	return c.value
+}
+
+// Key returns the cache result key
+func (c CacheResult) Key() string {
+	return c.key
+}
+
+// Unmarshal the result into the interface. Use it to retrieve data
+// set with SetValue
+func (c CacheResult) Unmarshal(i interface{}) error {
+	return json.Unmarshal([]byte(c.Value()), i)
+}
+
+// Iterates over cache by key
+func (c *Cache) All(fn func(CacheResult)) {
+	if !c.onDisk {
+		for k, v := range c.memory {
+			fn(CacheResult{key: k, value: v})
+		}
+		return
+	}
+
+	for key := range c.store.Keys(nil) {
+		val, _ := c.store.Read(key)
+		fn(CacheResult{key: key, value: string(val)})
+	}
+}
+
+// Clean the cache
+func (c *Cache) Clean() {
+	c.memory = make(map[string]string)
+	c.onDisk = false
+	os.RemoveAll(c.dir)
+}

pkg/api/core/image/cache_test.go

@@ -0,0 +1,98 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/api/core/context"
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Cache", func() {
+
+	ctx := context.NewContext()
+	Context("used as k/v store", func() {
+
+		cache := &Cache{}
+		var dir string
+
+		BeforeEach(func() {
+			ctx = context.NewContext()
+			var err error
+			dir, err = ctx.TempDir("foo")
+			Expect(err).ToNot(HaveOccurred())
+			cache = NewCache(dir, 10*1024*1024, 1) // 10MB Cache when upgrading to files. Max volatile memory of 1 row.
+		})
+
+		AfterEach(func() {
+			cache.Clean()
+		})
+
+		It("does handle automatically memory upgrade", func() {
+			cache.Set("foo", "bar")
+			v, found := cache.Get("foo")
+			Expect(found).To(BeTrue())
+			Expect(v).To(Equal("bar"))
+			Expect(file.Exists(filepath.Join(dir, "foo"))).To(BeFalse())
+			cache.Set("baz", "bar")
+			Expect(file.Exists(filepath.Join(dir, "foo"))).To(BeTrue())
+			Expect(file.Exists(filepath.Join(dir, "baz"))).To(BeTrue())
+			v, found = cache.Get("foo")
+			Expect(found).To(BeTrue())
+			Expect(v).To(Equal("bar"))
+
+			Expect(cache.Count()).To(Equal(2))
+		})
+
+		It("does CRUD", func() {
+			cache.Set("foo", "bar")
+
+			v, found := cache.Get("foo")
+			Expect(found).To(BeTrue())
+			Expect(v).To(Equal("bar"))
+
+			hit := false
+			cache.All(func(c CacheResult) {
+				hit = true
+				Expect(c.Key()).To(Equal("foo"))
+				Expect(c.Value()).To(Equal("bar"))
+			})
+			Expect(hit).To(BeTrue())
+
+		})
+
+		It("Unmarshals values", func() {
+			type testStruct struct {
+				Test string
+			}
+
+			cache.SetValue("foo", &testStruct{Test: "baz"})
+
+			n := &testStruct{}
+
+			cache.All(func(cr CacheResult) {
+				err := cr.Unmarshal(n)
+				Expect(err).ToNot(HaveOccurred())
+
+			})
+			Expect(n.Test).To(Equal("baz"))
+		})
+	})
+})

pkg/api/core/image/create.go

@@ -0,0 +1,97 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"io"
+	"os"
+
+	containerdCompression "github.com/containerd/containerd/archive/compression"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/pkg/errors"
+)
+
+func imageFromTar(imagename, architecture, OS string, opener func() (io.ReadCloser, error)) (name.Reference, v1.Image, error) {
+	newRef, err := name.ParseReference(imagename)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	layer, err := tarball.LayerFromOpener(opener)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	baseImage := empty.Image
+	cfg, err := baseImage.ConfigFile()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfg.Architecture = architecture
+	cfg.OS = OS
+
+	baseImage, err = mutate.ConfigFile(baseImage, cfg)
+	if err != nil {
+		return nil, nil, err
+	}
+	img, err := mutate.Append(baseImage, mutate.Addendum{
+		Layer: layer,
+		History: v1.History{
+			CreatedBy: "luet",
+			Comment:   "Custom image",
+		},
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return newRef, img, nil
+}
+
+// CreateTar a imagetarball from a standard tarball
+func CreateTar(srctar, dstimageTar, imagename, architecture, OS string) error {
+
+	dstFile, err := os.Create(dstimageTar)
+	if err != nil {
+		return errors.Wrap(err, "Cannot create "+dstimageTar)
+	}
+	defer dstFile.Close()
+
+	newRef, img, err := imageFromTar(imagename, architecture, OS, func() (io.ReadCloser, error) {
+		f, err := os.Open(srctar)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot open "+srctar)
+		}
+		decompressed, err := containerdCompression.DecompressStream(f)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot open "+srctar)
+		}
+
+		return decompressed, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// NOTE: We might also stream that back to the daemon with daemon.Write(tag, img)
+	return tarball.Write(newRef, img, dstFile)
+
+}

pkg/api/core/image/create_test.go

@@ -0,0 +1,80 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/mudler/luet/pkg/api/core/context"
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
+	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Create", func() {
+	Context("Creates an OCI image from a standard tar", func() {
+		It("creates an image which is loadable", func() {
+			ctx := context.NewContext()
+
+			dst, err := ctx.TempFile("dst")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(dst.Name())
+			srcTar, err := ctx.TempFile("srcTar")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(srcTar.Name())
+
+			b := backend.NewSimpleDockerBackend(ctx)
+
+			b.DownloadImage(backend.Options{ImageName: "alpine"})
+			img, err := b.ImageReference("alpine", false)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, dir, err := Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			defer os.RemoveAll(dir)
+
+			Expect(file.Touch(filepath.Join(dir, "test"))).ToNot(HaveOccurred())
+			Expect(file.Exists(filepath.Join(dir, "bin"))).To(BeTrue())
+
+			a := artifact.NewPackageArtifact(srcTar.Name())
+			a.Compress(dir, 1)
+
+			// Unfortunately there is no other easy way to test this
+			err = CreateTar(srcTar.Name(), dst.Name(), "testimage", runtime.GOARCH, runtime.GOOS)
+			Expect(err).ToNot(HaveOccurred())
+
+			b.LoadImage(dst.Name())
+
+			Expect(b.ImageExists("testimage")).To(BeTrue())
+
+			img, err = b.ImageReference("testimage", false)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, dir, err = Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			defer os.RemoveAll(dir)
+			Expect(file.Exists(filepath.Join(dir, "bin"))).To(BeTrue())
+			Expect(file.Exists(filepath.Join(dir, "test"))).To(BeTrue())
+		})
+	})
+})

pkg/api/core/image/delta_test.go

@@ -23,8 +23,8 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	daemon "github.com/google/go-containerregistry/pkg/v1/daemon"
+	"github.com/mudler/luet/pkg/api/core/context"
 	. "github.com/mudler/luet/pkg/api/core/image"
-	"github.com/mudler/luet/pkg/api/core/types"
 	"github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -47,21 +47,19 @@ var _ = Describe("Delta", func() {
 		})
 
 		Context("ExtractDeltaFiles", func() {
-			ctx := types.NewContext()
+			ctx := context.NewContext()
 			var tmpfile *os.File
 			var ref, ref2 name.Reference
 			var img, img2 v1.Image
-			var diff ImageDiff
 			var err error
 
 			ref, _ = name.ParseReference("alpine")
 			ref2, _ = name.ParseReference("golang:alpine")
 			img, _ = daemon.Image(ref)
 			img2, _ = daemon.Image(ref2)
-			diff, err = Delta(img, img2)
 
 			BeforeEach(func() {
-				ctx = types.NewContext()
+				ctx = context.NewContext()
 
 				tmpfile, err = ioutil.TempFile("", "delta")
 				Expect(err).ToNot(HaveOccurred())
@@ -69,15 +67,18 @@ var _ = Describe("Delta", func() {
 			})
 
 			It("Extract all deltas", func() {
+
+				f, err := ExtractDeltaAdditionsFiles(ctx, img, []string{}, []string{})
+				Expect(err).ToNot(HaveOccurred())
+
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
-					ExtractDeltaFiles(ctx, diff, []string{}, []string{}),
+					f,
 				)
 				Expect(err).ToNot(HaveOccurred())
 				defer os.RemoveAll(tmpdir) // clean up
-
+				Expect(file.Exists(filepath.Join(tmpdir, "home"))).To(BeFalse())
 				Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeTrue())
 				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
 				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
@@ -85,11 +86,14 @@ var _ = Describe("Delta", func() {
 			})
 
 			It("Extract deltas and excludes /usr/local/go", func() {
+				f, err := ExtractDeltaAdditionsFiles(ctx, img, []string{}, []string{"usr/local/go"})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(err).ToNot(HaveOccurred())
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
-					ExtractDeltaFiles(ctx, diff, []string{}, []string{"usr/local/go"}),
+					f,
 				)
 				Expect(err).ToNot(HaveOccurred())
 				defer os.RemoveAll(tmpdir) // clean up
@@ -97,11 +101,13 @@ var _ = Describe("Delta", func() {
 			})
 
 			It("Extract deltas and excludes /usr/local/go/bin, but includes /usr/local/go", func() {
+				f, err := ExtractDeltaAdditionsFiles(ctx, img, []string{"usr/local/go"}, []string{"usr/local/go/bin"})
+				Expect(err).ToNot(HaveOccurred())
+
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
-					ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{"usr/local/go/bin"}),
+					f,
 				)
 				Expect(err).ToNot(HaveOccurred())
 				defer os.RemoveAll(tmpdir) // clean up
@@ -110,11 +116,12 @@ var _ = Describe("Delta", func() {
 			})
 
 			It("Extract deltas and includes /usr/local/go", func() {
+				f, err := ExtractDeltaAdditionsFiles(ctx, img, []string{"usr/local/go"}, []string{})
+				Expect(err).ToNot(HaveOccurred())
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
-					ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{}),
+					f,
 				)
 				Expect(err).ToNot(HaveOccurred())
 				defer os.RemoveAll(tmpdir) // clean up

pkg/api/core/image/extract.go

@@ -22,38 +22,62 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"syscall"
 
 	containerdarchive "github.com/containerd/containerd/archive"
-	"github.com/docker/docker/pkg/system"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/mudler/luet/pkg/api/core/types"
 	"github.com/pkg/errors"
 )
 
-// Extract dir:
-// -> First extract delta considering the dir
-// Afterward create artifact pointing to the dir
-
-// ExtractDeltaFiles returns an handler to extract files in a list
-func ExtractDeltaFiles(
-	ctx *types.Context,
-	d ImageDiff,
+// ExtractDeltaAdditionsFromImages is a filter that takes two images
+// an includes and an excludes list. It computes the delta between the images
+// considering the added files only, and applies a filter on them based on the regexes
+// in the lists.
+func ExtractDeltaAdditionsFiles(
+	ctx types.Context,
+	srcimg v1.Image,
 	includes []string, excludes []string,
-) func(h *tar.Header) (bool, error) {
+) (func(h *tar.Header) (bool, error), error) {
 
 	includeRegexp := compileRegexes(includes)
 	excludeRegexp := compileRegexes(excludes)
 
-	additions := map[string]interface{}{}
-	for _, a := range d.Additions {
-		additions[a.Name] = nil
+	srcfilesd, err := ctx.TempDir("srcfiles")
+	if err != nil {
+		return nil, err
 	}
+	filesSrc := NewCache(srcfilesd, 50*1024*1024, 10000)
+
+	srcReader := mutate.Extract(srcimg)
+	defer srcReader.Close()
+
+	srcTar := tar.NewReader(srcReader)
+
+	for {
+		var hdr *tar.Header
+		hdr, err := srcTar.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			filesSrc.Set(filepath.Dir(hdr.Name), "")
+		default:
+			filesSrc.Set(hdr.Name, "")
+		}
+	}
+
 	return func(h *tar.Header) (bool, error) {
+
 		fileName := filepath.Join(string(os.PathSeparator), h.Name)
-		_, exists := additions[h.Name]
-		if !exists {
+		_, exists := filesSrc.Get(h.Name)
+		if exists {
 			return false, nil
 		}
 
@@ -97,11 +121,14 @@ func ExtractDeltaFiles(
 			return true, nil
 		}
 
-	}
+	}, nil
 }
 
+// ExtractFiles returns a filter that extracts files from the given path (if not empty)
+// It then filters files by an include and exclude list.
+// The list can be regexes
 func ExtractFiles(
-	ctx *types.Context,
+	ctx types.Context,
 	prefixPath string,
 	includes []string, excludes []string,
 ) func(h *tar.Header) (bool, error) {
@@ -163,65 +190,38 @@ func ExtractFiles(
 	}
 }
 
-func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+// ExtractReader perform the extracting action over the io.ReadCloser
+// it extracts the files over output. Accepts a filter as an option
+// and additional containerd Options
+func ExtractReader(ctx types.Context, reader io.ReadCloser, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	defer reader.Close()
 
-	perms := map[string][]int{}
-	xattrs := map[string]map[string]string{}
-	paxrecords := map[string]map[string]string{}
-
-	f := func(h *tar.Header) (bool, error) {
-		perms[h.Name] = []int{h.Gid, h.Uid}
-		xattrs[h.Name] = h.Xattrs
-		paxrecords[h.Name] = h.PAXRecords
-		if filter != nil {
-			return filter(h)
-		}
-		return true, nil
+	// If no filter is specified, grab all.
+	if filter == nil {
+		filter = func(h *tar.Header) (bool, error) { return true, nil }
 	}
 
-	opts = append(opts, containerdarchive.WithFilter(f))
+	opts = append(opts, containerdarchive.WithFilter(filter))
 
+	// Handle the extraction
 	c, err := containerdarchive.Apply(context.Background(), output, reader, opts...)
 	if err != nil {
 		return 0, "", err
 	}
 
-	// TODO: Parametrize this
-	if keepPerms {
-		for f, p := range perms {
-			ff := filepath.Join(output, f)
-			if _, err := os.Lstat(ff); err == nil {
-				if err := os.Lchown(ff, p[1], p[0]); err != nil {
-					ctx.Warning(err, "failed chowning file")
-				}
-			}
-		}
-
-		for _, m := range []map[string]map[string]string{xattrs, paxrecords} {
-			for key, attrs := range m {
-				ff := filepath.Join(output, key)
-				for k, attr := range attrs {
-					if err := system.Lsetxattr(ff, k, []byte(attr), 0); err != nil {
-						if errors.Is(err, syscall.ENOTSUP) {
-							ctx.Debug("ignored xattr %s in archive", key)
-						}
-					}
-				}
-			}
-		}
-	}
 	return c, output, nil
 }
 
-func Extract(ctx *types.Context, img v1.Image, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
-	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
+// Extract is just syntax sugar around ExtractReader. It extracts an image into a dir
+func Extract(ctx types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+	tmpdiffs, err := ctx.TempDir("extraction")
 	if err != nil {
 		return 0, "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
-	return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, keepPerms, filter, opts...)
+	return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, filter, opts...)
 }
 
-func ExtractTo(ctx *types.Context, img v1.Image, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
-	return ExtractReader(ctx, mutate.Extract(img), output, keepPerms, filter, opts...)
+// ExtractTo is just syntax sugar around ExtractReader
+func ExtractTo(ctx types.Context, img v1.Image, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+	return ExtractReader(ctx, mutate.Extract(img), output, filter, opts...)
 }

pkg/api/core/image/extract_test.go

@@ -23,8 +23,8 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	daemon "github.com/google/go-containerregistry/pkg/v1/daemon"
+	"github.com/mudler/luet/pkg/api/core/context"
 	. "github.com/mudler/luet/pkg/api/core/image"
-	"github.com/mudler/luet/pkg/api/core/types"
 	"github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -34,14 +34,14 @@ var _ = Describe("Extract", func() {
 
 	Context("extract files from images", func() {
 		Context("ExtractFiles", func() {
-			ctx := types.NewContext()
+			ctx := context.NewContext()
 			var tmpfile *os.File
 			var ref name.Reference
 			var img v1.Image
 			var err error
 
 			BeforeEach(func() {
-				ctx = types.NewContext()
+				ctx = context.NewContext()
 
 				tmpfile, err = ioutil.TempFile("", "extract")
 				Expect(err).ToNot(HaveOccurred())
@@ -58,7 +58,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "", []string{}, []string{}),
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -72,7 +71,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "/usr", []string{}, []string{}),
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -86,7 +84,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "/usr", []string{"bin"}, []string{"sbin"}),
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -101,7 +98,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "", []string{"/usr|/usr/bin"}, []string{"^/bin"}),
 				)
 				Expect(err).ToNot(HaveOccurred())

pkg/api/core/image/mutator_suite_test.go

@@ -18,7 +18,7 @@ package image_test
 import (
 	"testing"
 
-	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -26,7 +26,7 @@ import (
 
 func TestMutator(t *testing.T) {
 	RegisterFailHandler(Fail)
-	b := backend.NewSimpleDockerBackend(types.NewContext())
+	b := backend.NewSimpleDockerBackend(context.NewContext())
 	b.DownloadImage(backend.Options{ImageName: "alpine"})
 	b.DownloadImage(backend.Options{ImageName: "golang:alpine"})
 

pkg/api/core/logger/config_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package logger_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestAPITypes(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Types Suite")
+}

pkg/api/core/logger/logger.go

@@ -0,0 +1,367 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package logger
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"regexp"
+	"runtime"
+	"strings"
+	"sync"
+
+	log "github.com/ipfs/go-log/v2"
+	"github.com/kyokomi/emoji"
+	"github.com/pterm/pterm"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+// Logger is the default logger
+type Logger struct {
+	level       log.LogLevel
+	emoji       bool
+	logToFile   bool
+	noSpinner   bool
+	fileLogger  *zap.Logger
+	context     string
+	spinnerLock sync.Mutex
+	s           *pterm.SpinnerPrinter
+}
+
+// LogLevel represents a log severity level. Use the package variables as an
+// enum.
+type LogLevel zapcore.Level
+
+type LoggerOptions func(*Logger) error
+
+var NoSpinner LoggerOptions = func(l *Logger) error {
+	l.noSpinner = true
+	return nil
+}
+
+func WithLevel(level string) LoggerOptions {
+	return func(l *Logger) error {
+		lvl, _ := log.LevelFromString(level) // Defaults to Info
+		l.level = lvl
+		if l.level == log.LevelDebug {
+			pterm.EnableDebugMessages()
+		}
+		return nil
+	}
+}
+
+func WithContext(c string) LoggerOptions {
+	return func(l *Logger) error {
+		l.context = c
+		return nil
+	}
+}
+
+func WithFileLogging(p, encoding string) LoggerOptions {
+	return func(l *Logger) error {
+		if encoding == "" {
+			encoding = "console"
+		}
+		l.logToFile = true
+		var err error
+		cfg := zap.NewProductionConfig()
+		cfg.OutputPaths = []string{p}
+		cfg.Level = zap.NewAtomicLevelAt(zapcore.Level(l.level))
+		cfg.ErrorOutputPaths = []string{}
+		cfg.Encoding = encoding
+		cfg.DisableCaller = true
+		cfg.DisableStacktrace = true
+		cfg.EncoderConfig.TimeKey = "time"
+		cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+
+		l.fileLogger, err = cfg.Build()
+		return err
+	}
+}
+
+var EnableEmoji = func() LoggerOptions {
+	return func(l *Logger) error {
+		l.emoji = true
+		return nil
+	}
+}
+
+func New(opts ...LoggerOptions) (*Logger, error) {
+	l := &Logger{
+		level: log.LevelDebug,
+		s:     pterm.DefaultSpinner.WithShowTimer(false).WithRemoveWhenDone(true),
+	}
+	for _, o := range opts {
+		if err := o(l); err != nil {
+			return nil, err
+		}
+	}
+
+	return l, nil
+}
+
+func (l *Logger) Copy(opts ...LoggerOptions) (*Logger, error) {
+	c := *l
+	copy := &c
+	for _, o := range opts {
+		if err := o(copy); err != nil {
+			return nil, err
+		}
+	}
+
+	return copy, nil
+}
+
+func joinMsg(args ...interface{}) (message string) {
+	for _, m := range args {
+		message += " " + fmt.Sprintf("%v", m)
+	}
+	return
+}
+
+func (l *Logger) enabled(lvl log.LogLevel) bool {
+	return lvl >= l.level
+}
+
+var emojiStrip = regexp.MustCompile(`[:][\w]+[:]`)
+
+func (l *Logger) transform(args ...interface{}) (sanitized []interface{}) {
+	for _, a := range args {
+		var aString string
+
+		// Strip emoji if needed
+		if l.emoji {
+			aString = emoji.Sprint(a)
+		} else {
+			aString = emojiStrip.ReplaceAllString(joinMsg(a), "")
+		}
+
+		sanitized = append(sanitized, aString)
+	}
+
+	if l.context != "" {
+		sanitized = append([]interface{}{fmt.Sprintf("(%s)", l.context)}, sanitized...)
+	}
+	return
+}
+
+func prefixCodeLine(args ...interface{}) []interface{} {
+	pc, file, line, ok := runtime.Caller(3)
+	if ok {
+		args = append([]interface{}{fmt.Sprintf("(%s:#%d:%v)",
+			path.Base(file), line, runtime.FuncForPC(pc).Name())}, args...)
+	}
+	return args
+}
+
+func (l *Logger) send(ll log.LogLevel, f string, args ...interface{}) {
+	if !l.enabled(ll) {
+		return
+	}
+
+	sanitizedArgs := joinMsg(l.transform(args...)...)
+	sanitizedF := joinMsg(l.transform(f)...)
+	formatDefined := f != ""
+
+	switch {
+	case log.LevelDebug == ll && !formatDefined:
+		pterm.Debug.Println(prefixCodeLine(sanitizedArgs)...)
+		if l.logToFile {
+			l.fileLogger.Debug(joinMsg(prefixCodeLine(sanitizedArgs)...))
+		}
+	case log.LevelDebug == ll && formatDefined:
+		pterm.Debug.Printfln(joinMsg(prefixCodeLine(sanitizedF)...), args...)
+		if l.logToFile {
+			l.fileLogger.Sugar().Debugf(joinMsg(prefixCodeLine(sanitizedF)...), args...)
+		}
+	case log.LevelError == ll && !formatDefined:
+		pterm.Error.Println(pterm.LightRed(sanitizedArgs))
+		if l.logToFile {
+			l.fileLogger.Error(sanitizedArgs)
+		}
+	case log.LevelError == ll && formatDefined:
+		pterm.Error.Printfln(pterm.LightRed(sanitizedF), args...)
+		if l.logToFile {
+			l.fileLogger.Sugar().Errorf(sanitizedF, args...)
+		}
+
+	case log.LevelFatal == ll && !formatDefined:
+		pterm.Error.Println(sanitizedArgs)
+		if l.logToFile {
+			l.fileLogger.Error(sanitizedArgs)
+		}
+	case log.LevelFatal == ll && formatDefined:
+		pterm.Error.Printfln(sanitizedF, args...)
+		if l.logToFile {
+			l.fileLogger.Sugar().Errorf(sanitizedF, args...)
+		}
+		//INFO
+	case log.LevelInfo == ll && !formatDefined:
+		pterm.Info.Println(sanitizedArgs)
+		if l.logToFile {
+			l.fileLogger.Info(sanitizedArgs)
+		}
+	case log.LevelInfo == ll && formatDefined:
+		pterm.Info.Printfln(sanitizedF, args...)
+		if l.logToFile {
+			l.fileLogger.Sugar().Infof(sanitizedF, args...)
+		}
+		//WARN
+	case log.LevelWarn == ll && !formatDefined:
+		pterm.Warning.Println(sanitizedArgs)
+		if l.logToFile {
+			l.fileLogger.Warn(sanitizedArgs)
+		}
+	case log.LevelWarn == ll && formatDefined:
+		pterm.Warning.Printfln(sanitizedF, args...)
+		if l.logToFile {
+			l.fileLogger.Sugar().Warnf(sanitizedF, args...)
+		}
+	}
+}
+
+func (l *Logger) Debug(args ...interface{}) {
+	l.send(log.LevelDebug, "", args...)
+}
+
+func (l *Logger) Error(args ...interface{}) {
+	l.send(log.LevelError, "", args...)
+}
+
+func (l *Logger) Trace(args ...interface{}) {
+	l.send(log.LevelDebug, "", args...)
+}
+
+func (l *Logger) Tracef(t string, args ...interface{}) {
+	l.send(log.LevelDebug, t, args...)
+}
+
+func (l *Logger) Fatal(args ...interface{}) {
+	l.send(log.LevelFatal, "", args...)
+	os.Exit(1)
+}
+
+func (l *Logger) Info(args ...interface{}) {
+	l.send(log.LevelInfo, "", args...)
+}
+
+func (l *Logger) Success(args ...interface{}) {
+	l.Info(append([]interface{}{"SUCCESS"}, args...)...)
+}
+
+func (l *Logger) Panic(args ...interface{}) {
+	l.Fatal(args...)
+}
+
+func (l *Logger) Warn(args ...interface{}) {
+	l.send(log.LevelWarn, "", args...)
+}
+
+func (l *Logger) Warning(args ...interface{}) {
+	l.Warn(args...)
+}
+
+func (l *Logger) Debugf(f string, args ...interface{}) {
+	l.send(log.LevelDebug, f, args...)
+}
+
+func (l *Logger) Errorf(f string, args ...interface{}) {
+	l.send(log.LevelError, f, args...)
+}
+
+func (l *Logger) Fatalf(f string, args ...interface{}) {
+	l.send(log.LevelFatal, f, args...)
+}
+
+func (l *Logger) Infof(f string, args ...interface{}) {
+	l.send(log.LevelInfo, f, args...)
+}
+
+func (l *Logger) Panicf(f string, args ...interface{}) {
+	l.Fatalf(joinMsg(f), args...)
+}
+
+func (l *Logger) Warnf(f string, args ...interface{}) {
+	l.send(log.LevelWarn, f, args...)
+}
+
+func (l *Logger) Warningf(f string, args ...interface{}) {
+	l.Warnf(f, args...)
+}
+
+func (l *Logger) Ask() bool {
+	var input string
+
+	l.Info("Do you want to continue with this operation? [y/N]: ")
+	_, err := fmt.Scanln(&input)
+	if err != nil {
+		return false
+	}
+	input = strings.ToLower(input)
+
+	if input == "y" || input == "yes" {
+		return true
+	}
+	return false
+}
+
+// Spinner starts the spinner
+func (l *Logger) Spinner() {
+	if !IsTerminal() || l.noSpinner {
+		return
+	}
+
+	l.spinnerLock.Lock()
+	defer l.spinnerLock.Unlock()
+
+	if l.s != nil && !l.s.IsActive {
+		l.s, _ = l.s.Start()
+	}
+}
+
+func (l *Logger) Screen(text string) {
+	pterm.DefaultHeader.WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(2).Println(text)
+}
+
+func (l *Logger) SpinnerText(suffix, prefix string) {
+	if !IsTerminal() || l.noSpinner {
+		return
+	}
+	l.spinnerLock.Lock()
+	defer l.spinnerLock.Unlock()
+
+	if l.level == log.LevelDebug {
+		fmt.Printf("%s %s\n",
+			suffix, prefix,
+		)
+	} else {
+		l.s.UpdateText(suffix + prefix)
+	}
+}
+
+func (l *Logger) SpinnerStop() {
+	if !IsTerminal() || l.noSpinner {
+		return
+	}
+	l.spinnerLock.Lock()
+	defer l.spinnerLock.Unlock()
+
+	if l.s != nil {
+		l.s.Success()
+	}
+}

pkg/api/core/logger/logger_test.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package types_test
+package logger_test
 
 import (
 	"io"
@@ -21,7 +21,8 @@ import (
 	"os"
 
 	"github.com/gookit/color"
-	types "github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/logger"
+	. "github.com/mudler/luet/pkg/api/core/logger"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -44,30 +45,11 @@ func captureStdout(f func(w io.Writer)) string {
 }
 
 var _ = Describe("Context and logging", func() {
-	ctx := types.NewContext()
-
-	BeforeEach(func() {
-		ctx = types.NewContext()
-	})
-
-	Context("LogLevel", func() {
-		It("converts it correctly to number and zaplog", func() {
-			Expect(types.ErrorLevel.ToNumber()).To(Equal(0))
-			Expect(types.InfoLevel.ToNumber()).To(Equal(2))
-			Expect(types.WarningLevel.ToNumber()).To(Equal(1))
-			Expect(types.LogLevel("foo").ToNumber()).To(Equal(3))
-			Expect(types.WarningLevel.ZapLevel().String()).To(Equal("warn"))
-			Expect(types.InfoLevel.ZapLevel().String()).To(Equal("info"))
-			Expect(types.ErrorLevel.ZapLevel().String()).To(Equal("error"))
-			Expect(types.FatalLevel.ZapLevel().String()).To(Equal("fatal"))
-			Expect(types.LogLevel("foo").ZapLevel().String()).To(Equal("debug"))
-		})
-	})
 
 	Context("Context", func() {
 		It("detect if is a terminal", func() {
 			Expect(captureStdout(func(w io.Writer) {
-				_, _, err := ctx.GetTerminalSize()
+				_, _, err := GetTerminalSize()
 				Expect(err).To(HaveOccurred())
 				Expect(err.Error()).To(Equal("size not detectable"))
 				os.Stdout.Write([]byte(err.Error()))
@@ -75,47 +57,71 @@ var _ = Describe("Context and logging", func() {
 		})
 
 		It("respects loglevel", func() {
-			ctx.Config.GetGeneral().Debug = false
+
+			l, err := New(WithLevel("info"))
+			Expect(err).ToNot(HaveOccurred())
+
 			Expect(captureStdout(func(w io.Writer) {
-				ctx.Debug("")
+				l.Debug("")
 			})).To(Equal(""))
 
-			ctx.Config.GetGeneral().Debug = true
+			l, err = New(WithLevel("debug"))
+			Expect(err).ToNot(HaveOccurred())
+
 			Expect(captureStdout(func(w io.Writer) {
-				ctx.Debug("foo")
+				l.Debug("foo")
 			})).To(ContainSubstring("foo"))
 		})
 
+		It("logs with context", func() {
+			l, err := New(WithLevel("debug"), WithContext("foo"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(captureStdout(func(w io.Writer) {
+				l.Debug("bar")
+			})).To(ContainSubstring("(foo)  bar"))
+		})
+
+		It("returns copies with logged context", func() {
+			l, err := New(WithLevel("debug"))
+			l, _ = l.Copy(logger.WithContext("bazzz"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(captureStdout(func(w io.Writer) {
+				l.Debug("bar")
+			})).To(ContainSubstring("(bazzz)  bar"))
+		})
+
 		It("logs to file", func() {
-			ctx.NoColor()
 
 			t, err := ioutil.TempFile("", "tree")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(t.Name()) // clean up
-			ctx.Config.GetLogging().EnableLogFile = true
-			ctx.Config.GetLogging().Path = t.Name()
 
-			ctx.Init()
+			l, err := New(WithLevel("debug"), WithFileLogging(t.Name(), ""))
+			Expect(err).ToNot(HaveOccurred())
+
+			//	ctx.Init()
 
 			Expect(captureStdout(func(w io.Writer) {
-				ctx.Info("foot")
+				l.Info("foot")
 			})).To(And(ContainSubstring("INFO"), ContainSubstring("foot")))
 
 			Expect(captureStdout(func(w io.Writer) {
-				ctx.Success("test")
+				l.Success("test")
 			})).To(And(ContainSubstring("SUCCESS"), ContainSubstring("test")))
 
 			Expect(captureStdout(func(w io.Writer) {
-				ctx.Error("foobar")
+				l.Error("foobar")
 			})).To(And(ContainSubstring("ERROR"), ContainSubstring("foobar")))
 
 			Expect(captureStdout(func(w io.Writer) {
-				ctx.Warning("foowarn")
+				l.Warning("foowarn")
 			})).To(And(ContainSubstring("WARNING"), ContainSubstring("foowarn")))
 
-			l, err := ioutil.ReadFile(t.Name())
+			ll, err := ioutil.ReadFile(t.Name())
 			Expect(err).ToNot(HaveOccurred())
-			logs := string(l)
+			logs := string(ll)
 			Expect(logs).To(ContainSubstring("foot"))
 			Expect(logs).To(ContainSubstring("test"))
 			Expect(logs).To(ContainSubstring("foowarn"))

pkg/api/core/logger/terminal.go

@@ -0,0 +1,43 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package logger
+
+import (
+	"errors"
+	"os"
+
+	"github.com/mattn/go-isatty"
+	"golang.org/x/term"
+)
+
+func IsTerminal() bool {
+	return isatty.IsTerminal(os.Stdout.Fd())
+}
+
+// GetTerminalSize returns the width and the height of the active terminal.
+func GetTerminalSize() (width, height int, err error) {
+	w, h, err := term.GetSize(int(os.Stdout.Fd()))
+	if w <= 0 {
+		w = 0
+	}
+	if h <= 0 {
+		h = 0
+	}
+	if err != nil {
+		err = errors.New("size not detectable")
+	}
+	return w, h, err
+}

pkg/api/core/types/artifact/artifact.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 
 	"github.com/docker/docker/pkg/pools"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -40,7 +41,7 @@ import (
 	bus "github.com/mudler/luet/pkg/api/core/bus"
 	config "github.com/mudler/luet/pkg/api/core/config"
 	"github.com/mudler/luet/pkg/api/core/image"
-	types "github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/types"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
@@ -69,8 +70,8 @@ type PackageArtifact struct {
 	Runtime           *pkg.DefaultPackage               `json:"runtime,omitempty"`
 }
 
-func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, keepPerms bool, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
-	_, tmpdiffs, err := image.Extract(ctx, img, keepPerms, filter)
+func ImageToArtifact(ctx types.Context, img v1.Image, t compression.Implementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
+	_, tmpdiffs, err := image.Extract(ctx, img, filter)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
@@ -174,19 +175,13 @@ func (a *PackageArtifact) GetFileName() string {
 	return path.Base(a.Path)
 }
 
-func (a *PackageArtifact) genDockerfile() string {
-	return `
-FROM scratch
-COPY . /`
-}
-
 // CreateArtifactForFile creates a new artifact from the given file
-func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
+func CreateArtifactForFile(ctx types.Context, s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
 	if _, err := os.Stat(s); os.IsNotExist(err) {
 		return nil, errors.Wrap(err, "artifact path doesn't exist")
 	}
 	fileName := path.Base(s)
-	archive, err := ctx.Config.GetSystem().TempDir("archive")
+	archive, err := ctx.TempDir("archive")
 	if err != nil {
 		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
 	}
@@ -196,7 +191,7 @@ func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageAr
 		return nil, errors.Wrapf(err, "error while copying %s to %s", s, dst)
 	}
 
-	artifact, err := ctx.Config.GetSystem().TempDir("artifact")
+	artifact, err := ctx.TempDir("artifact")
 	if err != nil {
 		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
 	}
@@ -211,52 +206,26 @@ func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageAr
 
 type ImageBuilder interface {
 	BuildImage(backend.Options) error
+	LoadImage(path string) error
 }
 
 // GenerateFinalImage takes an artifact and builds a Docker image with its content
-func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) {
-	builderOpts := backend.Options{}
-	archive, err := ctx.Config.GetSystem().TempDir("archive")
+func (a *PackageArtifact) GenerateFinalImage(ctx types.Context, imageName string, b ImageBuilder, keepPerms bool) error {
+
+	tempimage, err := ctx.TempFile("tempimage")
 	if err != nil {
-		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+		return errors.Wrap(err, "error met while creating tempdir for "+a.Path)
 	}
-	defer os.RemoveAll(archive) // clean up
+	defer os.RemoveAll(tempimage.Name()) // clean up
 
-	uncompressedFiles := filepath.Join(archive, "files")
-	dockerFile := filepath.Join(archive, "Dockerfile")
-
-	if err := os.MkdirAll(uncompressedFiles, os.ModePerm); err != nil {
-		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	if err := image.CreateTar(a.Path, tempimage.Name(), imageName, runtime.GOARCH, runtime.GOOS); err != nil {
+		return errors.Wrap(err, "could not create image from tar")
 	}
 
-	if err := a.Unpack(ctx, uncompressedFiles, keepPerms); err != nil {
-		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
+	if err := b.LoadImage(tempimage.Name()); err != nil {
+		return errors.Wrap(err, "while loading image")
 	}
-
-	empty, err := fileHelper.DirectoryIsEmpty(uncompressedFiles)
-	if err != nil {
-		return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles)
-	}
-
-	// See https://github.com/moby/moby/issues/38039.
-	// We can't generate FROM scratch empty images. Docker will refuse to export them
-	// workaround: Inject a .virtual empty file
-	if empty {
-		fileHelper.Touch(filepath.Join(uncompressedFiles, ".virtual"))
-	}
-
-	data := a.genDockerfile()
-	if err := ioutil.WriteFile(dockerFile, []byte(data), 0644); err != nil {
-		return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path)
-	}
-
-	builderOpts = backend.Options{
-		ImageName:      imageName,
-		SourcePath:     archive,
-		DockerFileName: dockerFile,
-		Context:        uncompressedFiles,
-	}
-	return builderOpts, b.BuildImage(builderOpts)
+	return nil
 }
 
 // Compress is responsible to archive and compress to the artifact Path.
@@ -459,7 +428,7 @@ func replaceFileTarWrapper(dst string, inputTarStream io.ReadCloser, mods []stri
 	return pipeReader
 }
 
-func tarModifierWrapperFunc(ctx *types.Context) func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+func tarModifierWrapperFunc(ctx types.Context) func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
 	return func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
 		// If the destination path already exists I rename target file name with postfix.
 		var destPath string
@@ -526,10 +495,10 @@ func tarModifierWrapperFunc(ctx *types.Context) func(dst, path string, header *t
 	}
 }
 
-func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) (res []string) {
+func (a *PackageArtifact) GetProtectFiles(ctx types.Context) (res []string) {
 	annotationDir := ""
 
-	if !ctx.Config.ConfigProtectSkip {
+	if !ctx.GetConfig().ConfigProtectSkip {
 
 		// a.CompileSpec could be nil when artifact.Unpack is used for tree tarball
 		if a.CompileSpec != nil &&
@@ -542,7 +511,7 @@ func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) (res []string) {
 		// TODO: check if skip this if we have a.CompileSpec nil
 
 		cp := config.NewConfigProtect(annotationDir)
-		cp.Map(a.Files, ctx.Config.GetConfigProtectConfFiles())
+		cp.Map(a.Files, ctx.GetConfig().ConfigProtectConfFiles)
 
 		// NOTE: for unpack we need files path without initial /
 		res = cp.GetProtectFiles(false)
@@ -552,7 +521,7 @@ func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) (res []string) {
 }
 
 // Unpack Untar and decompress (TODO) to the given path
-func (a *PackageArtifact) Unpack(ctx *types.Context, dst string, keepPerms bool) error {
+func (a *PackageArtifact) Unpack(ctx types.Context, dst string, keepPerms bool) error {
 
 	if !strings.HasPrefix(dst, string(os.PathSeparator)) {
 		return errors.New("destination must be an absolute path")
@@ -591,7 +560,7 @@ func (a *PackageArtifact) Unpack(ctx *types.Context, dst string, keepPerms bool)
 	// 	//	tarModifier.Modifier()
 	// 	return true, nil
 	// },
-	_, _, err = image.ExtractReader(ctx, replacerArchive, dst, ctx.Config.GetGeneral().SameOwner, nil)
+	_, _, err = image.ExtractReader(ctx, replacerArchive, dst, nil)
 	return err
 }
 

pkg/api/core/types/artifact/artifact_test.go

@@ -20,10 +20,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/api/core/image"
-	"github.com/mudler/luet/pkg/api/core/types"
 	. "github.com/mudler/luet/pkg/api/core/types/artifact"
-	. "github.com/mudler/luet/pkg/compiler/backend"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
 	"github.com/mudler/luet/pkg/compiler/types/options"
@@ -39,7 +38,7 @@ import (
 
 var _ = Describe("Artifact", func() {
 	Context("Simple package build definition", func() {
-		ctx := types.NewContext()
+		ctx := context.NewContext()
 		It("Generates a verified delta", func() {
 
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
@@ -49,7 +48,7 @@ var _ = Describe("Artifact", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase(), options.WithContext(context.NewContext()))
 			lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -83,7 +82,7 @@ WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
 ENV PACKAGE_CATEGORY=app-admin`))
-			b := NewSimpleDockerBackend(ctx)
+			b := backend.NewSimpleDockerBackend(ctx)
 			opts := backend.Options{
 				ImageName:      "luet/base",
 				SourcePath:     tmpdir,
@@ -120,7 +119,7 @@ RUN echo bar > /test2`))
 		})
 
 		It("Generates packages images", func() {
-			b := NewSimpleDockerBackend(ctx)
+			b := backend.NewSimpleDockerBackend(ctx)
 			imageprefix := "foo/"
 			testString := []byte(`funky test data`)
 
@@ -146,9 +145,8 @@ RUN echo bar > /test2`))
 			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false)
+			err = a.GenerateFinalImage(ctx, resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(opts.ImageName).To(Equal(resultingImage))
 
 			Expect(b.ImageExists(resultingImage)).To(BeTrue())
 
@@ -156,13 +154,12 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(result) // clean up
 
-			img, err := b.ImageReference(resultingImage)
+			img, err := b.ImageReference(resultingImage, true)
 			Expect(err).ToNot(HaveOccurred())
 			_, _, err = image.ExtractTo(
 				ctx,
 				img,
 				result,
-				false,
 				nil,
 			)
 			Expect(err).ToNot(HaveOccurred())
@@ -179,7 +176,7 @@ RUN echo bar > /test2`))
 		})
 
 		It("Generates empty packages images", func() {
-			b := NewSimpleDockerBackend(ctx)
+			b := backend.NewSimpleDockerBackend(ctx)
 			imageprefix := "foo/"
 
 			tmpdir, err := ioutil.TempDir(os.TempDir(), "artifact")
@@ -196,9 +193,8 @@ RUN echo bar > /test2`))
 			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false)
+			err = a.GenerateFinalImage(ctx, resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(opts.ImageName).To(Equal(resultingImage))
 
 			Expect(b.ImageExists(resultingImage)).To(BeTrue())
 
@@ -206,22 +202,17 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(result) // clean up
 
-			img, err := b.ImageReference(resultingImage)
+			img, err := b.ImageReference(resultingImage, false)
 			Expect(err).ToNot(HaveOccurred())
 			_, _, err = image.ExtractTo(
 				ctx,
 				img,
 				result,
-				false,
 				nil,
 			)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(fileHelper.DirectoryIsEmpty(result)).To(BeFalse())
-			content, err := ioutil.ReadFile(filepath.Join(result, ".virtual"))
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(string(content)).To(Equal(""))
+			Expect(fileHelper.DirectoryIsEmpty(result)).To(BeTrue())
 		})
 
 		It("Retrieves uncompressed name", func() {

pkg/api/core/types/artifact/cache_test.go

@@ -20,7 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
-	types "github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/context"
 	. "github.com/mudler/luet/pkg/api/core/types/artifact"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
@@ -59,7 +59,7 @@ var _ = Describe("Cache", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			b := NewPackageArtifact(path)
-			ctx := types.NewContext()
+			ctx := context.NewContext()
 			err = b.Unpack(ctx, tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 

pkg/api/core/types/artifact/checksum.go

@@ -45,7 +45,7 @@ type HashOptions struct {
 
 func (c Checksums) List() (res [][]string) {
 	keys := make([]string, 0)
-	for k, _ := range c {
+	for k := range c {
 		keys = append(keys, k)
 	}
 	sort.Strings(keys)

pkg/api/core/types/config.go

@@ -1,6 +1,4 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//                  Daniele Rondina <geaaru@sabayonlinux.org>
-//             2021 Ettore Di Giacinto <mudler@mocaccino.org>
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@gentoo.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -39,19 +37,22 @@ var AvailableResolvers = strings.Join([]string{solver.QLearningResolverType}, "
 
 type LuetLoggingConfig struct {
 	// Path of the logfile
-	Path string `mapstructure:"path"`
+	Path string `yaml:"path" mapstructure:"path"`
 	// Enable/Disable logging to file
-	EnableLogFile bool `mapstructure:"enable_logfile"`
+	EnableLogFile bool `yaml:"enable_logfile" mapstructure:"enable_logfile"`
 	// Enable JSON format logging in file
-	JsonFormat bool `mapstructure:"json_format"`
+	JsonFormat bool `yaml:"json_format" mapstructure:"json_format"`
 
 	// Log level
-	Level LogLevel `mapstructure:"level"`
+	Level string `yaml:"level" mapstructure:"level"`
 
 	// Enable emoji
-	EnableEmoji bool `mapstructure:"enable_emoji"`
+	EnableEmoji bool `yaml:"enable_emoji" mapstructure:"enable_emoji"`
 	// Enable/Disable color in logging
-	Color bool `mapstructure:"color"`
+	Color bool `yaml:"color" mapstructure:"color"`
+
+	// NoSpinner disable spinner
+	NoSpinner bool `yaml:"no_spinner" mapstructure:"no_spinner"`
 }
 
 type LuetGeneralConfig struct {
@@ -61,6 +62,7 @@ type LuetGeneralConfig struct {
 	ShowBuildOutput bool `yaml:"show_build_output,omitempty" mapstructure:"show_build_output"`
 	FatalWarns      bool `yaml:"fatal_warnings,omitempty" mapstructure:"fatal_warnings"`
 	HTTPTimeout     int  `yaml:"http_timeout,omitempty" mapstructure:"http_timeout"`
+	Quiet           bool `yaml:"quiet" mapstructure:"quiet"`
 }
 
 type LuetSolverOptions struct {
@@ -107,8 +109,42 @@ type LuetSystemConfig struct {
 	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
 }
 
-func (s *LuetSystemConfig) SetRootFS(path string) error {
-	p, err := fileHelper.Rel2Abs(path)
+// Init reads the config and replace user-defined paths with
+// absolute paths where necessary, and construct the paths for the cache
+// and database on the real system
+func (c *LuetConfig) Init() error {
+	if err := c.System.init(); err != nil {
+		return err
+	}
+
+	if err := c.loadConfigProtect(); err != nil {
+		return err
+	}
+
+	// Load repositories
+	if err := c.loadRepositories(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *LuetSystemConfig) init() error {
+	if err := s.setRootfs(); err != nil {
+		return err
+	}
+
+	if err := s.setDBPath(); err != nil {
+		return err
+	}
+
+	s.setCachePath()
+
+	return nil
+}
+
+func (s *LuetSystemConfig) setRootfs() error {
+	p, err := fileHelper.Rel2Abs(s.Rootfs)
 	if err != nil {
 		return err
 	}
@@ -117,9 +153,8 @@ func (s *LuetSystemConfig) SetRootFS(path string) error {
 	return nil
 }
 
-func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
-	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
-	dbpath = filepath.Join(dbpath, "repos/"+name)
+func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+	dbpath := filepath.Join(sc.DatabasePath, "repos/"+name)
 	err := os.MkdirAll(dbpath, os.ModePerm)
 	if err != nil {
 		panic(err)
@@ -127,43 +162,48 @@ func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	return dbpath
 }
 
-func (sc *LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+func (sc *LuetSystemConfig) setDBPath() error {
 	dbpath := filepath.Join(sc.Rootfs,
 		sc.DatabasePath)
 	err := os.MkdirAll(dbpath, os.ModePerm)
 	if err != nil {
-		panic(err)
+		return err
 	}
-	return dbpath
+	sc.DatabasePath = dbpath
+	return nil
 }
 
-func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+func (sc *LuetSystemConfig) setCachePath() {
 	var cachepath string
 	if sc.PkgsCachePath != "" {
-		cachepath = sc.PkgsCachePath
+		if !filepath.IsAbs(cachepath) {
+			cachepath = filepath.Join(sc.DatabasePath, sc.PkgsCachePath)
+			os.MkdirAll(cachepath, os.ModePerm)
+		} else {
+			cachepath = sc.PkgsCachePath
+		}
 	} else {
 		// Create dynamic cache for test suites
 		cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
 	}
 
-	if filepath.IsAbs(cachepath) {
-		ans = cachepath
-	} else {
-		ans = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
-	}
-
-	return
+	sc.PkgsCachePath = cachepath // Be consistent with the path we set
 }
 
-func (sc *LuetSystemConfig) GetRootFsAbs() (string, error) {
-	return filepath.Abs(sc.Rootfs)
-}
-
-type LuetKV struct {
+type FinalizerEnv struct {
 	Key   string `json:"key" yaml:"key" mapstructure:"key"`
 	Value string `json:"value" yaml:"value" mapstructure:"value"`
 }
 
+type Finalizers []FinalizerEnv
+
+func (f Finalizers) Slice() (sl []string) {
+	for _, kv := range f {
+		sl = append(sl, fmt.Sprintf("%s=%s", kv.Key, kv.Value))
+	}
+	return
+}
+
 type LuetConfig struct {
 	Logging LuetLoggingConfig `yaml:"logging,omitempty" mapstructure:"logging"`
 	General LuetGeneralConfig `yaml:"general,omitempty" mapstructure:"general"`
@@ -176,16 +216,16 @@ type LuetConfig struct {
 	ConfigFromHost       bool             `yaml:"config_from_host,omitempty" mapstructure:"config_from_host"`
 	SystemRepositories   LuetRepositories `yaml:"repositories,omitempty" mapstructure:"repositories"`
 
-	FinalizerEnvs []LuetKV `json:"finalizer_envs,omitempty" yaml:"finalizer_envs,omitempty" mapstructure:"finalizer_envs,omitempty"`
+	FinalizerEnvs Finalizers `json:"finalizer_envs,omitempty" yaml:"finalizer_envs,omitempty" mapstructure:"finalizer_envs,omitempty"`
 
 	ConfigProtectConfFiles []config.ConfigProtectConfFile `yaml:"-" mapstructure:"-"`
 }
 
 func (c *LuetConfig) GetSystemDB() pkg.PackageDatabase {
-	switch c.GetSystem().DatabaseEngine {
+	switch c.System.DatabaseEngine {
 	case "boltdb":
 		return pkg.NewBoltDatabase(
-			filepath.Join(c.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+			filepath.Join(c.System.DatabasePath, "luet.db"))
 	default:
 		return pkg.NewInMemoryDatabase(true)
 	}
@@ -195,83 +235,30 @@ func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
 	c.SystemRepositories = append(c.SystemRepositories, r)
 }
 
-func (c *LuetConfig) GetFinalizerEnvsMap() map[string]string {
-	ans := make(map[string]string)
-
-	for _, kv := range c.FinalizerEnvs {
-		ans[kv.Key] = kv.Value
-	}
-	return ans
-}
-
 func (c *LuetConfig) SetFinalizerEnv(k, v string) {
 	keyPresent := false
-	envs := []LuetKV{}
+	envs := []FinalizerEnv{}
 
 	for _, kv := range c.FinalizerEnvs {
 		if kv.Key == k {
 			keyPresent = true
-			envs = append(envs, LuetKV{Key: kv.Key, Value: v})
+			envs = append(envs, FinalizerEnv{Key: kv.Key, Value: v})
 		} else {
 			envs = append(envs, kv)
 		}
 	}
 	if !keyPresent {
-		envs = append(envs, LuetKV{Key: k, Value: v})
+		envs = append(envs, FinalizerEnv{Key: k, Value: v})
 	}
 
 	c.FinalizerEnvs = envs
 }
 
-func (c *LuetConfig) GetFinalizerEnvs() []string {
-	ans := []string{}
-	for _, kv := range c.FinalizerEnvs {
-		ans = append(ans, fmt.Sprintf("%s=%s", kv.Key, kv.Value))
-	}
-	return ans
-}
-
-func (c *LuetConfig) GetFinalizerEnv(k string) (string, error) {
-	keyNotPresent := true
-	ans := ""
-	for _, kv := range c.FinalizerEnvs {
-		if kv.Key == k {
-			keyNotPresent = false
-			ans = kv.Value
-		}
-	}
-
-	if keyNotPresent {
-		return "", errors.New("Finalizer key " + k + " not found")
-	}
-	return ans, nil
-}
-
-func (c *LuetConfig) GetLogging() *LuetLoggingConfig {
-	return &c.Logging
-}
-
-func (c *LuetConfig) GetGeneral() *LuetGeneralConfig {
-	return &c.General
-}
-
-func (c *LuetConfig) GetSystem() *LuetSystemConfig {
-	return &c.System
-}
-
-func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
-	return &c.Solver
-}
-
 func (c *LuetConfig) YAML() ([]byte, error) {
 	return yaml.Marshal(c)
 }
 
-func (c *LuetConfig) GetConfigProtectConfFiles() []config.ConfigProtectConfFile {
-	return c.ConfigProtectConfFiles
-}
-
-func (c *LuetConfig) AddConfigProtectConfFile(file *config.ConfigProtectConfFile) {
+func (c *LuetConfig) addProtectFile(file *config.ConfigProtectConfFile) {
 	if c.ConfigProtectConfFiles == nil {
 		c.ConfigProtectConfFiles = []config.ConfigProtectConfFile{*file}
 	} else {
@@ -279,28 +266,21 @@ func (c *LuetConfig) AddConfigProtectConfFile(file *config.ConfigProtectConfFile
 	}
 }
 
-func (c *LuetConfig) LoadRepositories(ctx *Context) error {
+func (c *LuetConfig) loadRepositories() error {
 	var regexRepo = regexp.MustCompile(`.yml$|.yaml$`)
-	var err error
 	rootfs := ""
 
 	// Respect the rootfs param on read repositories
 	if !c.ConfigFromHost {
-		rootfs, err = c.GetSystem().GetRootFsAbs()
-		if err != nil {
-			return err
-		}
+		rootfs = c.System.Rootfs
 	}
 
 	for _, rdir := range c.RepositoriesConfDir {
 
 		rdir = filepath.Join(rootfs, rdir)
 
-		ctx.Debug("Parsing Repository Directory", rdir, "...")
-
 		files, err := ioutil.ReadDir(rdir)
 		if err != nil {
-			ctx.Debug("Skip dir", rdir, ":", err.Error())
 			continue
 		}
 
@@ -310,27 +290,20 @@ func (c *LuetConfig) LoadRepositories(ctx *Context) error {
 			}
 
 			if !regexRepo.MatchString(file.Name()) {
-				ctx.Debug("File", file.Name(), "skipped.")
 				continue
 			}
 
 			content, err := ioutil.ReadFile(path.Join(rdir, file.Name()))
 			if err != nil {
-				ctx.Warning("On read file", file.Name(), ":", err.Error())
-				ctx.Warning("File", file.Name(), "skipped.")
 				continue
 			}
 
 			r, err := LoadRepository(content)
 			if err != nil {
-				ctx.Warning("On parse file", file.Name(), ":", err.Error())
-				ctx.Warning("File", file.Name(), "skipped.")
 				continue
 			}
 
 			if r.Name == "" || len(r.Urls) == 0 || r.Type == "" {
-				ctx.Warning("Invalid repository ", file.Name())
-				ctx.Warning("File", file.Name(), "skipped.")
 				continue
 			}
 
@@ -356,28 +329,20 @@ func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
 	return ans, nil
 }
 
-func (c *LuetConfig) LoadConfigProtect(ctx *Context) error {
+func (c *LuetConfig) loadConfigProtect() error {
 	var regexConfs = regexp.MustCompile(`.yml$`)
-	var err error
-
 	rootfs := ""
 
 	// Respect the rootfs param on read repositories
 	if !c.ConfigFromHost {
-		rootfs, err = c.GetSystem().GetRootFsAbs()
-		if err != nil {
-			return err
-		}
+		rootfs = c.System.Rootfs
 	}
 
 	for _, cdir := range c.ConfigProtectConfDir {
 		cdir = filepath.Join(rootfs, cdir)
 
-		ctx.Debug("Parsing Config Protect Directory", cdir, "...")
-
 		files, err := ioutil.ReadDir(cdir)
 		if err != nil {
-			ctx.Debug("Skip dir", cdir, ":", err.Error())
 			continue
 		}
 
@@ -387,38 +352,31 @@ func (c *LuetConfig) LoadConfigProtect(ctx *Context) error {
 			}
 
 			if !regexConfs.MatchString(file.Name()) {
-				ctx.Debug("File", file.Name(), "skipped.")
 				continue
 			}
 
 			content, err := ioutil.ReadFile(path.Join(cdir, file.Name()))
 			if err != nil {
-				ctx.Warning("On read file", file.Name(), ":", err.Error())
-				ctx.Warning("File", file.Name(), "skipped.")
 				continue
 			}
 
-			r, err := loadConfigProtectConFile(file.Name(), content)
+			r, err := loadConfigProtectConfFile(file.Name(), content)
 			if err != nil {
-				ctx.Warning("On parse file", file.Name(), ":", err.Error())
-				ctx.Warning("File", file.Name(), "skipped.")
 				continue
 			}
 
 			if r.Name == "" || len(r.Directories) == 0 {
-				ctx.Warning("Invalid config protect file", file.Name())
-				ctx.Warning("File", file.Name(), "skipped.")
 				continue
 			}
 
-			c.AddConfigProtectConfFile(r)
+			c.addProtectFile(r)
 		}
 	}
 	return nil
 
 }
 
-func loadConfigProtectConFile(filename string, data []byte) (*config.ConfigProtectConfFile, error) {
+func loadConfigProtectConfFile(filename string, data []byte) (*config.ConfigProtectConfFile, error) {
 	ans := config.NewConfigProtectConfFile(filename)
 	err := yaml.Unmarshal(data, &ans)
 	if err != nil {
@@ -426,47 +384,3 @@ func loadConfigProtectConFile(filename string, data []byte) (*config.ConfigProte
 	}
 	return ans, nil
 }
-
-func (c *LuetLoggingConfig) SetLogLevel(s LogLevel) {
-	c.Level = s
-}
-
-func (c *LuetSystemConfig) InitTmpDir() error {
-	if !filepath.IsAbs(c.TmpDirBase) {
-		abs, err := fileHelper.Rel2Abs(c.TmpDirBase)
-		if err != nil {
-			return errors.Wrap(err, "while converting relative path to absolute path")
-		}
-		c.TmpDirBase = abs
-	}
-
-	if _, err := os.Stat(c.TmpDirBase); err != nil {
-		if os.IsNotExist(err) {
-			err = os.MkdirAll(c.TmpDirBase, os.ModePerm)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (c *LuetSystemConfig) CleanupTmpDir() error {
-	return os.RemoveAll(c.TmpDirBase)
-}
-
-func (c *LuetSystemConfig) TempDir(pattern string) (string, error) {
-	err := c.InitTmpDir()
-	if err != nil {
-		return "", err
-	}
-	return ioutil.TempDir(c.TmpDirBase, pattern)
-}
-
-func (c *LuetSystemConfig) TempFile(pattern string) (*os.File, error) {
-	err := c.InitTmpDir()
-	if err != nil {
-		return nil, err
-	}
-	return ioutil.TempFile(c.TmpDirBase, pattern)
-}

pkg/api/core/types/config_test.go

@@ -17,53 +17,86 @@
 package types_test
 
 import (
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
-	types "github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/context"
+	"github.com/mudler/luet/pkg/api/core/types"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 var _ = Describe("Config", func() {
-	Context("Load Repository1", func() {
-		ctx := types.NewContext()
 
-		ctx.Config.RepositoriesConfDir = []string{
-			"../../../../tests/fixtures/repos.conf.d",
-		}
-		err := ctx.Config.LoadRepositories(ctx)
+	Context("Inits paths", func() {
+		t, _ := ioutil.TempDir("", "tests")
+		defer os.RemoveAll(t)
+		c := &types.LuetConfig{
+			System: types.LuetSystemConfig{
+				Rootfs:        t,
+				PkgsCachePath: "foo",
+				DatabasePath:  "baz",
+			}}
+		It("sets default", func() {
+			err := c.Init()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(c.System.Rootfs).To(Equal(t))
+			Expect(c.System.PkgsCachePath).To(Equal(filepath.Join(t, "baz", "foo")))
+			Expect(c.System.DatabasePath).To(Equal(filepath.Join(t, "baz")))
+		})
+	})
+
+	Context("Load Repository1", func() {
+		var ctx *context.Context
+		BeforeEach(func() {
+			ctx = context.NewContext(context.WithConfig(&types.LuetConfig{
+				RepositoriesConfDir: []string{
+					"../../../../tests/fixtures/repos.conf.d",
+				},
+			}))
+			ctx.Config.Init()
+		})
 
 		It("Check Load Repository 1", func() {
-			Expect(err).Should(BeNil())
-			Expect(len(ctx.Config.SystemRepositories)).Should(Equal(2))
-			Expect(ctx.Config.SystemRepositories[0].Name).Should(Equal("test1"))
-			Expect(ctx.Config.SystemRepositories[0].Priority).Should(Equal(999))
-			Expect(ctx.Config.SystemRepositories[0].Type).Should(Equal("disk"))
-			Expect(len(ctx.Config.SystemRepositories[0].Urls)).Should(Equal(1))
-			Expect(ctx.Config.SystemRepositories[0].Urls[0]).Should(Equal("tests/repos/test1"))
+			Expect(len(ctx.GetConfig().SystemRepositories)).Should(Equal(2))
+			Expect(ctx.GetConfig().SystemRepositories[0].Name).Should(Equal("test1"))
+			Expect(ctx.GetConfig().SystemRepositories[0].Priority).Should(Equal(999))
+			Expect(ctx.GetConfig().SystemRepositories[0].Type).Should(Equal("disk"))
+			Expect(len(ctx.GetConfig().SystemRepositories[0].Urls)).Should(Equal(1))
+			Expect(ctx.GetConfig().SystemRepositories[0].Urls[0]).Should(Equal("tests/repos/test1"))
 		})
 
 		It("Chec Load Repository 2", func() {
-			Expect(err).Should(BeNil())
-			Expect(len(ctx.Config.SystemRepositories)).Should(Equal(2))
-			Expect(ctx.Config.SystemRepositories[1].Name).Should(Equal("test2"))
-			Expect(ctx.Config.SystemRepositories[1].Priority).Should(Equal(1000))
-			Expect(ctx.Config.SystemRepositories[1].Type).Should(Equal("disk"))
-			Expect(len(ctx.Config.SystemRepositories[1].Urls)).Should(Equal(1))
-			Expect(ctx.Config.SystemRepositories[1].Urls[0]).Should(Equal("tests/repos/test2"))
+			Expect(len(ctx.GetConfig().SystemRepositories)).Should(Equal(2))
+			Expect(ctx.GetConfig().SystemRepositories[1].Name).Should(Equal("test2"))
+			Expect(ctx.GetConfig().SystemRepositories[1].Priority).Should(Equal(1000))
+			Expect(ctx.GetConfig().SystemRepositories[1].Type).Should(Equal("disk"))
+			Expect(len(ctx.GetConfig().SystemRepositories[1].Urls)).Should(Equal(1))
+			Expect(ctx.GetConfig().SystemRepositories[1].Urls[0]).Should(Equal("tests/repos/test2"))
 		})
 	})
+
 	Context("Simple temporary directory creation", func() {
+		ctx := context.NewContext(context.WithConfig(&types.LuetConfig{
+			System: types.LuetSystemConfig{
+				TmpDirBase: os.TempDir() + "/tmpluet",
+			},
+		}))
+
+		BeforeEach(func() {
+			ctx = context.NewContext(context.WithConfig(&types.LuetConfig{
+				System: types.LuetSystemConfig{
+					TmpDirBase: os.TempDir() + "/tmpluet",
+				},
+			}))
+
+		})
 
 		It("Create Temporary directory", func() {
-			ctx := types.NewContext()
-
-			ctx.Config.GetSystem().TmpDirBase = os.TempDir() + "/tmpluet"
-
-			tmpDir, err := ctx.Config.GetSystem().TempDir("test1")
+			tmpDir, err := ctx.TempDir("test1")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(strings.HasPrefix(tmpDir, filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
 			Expect(fileHelper.Exists(tmpDir)).To(BeTrue())
@@ -72,11 +105,7 @@ var _ = Describe("Config", func() {
 		})
 
 		It("Create Temporary file", func() {
-			ctx := types.NewContext()
-
-			ctx.Config.GetSystem().TmpDirBase = os.TempDir() + "/tmpluet"
-
-			tmpFile, err := ctx.Config.GetSystem().TempFile("testfile1")
+			tmpFile, err := ctx.TempFile("testfile1")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(strings.HasPrefix(tmpFile.Name(), filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
 			Expect(fileHelper.Exists(tmpFile.Name())).To(BeTrue())

pkg/api/core/types/context.go

@@ -15,402 +15,16 @@
 
 package types
 
-import (
-	"context"
-	"fmt"
-	"os"
-	"path"
-	"path/filepath"
-	"regexp"
-	"runtime"
-	"strings"
-	"sync"
+type Context interface {
+	Logger
+	GarbageCollector
+	GetConfig() LuetConfig
+	Copy() Context
+	// SetAnnotation sets generic annotations to hold in a context
+	SetAnnotation(s string, i interface{})
 
-	"github.com/kyokomi/emoji"
-	"github.com/mudler/luet/pkg/helpers/terminal"
-	"github.com/pkg/errors"
-	"github.com/pterm/pterm"
-	"go.uber.org/zap"
-	"go.uber.org/zap/zapcore"
-	"golang.org/x/term"
-)
+	// GetAnnotation gets generic annotations to hold in a context
+	GetAnnotation(s string) interface{}
 
-const (
-	ErrorLevel   LogLevel = "error"
-	WarningLevel LogLevel = "warning"
-	InfoLevel    LogLevel = "info"
-	SuccessLevel LogLevel = "success"
-	FatalLevel   LogLevel = "fatal"
-)
-
-type Context struct {
-	context.Context
-	Config     *LuetConfig
-	IsTerminal bool
-	NoSpinner  bool
-
-	s           *pterm.SpinnerPrinter
-	spinnerLock *sync.Mutex
-	z           *zap.Logger
-	AreaPrinter *pterm.AreaPrinter
-	ProgressBar *pterm.ProgressbarPrinter
-}
-
-func NewContext() *Context {
-	return &Context{
-		spinnerLock: &sync.Mutex{},
-		IsTerminal:  terminal.IsTerminal(os.Stdout),
-		Config: &LuetConfig{
-			ConfigFromHost: true,
-			Logging:        LuetLoggingConfig{},
-			General:        LuetGeneralConfig{},
-			System: LuetSystemConfig{
-				DatabasePath: filepath.Join("var", "db", "packages"),
-				TmpDirBase:   filepath.Join(os.TempDir(), "tmpluet")},
-			Solver: LuetSolverOptions{},
-		},
-		s: pterm.DefaultSpinner.WithShowTimer(false).WithRemoveWhenDone(true),
-	}
-}
-
-func (c *Context) Copy() *Context {
-
-	configCopy := *c.Config
-	configCopy.System = *c.Config.GetSystem()
-	configCopy.General = *c.Config.GetGeneral()
-	configCopy.Logging = *c.Config.GetLogging()
-
-	ctx := *c
-	ctxCopy := &ctx
-	ctxCopy.Config = &configCopy
-
-	return ctxCopy
-}
-
-// GetTerminalSize returns the width and the height of the active terminal.
-func (c *Context) GetTerminalSize() (width, height int, err error) {
-	w, h, err := term.GetSize(int(os.Stdout.Fd()))
-	if w <= 0 {
-		w = 0
-	}
-	if h <= 0 {
-		h = 0
-	}
-	if err != nil {
-		err = errors.New("size not detectable")
-	}
-	return w, h, err
-}
-
-func (c *Context) Init() (err error) {
-	if c.IsTerminal {
-		if !c.Config.Logging.Color {
-			c.Debug("Disabling colors")
-			c.NoColor()
-		}
-	} else {
-		c.Debug("Not a terminal, disabling colors")
-		c.NoColor()
-	}
-
-	c.Debug("Colors", c.Config.GetLogging().Color)
-	c.Debug("Logging level", c.Config.GetLogging().Level)
-	c.Debug("Debug mode", c.Config.GetGeneral().Debug)
-
-	if c.Config.GetLogging().EnableLogFile && c.Config.GetLogging().Path != "" {
-		// Init zap logger
-		err = c.InitZap()
-		if err != nil {
-			return
-		}
-	}
-
-	// Load repositories
-	err = c.Config.LoadRepositories(c)
-	if err != nil {
-		return
-	}
-	return
-}
-
-func (c *Context) NoColor() {
-	pterm.DisableColor()
-}
-
-func (c *Context) Ask() bool {
-	var input string
-
-	c.Info("Do you want to continue with this operation? [y/N]: ")
-	_, err := fmt.Scanln(&input)
-	if err != nil {
-		return false
-	}
-	input = strings.ToLower(input)
-
-	if input == "y" || input == "yes" {
-		return true
-	}
-	return false
-}
-
-func (c *Context) InitZap() error {
-	var err error
-	if c.z == nil {
-		// TODO: test permission for open logfile.
-		cfg := zap.NewProductionConfig()
-		cfg.OutputPaths = []string{c.Config.GetLogging().Path}
-		cfg.Level = c.Config.GetLogging().Level.ZapLevel()
-		cfg.ErrorOutputPaths = []string{}
-		if c.Config.GetLogging().JsonFormat {
-			cfg.Encoding = "json"
-		} else {
-			cfg.Encoding = "console"
-		}
-		cfg.DisableCaller = true
-		cfg.DisableStacktrace = true
-		cfg.EncoderConfig.TimeKey = "time"
-		cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
-
-		c.z, err = cfg.Build()
-		if err != nil {
-			fmt.Fprint(os.Stderr, "Error on initialize file logger: "+err.Error()+"\n")
-			return err
-		}
-	}
-
-	return nil
-}
-
-// Spinner starts the spinner
-func (c *Context) Spinner() {
-	if !c.IsTerminal || c.NoSpinner {
-		return
-	}
-
-	c.spinnerLock.Lock()
-	defer c.spinnerLock.Unlock()
-	var confLevel int
-	if c.Config.GetGeneral().Debug {
-		confLevel = 3
-	} else {
-		confLevel = c.Config.GetLogging().Level.ToNumber()
-	}
-	if 2 > confLevel {
-		return
-	}
-
-	if !c.s.IsActive {
-		c.s, _ = c.s.Start()
-	}
-}
-
-func (c *Context) Screen(text string) {
-	pterm.DefaultHeader.WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(2).Println(text)
-	//pterm.DefaultCenter.Print(pterm.DefaultHeader.WithFullWidth().WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(10).Sprint(text))
-}
-
-func (c *Context) SpinnerText(suffix, prefix string) {
-	if !c.IsTerminal || c.NoSpinner {
-		return
-	}
-
-	c.spinnerLock.Lock()
-	defer c.spinnerLock.Unlock()
-	if c.Config.GetGeneral().Debug {
-		fmt.Printf("%s %s\n",
-			suffix, prefix,
-		)
-	} else {
-		c.s.UpdateText(suffix + prefix)
-	}
-}
-
-func (c *Context) SpinnerStop() {
-	if !c.IsTerminal {
-		return
-	}
-
-	c.spinnerLock.Lock()
-	defer c.spinnerLock.Unlock()
-	var confLevel int
-	if c.Config.GetGeneral().Debug {
-		confLevel = 3
-	} else {
-		confLevel = c.Config.GetLogging().Level.ToNumber()
-	}
-	if 2 > confLevel {
-		return
-	}
-	if c.s != nil {
-		c.s.Success()
-	}
-}
-
-func (c *Context) log2File(level LogLevel, msg string) {
-	switch level {
-	case FatalLevel:
-		c.z.Fatal(msg)
-	case ErrorLevel:
-		c.z.Error(msg)
-	case WarningLevel:
-		c.z.Warn(msg)
-	case InfoLevel, SuccessLevel:
-		c.z.Info(msg)
-	default:
-		c.z.Debug(msg)
-	}
-}
-
-func (c *Context) Msg(level LogLevel, ln bool, msg ...interface{}) {
-	var message string
-	var confLevel, msgLevel int
-
-	if c.Config.GetGeneral().Debug {
-		confLevel = 3
-		pterm.EnableDebugMessages()
-	} else {
-		confLevel = c.Config.GetLogging().Level.ToNumber()
-	}
-	msgLevel = level.ToNumber()
-
-	if msgLevel > confLevel {
-		return
-	}
-
-	for _, m := range msg {
-		message += " " + fmt.Sprintf("%v", m)
-	}
-
-	// Color message
-	levelMsg := message
-
-	if c.Config.GetLogging().Color {
-		switch level {
-		case WarningLevel:
-			levelMsg = pterm.LightYellow(":construction: warning" + message)
-		case InfoLevel:
-			levelMsg = message
-		case SuccessLevel:
-			levelMsg = pterm.LightGreen(message)
-		case ErrorLevel:
-			levelMsg = pterm.Red(message)
-		default:
-			levelMsg = pterm.Blue(message)
-		}
-	}
-
-	// Strip emoji if needed
-	if c.Config.GetLogging().EnableEmoji && c.IsTerminal {
-		levelMsg = emoji.Sprint(levelMsg)
-	} else {
-		re := regexp.MustCompile(`[:][\w]+[:]`)
-		levelMsg = re.ReplaceAllString(levelMsg, "")
-	}
-
-	if c.z != nil {
-		c.log2File(level, message)
-	}
-
-	// Print the message based on the level
-	switch level {
-	case SuccessLevel:
-		if ln {
-			pterm.Success.Println(levelMsg)
-		} else {
-			pterm.Success.Print(levelMsg)
-		}
-	case InfoLevel:
-		if ln {
-			pterm.Info.Println(levelMsg)
-		} else {
-			pterm.Info.Print(levelMsg)
-		}
-	case WarningLevel:
-		if ln {
-			pterm.Warning.Println(levelMsg)
-		} else {
-			pterm.Warning.Print(levelMsg)
-		}
-	case ErrorLevel:
-		if ln {
-			pterm.Error.Println(levelMsg)
-		} else {
-			pterm.Error.Print(levelMsg)
-		}
-	case FatalLevel:
-		if ln {
-			pterm.Fatal.Println(levelMsg)
-		} else {
-			pterm.Fatal.Print(levelMsg)
-		}
-	default:
-		if ln {
-			pterm.Debug.Println(levelMsg)
-		} else {
-			pterm.Debug.Print(levelMsg)
-		}
-	}
-}
-
-func (c *Context) Warning(mess ...interface{}) {
-	c.Msg("warning", true, mess...)
-	if c.Config.GetGeneral().FatalWarns {
-		os.Exit(2)
-	}
-}
-
-func (c *Context) Debug(mess ...interface{}) {
-	pc, file, line, ok := runtime.Caller(1)
-	if ok {
-		mess = append([]interface{}{fmt.Sprintf("(%s:#%d:%v)",
-			path.Base(file), line, runtime.FuncForPC(pc).Name())}, mess...)
-	}
-	c.Msg("debug", true, mess...)
-}
-
-func (c *Context) Info(mess ...interface{}) {
-	c.Msg("info", true, mess...)
-}
-
-func (c *Context) Success(mess ...interface{}) {
-	c.Msg("success", true, mess...)
-}
-
-func (c *Context) Error(mess ...interface{}) {
-	c.Msg("error", true, mess...)
-}
-
-func (c *Context) Fatal(mess ...interface{}) {
-	c.Error(mess...)
-	os.Exit(1)
-}
-
-type LogLevel string
-
-func (level LogLevel) ToNumber() int {
-	switch level {
-	case ErrorLevel, FatalLevel:
-		return 0
-	case WarningLevel:
-		return 1
-	case InfoLevel, SuccessLevel:
-		return 2
-	default: // debug
-		return 3
-	}
-}
-
-func (level LogLevel) ZapLevel() zap.AtomicLevel {
-	switch level {
-	case FatalLevel:
-		return zap.NewAtomicLevelAt(zap.FatalLevel)
-	case ErrorLevel:
-		return zap.NewAtomicLevelAt(zap.ErrorLevel)
-	case WarningLevel:
-		return zap.NewAtomicLevelAt(zap.WarnLevel)
-	case InfoLevel, SuccessLevel:
-		return zap.NewAtomicLevelAt(zap.InfoLevel)
-	default:
-		return zap.NewAtomicLevelAt(zap.DebugLevel)
-	}
+	WithLoggingContext(s string) Context
 }

pkg/api/core/types/garbagecollector.go

@@ -0,0 +1,25 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types
+
+import "os"
+
+type GarbageCollector interface {
+	Clean() error
+	TempDir(pattern string) (string, error)
+	TempFile(s string) (*os.File, error)
+	String() string
+}

pkg/api/core/types/logger.go

@@ -0,0 +1,41 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types
+
+// Logger is a standard logging interface
+type Logger interface {
+	Info(...interface{})
+	Success(...interface{})
+	Warning(...interface{})
+	Warn(...interface{})
+	Debug(...interface{})
+	Error(...interface{})
+	Fatal(...interface{})
+	Panic(...interface{})
+	Trace(...interface{})
+	Infof(string, ...interface{})
+	Warnf(string, ...interface{})
+	Debugf(string, ...interface{})
+	Errorf(string, ...interface{})
+	Fatalf(string, ...interface{})
+	Panicf(string, ...interface{})
+	Tracef(string, ...interface{})
+
+	SpinnerStop()
+	Spinner()
+	Ask() bool
+	Screen(string)
+}

pkg/compiler/backend.go

@@ -2,13 +2,13 @@ package compiler
 
 import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/mudler/luet/pkg/api/core/types"
 
+	"github.com/mudler/luet/pkg/api/core/types"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/pkg/errors"
 )
 
-func NewBackend(ctx *types.Context, s string) (CompilerBackend, error) {
+func NewBackend(ctx types.Context, s string) (CompilerBackend, error) {
 	var compilerBackend CompilerBackend
 
 	switch s {
@@ -26,6 +26,7 @@ func NewBackend(ctx *types.Context, s string) (CompilerBackend, error) {
 type CompilerBackend interface {
 	BuildImage(backend.Options) error
 	ExportImage(backend.Options) error
+	LoadImage(string) error
 	RemoveImage(backend.Options) error
 	ImageDefinitionToTar(backend.Options) error
 
@@ -35,6 +36,6 @@ type CompilerBackend interface {
 	Push(opts backend.Options) error
 	ImageAvailable(string) bool
 
-	ImageReference(img1 string) (v1.Image, error)
+	ImageReference(img1 string, ondisk bool) (v1.Image, error)
 	ImageExists(string) bool
 }

pkg/compiler/backend/common.go

@@ -18,9 +18,8 @@ package backend
 import (
 	"os/exec"
 
-	"github.com/mudler/luet/pkg/api/core/types"
-
 	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/mudler/luet/pkg/api/core/types"
 	"github.com/pkg/errors"
 )
 
@@ -43,9 +42,9 @@ type Options struct {
 	BackendArgs    []string
 }
 
-func runCommand(ctx *types.Context, cmd *exec.Cmd) error {
+func runCommand(ctx types.Context, cmd *exec.Cmd) error {
 	output := ""
-	buffered := !ctx.Config.GetGeneral().ShowBuildOutput
+	buffered := !ctx.GetConfig().General.ShowBuildOutput
 	writer := NewBackendWriter(buffered, ctx)
 
 	cmd.Stdout = writer

pkg/compiler/backend/simpledocker.go

@@ -1,4 +1,4 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@gentoo.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -16,10 +16,13 @@
 package backend
 
 import (
+	"io"
 	"os/exec"
 
+	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	bus "github.com/mudler/luet/pkg/api/core/bus"
 	"github.com/mudler/luet/pkg/api/core/types"
 
@@ -29,10 +32,10 @@ import (
 )
 
 type SimpleDocker struct {
-	ctx *types.Context
+	ctx types.Context
 }
 
-func NewSimpleDockerBackend(ctx *types.Context) *SimpleDocker {
+func NewSimpleDockerBackend(ctx types.Context) *SimpleDocker {
 	return &SimpleDocker{ctx: ctx}
 }
 
@@ -68,6 +71,17 @@ func (s *SimpleDocker) CopyImage(src, dst string) error {
 	return nil
 }
 
+func (s *SimpleDocker) LoadImage(path string) error {
+	s.ctx.Debug(":whale: Loading image:", path)
+	cmd := exec.Command("docker", "load", "-i", path)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed loading image: "+string(out))
+	}
+	s.ctx.Success(":whale: Loaded image:", path)
+	return nil
+}
+
 func (s *SimpleDocker) DownloadImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePull, opts)
@@ -138,19 +152,68 @@ func (s *SimpleDocker) Push(opts Options) error {
 	return nil
 }
 
-func (s *SimpleDocker) ImageReference(a string) (v1.Image, error) {
+func (s *SimpleDocker) imagefromDaemon(a string) (v1.Image, error) {
 	ref, err := name.ParseReference(a)
 	if err != nil {
 		return nil, err
 	}
-
-	img, err := daemon.Image(ref)
+	img, err := daemon.Image(ref, daemon.WithUnbufferedOpener())
 	if err != nil {
 		return nil, err
 	}
 	return img, nil
 }
 
+// TODO: Make it possible optionally to use this?
+// It might be unsafer, as it relies on the pipe.
+// imageFromCLIPipe returns a new image from a tarball by providing a reader from the docker stdout pipe.
+// See also daemon.Image implementation below for an example (which returns the tarball stream
+// from the HTTP api endpoint instead ).
+func (s *SimpleDocker) imageFromCLIPipe(a string) (v1.Image, error) {
+	return tarball.Image(func() (io.ReadCloser, error) {
+		buildarg := []string{"save", a}
+		s.ctx.Spinner()
+		defer s.ctx.SpinnerStop()
+		c := exec.Command("docker", buildarg...)
+		p, err := c.StdoutPipe()
+		if err != nil {
+			return nil, err
+		}
+		err = c.Start()
+		if err != nil {
+			return nil, err
+		}
+
+		go func() { c.Wait() }()
+		return p, nil
+	}, nil)
+}
+
+func (s *SimpleDocker) imageFromDisk(a string) (v1.Image, error) {
+	f, err := s.ctx.TempFile("snapshot")
+	if err != nil {
+		return nil, err
+	}
+	buildarg := []string{"save", a, "-o", f.Name()}
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
+
+	out, err := exec.Command("docker", buildarg...).CombinedOutput()
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed saving image: "+string(out))
+	}
+
+	return crane.Load(f.Name())
+}
+
+func (s *SimpleDocker) ImageReference(a string, ondisk bool) (v1.Image, error) {
+	if ondisk {
+		return s.imageFromDisk(a)
+	}
+
+	return s.imagefromDaemon(a)
+}
+
 func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")

pkg/compiler/backend/simpledocker_test.go

@@ -16,7 +16,7 @@
 package backend_test
 
 import (
-	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/context"
 	. "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
@@ -34,7 +34,7 @@ import (
 
 var _ = Describe("Docker backend", func() {
 	Context("Simple Docker backend satisfies main interface functionalities", func() {
-		ctx := types.NewContext()
+		ctx := context.NewContext()
 		It("Builds and generate tars", func() {
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 

pkg/compiler/backend/simpleimg.go

@@ -28,13 +28,17 @@ import (
 )
 
 type SimpleImg struct {
-	ctx *types.Context
+	ctx types.Context
 }
 
-func NewSimpleImgBackend(ctx *types.Context) *SimpleImg {
+func NewSimpleImgBackend(ctx types.Context) *SimpleImg {
 	return &SimpleImg{ctx: ctx}
 }
 
+func (s *SimpleImg) LoadImage(string) error {
+	return errors.New("Not supported")
+}
+
 // TODO: Missing still: labels, and build args expansion
 func (s *SimpleImg) BuildImage(opts Options) error {
 	name := opts.ImageName
@@ -71,9 +75,9 @@ func (s *SimpleImg) RemoveImage(opts Options) error {
 	return nil
 }
 
-func (s *SimpleImg) ImageReference(a string) (v1.Image, error) {
+func (s *SimpleImg) ImageReference(a string, ondisk bool) (v1.Image, error) {
 
-	f, err := s.ctx.Config.GetSystem().TempFile("snapshot")
+	f, err := s.ctx.TempFile("snapshot")
 	if err != nil {
 		return nil, err
 	}

pkg/compiler/backend/writer.go

@@ -25,10 +25,10 @@ import (
 type BackendWriter struct {
 	BufferedOutput bool
 	Buffer         *bytes.Buffer
-	ctx            *types.Context
+	ctx            types.Context
 }
 
-func NewBackendWriter(buffered bool, ctx *types.Context) *BackendWriter {
+func NewBackendWriter(buffered bool, ctx types.Context) *BackendWriter {
 	return &BackendWriter{
 		BufferedOutput: buffered,
 		Buffer:         &bytes.Buffer{},
@@ -41,7 +41,7 @@ func (b *BackendWriter) Write(p []byte) (int, error) {
 		return b.Buffer.Write(p)
 	}
 
-	b.ctx.Msg("info", false, (string(p)))
+	b.ctx.Info((string(p)))
 
 	return len(p), nil
 }

pkg/compiler/compiler.go

@@ -30,8 +30,8 @@ import (
 	"time"
 
 	bus "github.com/mudler/luet/pkg/api/core/bus"
+	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/api/core/image"
-	"github.com/mudler/luet/pkg/api/core/types"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/compiler/types/options"
@@ -82,7 +82,7 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, compilerOp
 
 	c := NewCompiler(compilerOpts...)
 	if c.Options.Context == nil {
-		c.Options.Context = types.NewContext()
+		c.Options.Context = context.NewContext()
 	}
 	//	c.Options.BackendType
 	c.Backend = backend
@@ -227,28 +227,23 @@ func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, includ
 }
 
 func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
-	img, err := cs.Backend.ImageReference(runnerOpts.ImageName)
+
+	if !cs.Backend.ImageExists(runnerOpts.ImageName) {
+		if err := cs.Backend.DownloadImage(runnerOpts); err != nil {
+			return nil, errors.Wrap(err, "failed pulling image "+runnerOpts.ImageName+" during extraction")
+		}
+	}
+
+	img, err := cs.Backend.ImageReference(runnerOpts.ImageName, true)
 	if err != nil {
 		return nil, err
 	}
 
-	// artifact.ImageToArtifact(
-	// 	cs.Options.Context,
-	// 	img,
-	// 	cs.Options.CompressionType,
-	// 	p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"),
-	// 	image.ExtractFiles(
-	// 		cs.Options.Context,
-	// 		strings.TrimLeft(p.GetPackageDir(), "/"),
-	// 		p.GetIncludes(),
-	// 		p.GetExcludes(),
-	// 	),
-	// )
-	// TODO: Trim includes/excludes from "/" ?
+	ctx := cs.Options.Context.WithLoggingContext(fmt.Sprintf("extract %s", runnerOpts.ImageName))
+
 	_, rootfs, err := image.Extract(
-		cs.Options.Context,
+		ctx,
 		img,
-		keepPermissions,
 		image.ExtractFiles(
 			cs.Options.Context,
 			p.GetPackageDir(),
@@ -280,45 +275,58 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
 
 func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, builderOpts, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
 
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	rootfs, err := cs.Options.Context.TempDir("rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
 	}
-	defer os.RemoveAll(rootfs) // clean up
+	defer os.RemoveAll(rootfs)
 
 	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
-	if cs.Options.PullFirst && !cs.Backend.ImageExists(builderOpts.ImageName) && cs.Backend.ImageAvailable(builderOpts.ImageName) {
-		err := cs.Backend.DownloadImage(builderOpts)
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not pull image")
+	if cs.Options.PullFirst {
+		if !cs.Backend.ImageExists(builderOpts.ImageName) {
+			err := cs.Backend.DownloadImage(builderOpts)
+			if err != nil {
+				return nil, errors.Wrap(err, "Could not pull image")
+			}
+		}
+		if !cs.Backend.ImageExists(runnerOpts.ImageName) {
+			err := cs.Backend.DownloadImage(runnerOpts)
+			if err != nil {
+				return nil, errors.Wrap(err, "Could not pull image")
+			}
 		}
 	}
 
 	cs.Options.Context.Info(pkgTag, ":hammer: Generating delta")
 
-	ref, err := cs.Backend.ImageReference(builderOpts.ImageName)
+	cs.Options.Context.Debug(pkgTag, ":hammer: Retrieving reference for", builderOpts.ImageName)
+
+	ref, err := cs.Backend.ImageReference(builderOpts.ImageName, true)
 	if err != nil {
 		return nil, err
 	}
 
-	ref2, err := cs.Backend.ImageReference(runnerOpts.ImageName)
+	cs.Options.Context.Debug(pkgTag, ":hammer: Retrieving reference for", runnerOpts.ImageName)
+
+	ref2, err := cs.Backend.ImageReference(runnerOpts.ImageName, true)
 	if err != nil {
 		return nil, err
 	}
 
-	diff, err := image.Delta(ref, ref2)
+	cs.Options.Context.Debug(pkgTag, ":hammer: Generating filters for extraction")
+
+	filter, err := image.ExtractDeltaAdditionsFiles(cs.Options.Context, ref, p.GetIncludes(), p.GetExcludes())
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed generating filter for extraction")
 	}
 
-	// TODO: includes/excludes might need to get "/" stripped from prefix
+	cs.Options.Context.Info(pkgTag, ":hammer: Extracting artifact from image", runnerOpts.ImageName)
 	a, err := artifact.ImageToArtifact(
 		cs.Options.Context,
 		ref2,
 		cs.Options.CompressionType,
 		p.Rel(fmt.Sprintf("%s%s", p.GetPackage().GetFingerPrint(), ".package.tar")),
-		keepPermissions,
-		image.ExtractDeltaFiles(cs.Options.Context, diff, p.GetIncludes(), p.GetExcludes()),
+		filter,
 	)
 	if err != nil {
 		return nil, err
@@ -344,7 +352,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
 	// and we build all the images first.
 
-	buildDir, err := cs.Options.Context.Config.System.TempDir("build")
+	buildDir, err := cs.Options.Context.TempDir("build")
 	if err != nil {
 		return builderOpts, runnerOpts, err
 	}
@@ -365,7 +373,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	}
 
 	// First we create the builder image
-	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")); err != nil {
+	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().ImageID()+"-builder.dockerfile")); err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
 	}
 
@@ -380,21 +388,21 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	// steps in prelude are == 0 those are equivalent.
 
 	// Then we write the step image, which uses the builder one
-	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")); err != nil {
+	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().ImageID()+".dockerfile")); err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
 	}
 
 	builderOpts = backend.Options{
 		ImageName:      buildertaggedImage,
 		SourcePath:     buildDir,
-		DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
+		DockerFileName: p.GetPackage().ImageID() + "-builder.dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 		BackendArgs:    cs.Options.BackendArgs,
 	}
 	runnerOpts = backend.Options{
 		ImageName:      packageImage,
 		SourcePath:     buildDir,
-		DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
+		DockerFileName: p.GetPackage().ImageID() + ".dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 		BackendArgs:    cs.Options.BackendArgs,
 	}
@@ -452,11 +460,11 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	if p.EmptyPackage() {
 		fakePackage := p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")
 
-		rootfs, err = ioutil.TempDir(p.GetOutputPath(), "rootfs")
+		rootfs, err = cs.Options.Context.TempDir("rootfs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not create tempdir")
 		}
-		defer os.RemoveAll(rootfs) // clean up
+		defer os.RemoveAll(rootfs)
 
 		a := artifact.NewPackageArtifact(fakePackage)
 		a.CompressionType = cs.Options.CompressionType
@@ -467,12 +475,16 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 
 		a.CompileSpec = p
 		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
-
 		err = a.WriteYAML(p.GetOutputPath())
 		if err != nil {
 			return a, errors.Wrap(err, "Failed while writing metadata file")
 		}
 		cs.Options.Context.Success(pkgTag, "   :white_check_mark: done (empty virtual package)")
+		if cs.Options.PushFinalImages {
+			if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil {
+				return nil, err
+			}
+		}
 		return a, nil
 	}
 
@@ -502,11 +514,55 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	if err != nil {
 		return a, errors.Wrap(err, "Failed while writing metadata file")
 	}
-	cs.Options.Context.Success(pkgTag, "   :white_check_mark: Done")
+	cs.Options.Context.Success(pkgTag, "   :white_check_mark: Done building")
+
+	if cs.Options.PushFinalImages {
+		if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil {
+			return nil, err
+		}
+	}
 
 	return a, nil
 }
 
+// TODO: A small readaptation of repository_docker.go pushImageFromArtifact()
+//       Move this to a common place
+func (cs *LuetCompiler) pushFinalArtifact(a *artifact.PackageArtifact, p *compilerspec.LuetCompilationSpec, keepPermissions bool) error {
+	cs.Options.Context.Info("Pushing final image for", a.CompileSpec.Package.HumanReadableString())
+	imageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, a.CompileSpec.Package.ImageID())
+
+	// First push the package image
+	if !cs.Backend.ImageAvailable(imageID) || cs.Options.PushFinalImagesForce {
+		cs.Options.Context.Info("Generating and pushing final image for", a.CompileSpec.Package.HumanReadableString(), "as", imageID)
+
+		if err := a.GenerateFinalImage(cs.Options.Context, imageID, cs.GetBackend(), true); err != nil {
+			return errors.Wrap(err, "while creating final image")
+		}
+		if err := cs.Backend.Push(backend.Options{ImageName: imageID}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", imageID)
+		}
+	}
+
+	// Then the image ID
+	metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, helpers.SanitizeImageString(a.CompileSpec.GetPackage().GetMetadataFilePath()))
+	if !cs.Backend.ImageAvailable(metadataImageID) || cs.Options.PushFinalImagesForce {
+		cs.Options.Context.Info("Generating metadata image for", a.CompileSpec.Package.HumanReadableString(), metadataImageID)
+
+		a := artifact.NewPackageArtifact(filepath.Join(p.GetOutputPath(), a.CompileSpec.GetPackage().GetMetadataFilePath()))
+		metadataArchive, err := artifact.CreateArtifactForFile(cs.Options.Context, a.Path)
+		if err != nil {
+			return errors.Wrap(err, "failed generating checksums for tree")
+		}
+		if err := metadataArchive.GenerateFinalImage(cs.Options.Context, metadataImageID, cs.Backend, keepPermissions); err != nil {
+			return errors.Wrap(err, "Failed generating metadata tree "+metadataImageID)
+		}
+		if err = cs.Backend.Push(backend.Options{ImageName: metadataImageID}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", metadataImageID)
+		}
+	}
+	return nil
+}
+
 func (cs *LuetCompiler) waitForImages(images []string) {
 	if cs.Options.PullFirst && cs.Options.Wait {
 		available, _ := oneOfImagesAvailable(images, cs.Backend)
@@ -885,11 +941,11 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 	}
 
 	// otherwise, generate it and push it aside
-	joinDir, err := ioutil.TempDir(p.GetOutputPath(), "join")
+	joinDir, err := cs.Options.Context.TempDir("join")
 	if err != nil {
 		return errors.Wrap(err, "could not create tempdir for joining images")
 	}
-	defer os.RemoveAll(joinDir) // clean up
+	defer os.RemoveAll(joinDir)
 
 	for _, p := range fromPackages {
 		cs.Options.Context.Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
@@ -924,11 +980,11 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 		}
 	}
 
-	artifactDir, err := ioutil.TempDir(p.GetOutputPath(), "artifact")
+	artifactDir, err := cs.Options.Context.TempDir("join")
 	if err != nil {
 		return errors.Wrap(err, "could not create tempdir for final artifact")
 	}
-	defer os.RemoveAll(joinDir) // clean up
+	defer os.RemoveAll(artifactDir)
 
 	cs.Options.Context.Info(joinTag, ":droplet: generating artifact for source image of", p.GetPackage().HumanReadableString())
 
@@ -941,14 +997,14 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 
 	joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp)
 	cs.Options.Context.Info(joinTag, ":droplet: generating image from artifact", joinImageName)
-	opts, err := a.GenerateFinalImage(cs.Options.Context, joinImageName, cs.Backend, keepPermissions)
+	err = a.GenerateFinalImage(cs.Options.Context, joinImageName, cs.Backend, keepPermissions)
 	if err != nil {
 		return errors.Wrap(err, "could not create final image")
 	}
 	if cs.Options.Push {
 		cs.Options.Context.Info(joinTag, ":droplet: pushing image from artifact", joinImageName)
-		if err = cs.Backend.Push(opts); err != nil {
-			return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
+		if err = cs.Backend.Push(backend.Options{ImageName: joinImageName}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", joinImageName)
 		}
 	}
 	cs.Options.Context.Info(joinTag, ":droplet: Consuming image", joinImageName)
@@ -999,6 +1055,36 @@ func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions
 	return nil
 }
 
+func CompilerFinalImages(cs *LuetCompiler) (*LuetCompiler, error) {
+	// When computing the hash tree, we need to take into consideration
+	// that packages that require final images have to be seen as packages without deps
+	// This is because we don't really want to calculate the deptree of them as
+	// as it is handled already when we are creating the images in resolveFinalImages().
+	c := *cs
+	copy := &c
+	memDB := pkg.NewInMemoryDatabase(false)
+	// Create a copy to avoid races
+	dbCopy := pkg.NewInMemoryDatabase(false)
+	err := cs.Database.Clone(dbCopy)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed cloning db")
+	}
+	for _, p := range dbCopy.World() {
+		copy := p.Clone()
+		spec, err := cs.FromPackage(p)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed getting compile spec for package "+p.HumanReadableString())
+		}
+		if spec.RequiresFinalImages {
+			copy.Requires([]*pkg.DefaultPackage{})
+		}
+
+		memDB.CreatePackage(copy)
+	}
+	copy.Database = memDB
+	return copy, nil
+}
+
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateFinalArtifact *bool, generateDependenciesFinalArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
 	cs.Options.Context.Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
@@ -1022,8 +1108,11 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 	}
 
 	ht := NewHashTree(cs.Database)
-
-	packageHashTree, err := ht.Query(cs, p)
+	copy, err := CompilerFinalImages(cs)
+	if err != nil {
+		return nil, err
+	}
+	packageHashTree, err := ht.Query(copy, p)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed querying hashtree")
 	}
@@ -1081,6 +1170,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 	buildTarget := !cs.Options.OnlyDeps
 
 	if buildDeps {
+
 		cs.Options.Context.Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
 		for _, assertion := range dependencies { //highly dependent on the order
 			depsN++
@@ -1096,6 +1186,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
 			}
 			compileSpec.BuildOptions.PullImageRepository = append(compileSpec.BuildOptions.PullImageRepository, p.BuildOptions.PullImageRepository...)
+
 			cs.Options.Context.Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
 
 			compileSpec.SetOutputPath(p.GetOutputPath())
@@ -1252,11 +1343,12 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
 	} else {
 		bv := cs.Options.BuildValuesFile
 		if len(vals) > 0 {
-			valuesdir, err := ioutil.TempDir("", "genvalues")
+			valuesdir, err := cs.Options.Context.TempDir("genvalues")
 			if err != nil {
 				return nil, errors.Wrap(err, "Could not create tempdir")
 			}
-			defer os.RemoveAll(valuesdir) // clean up
+			defer os.RemoveAll(valuesdir)
+
 			for _, b := range vals {
 				out, err := yaml.Marshal(b)
 				if err != nil {

pkg/compiler/compiler_test.go

@@ -16,11 +16,16 @@
 package compiler_test
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
-	"github.com/mudler/luet/pkg/api/core/types"
+	helpers "github.com/mudler/luet/tests/helpers"
+
+	"github.com/mudler/luet/pkg/api/core/context"
+	"github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	. "github.com/mudler/luet/pkg/compiler"
 	sd "github.com/mudler/luet/pkg/compiler/backend"
@@ -35,7 +40,7 @@ import (
 )
 
 var _ = Describe("Compiler", func() {
-	ctx := types.NewContext()
+	ctx := context.NewContext()
 
 	Context("Simple package build definition", func() {
 		It("Compiles it correctly", func() {
@@ -46,7 +51,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(context.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -89,7 +94,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(context.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
 			Expect(err).ToNot(HaveOccurred())
@@ -119,7 +124,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(context.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
 			Expect(err).ToNot(HaveOccurred())
@@ -151,7 +156,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(1), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(1), options.WithContext(context.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -185,7 +190,7 @@ var _ = Describe("Compiler", func() {
 
 			err = generalRecipe.Load("../../tests/fixtures/templates")
 			Expect(err).ToNot(HaveOccurred())
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(context.NewContext()))
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 			pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -209,7 +214,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(context.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -265,7 +270,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(1), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(1), options.WithContext(context.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -789,7 +794,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2), options.WithContext(context.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -853,6 +858,67 @@ var _ = Describe("Compiler", func() {
 			Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
+
+		It("Pushes final images along", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			randString := strings.ToLower(helpers.String(10))
+			imageName := fmt.Sprintf("ttl.sh/%s", randString)
+			b := sd.NewSimpleDockerBackend(ctx)
+
+			err := generalRecipe.Load("../../tests/fixtures/packagelayers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(b, generalRecipe.GetDatabase(),
+				options.EnablePushFinalImages, options.ForcePushFinalImages, options.WithFinalRepository(imageName))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "build", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(2))
+			//Expect(len(artifacts[0].Dependencies)).To(Equal(1))
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.ImageID()))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.GetMetadataFilePath()))).To(BeTrue())
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.ImageID()))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.GetMetadataFilePath()))).To(BeTrue())
+
+			img, err := b.ImageReference(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.ImageID()), true)
+			Expect(err).ToNot(HaveOccurred())
+			_, path, err := image.Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(path) // clean up
+
+			Expect(fileHelper.Exists(filepath.Join(path, "bin/busybox"))).To(BeTrue())
+
+			img, err = b.ImageReference(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.GetMetadataFilePath()), true)
+			Expect(err).ToNot(HaveOccurred())
+			_, path, err = image.Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(path) // clean up
+
+			meta := filepath.Join(path, artifacts[1].Runtime.GetMetadataFilePath())
+			Expect(fileHelper.Exists(meta)).To(BeTrue())
+
+			d, err := ioutil.ReadFile(meta)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(d)).To(ContainSubstring(artifacts[1].CompileSpec.GetPackage().GetName()))
+		})
 	})
 
 	Context("Packages which conents are a package folder", func() {
@@ -917,7 +983,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(context.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
@@ -952,7 +1018,7 @@ var _ = Describe("Compiler", func() {
 			err := generalRecipe.Load("../../tests/fixtures/includeimage")
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(context.NewContext()))
 
 			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
 			Expect(err).ToNot(HaveOccurred())
@@ -971,7 +1037,7 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(context.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())

pkg/compiler/imagehashtree_test.go

@@ -16,7 +16,7 @@
 package compiler_test
 
 import (
-	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/context"
 	. "github.com/mudler/luet/pkg/compiler"
 	sd "github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/compiler/types/options"
@@ -27,7 +27,7 @@ import (
 )
 
 var _ = Describe("ImageHashTree", func() {
-	ctx := types.NewContext()
+	ctx := context.NewContext()
 	generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 	compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
 	hashtree := NewHashTree(generalRecipe.GetDatabase())

pkg/compiler/types/options/compiler_options.go

@@ -47,7 +47,13 @@ type Compiler struct {
 	// TemplatesFolder. should default to tree/templates
 	TemplatesFolder []string
 
-	Context *types.Context
+	// Tells wether to push final container images after building
+	PushFinalImages      bool
+	PushFinalImagesForce bool
+	// Image repository to push to
+	PushFinalImagesRepository string
+
+	Context types.Context
 }
 
 func NewDefaultCompiler() *Compiler {
@@ -85,6 +91,25 @@ func WithOptions(opt *Compiler) func(cfg *Compiler) error {
 	}
 }
 
+// WithFinalRepository Sets the final repository where to push
+// images of built artifacts
+func WithFinalRepository(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PushFinalImagesRepository = r
+		return nil
+	}
+}
+
+func EnablePushFinalImages(cfg *Compiler) error {
+	cfg.PushFinalImages = true
+	return nil
+}
+
+func ForcePushFinalImages(cfg *Compiler) error {
+	cfg.PushFinalImagesForce = true
+	return nil
+}
+
 func WithBackendType(r string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		cfg.BackendType = r
@@ -113,6 +138,8 @@ func WithPullRepositories(r []string) func(cfg *Compiler) error {
 	}
 }
 
+// WithPushRepository Sets the image reference where to push
+// cache images
 func WithPushRepository(r string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		if len(cfg.PullImageRepository) == 0 {
@@ -210,7 +237,7 @@ func WithSolverOptions(c types.LuetSolverOptions) func(cfg *Compiler) error {
 	}
 }
 
-func WithContext(c *types.Context) func(cfg *Compiler) error {
+func WithContext(c types.Context) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		cfg.Context = c
 		return nil

pkg/helpers/docker/docker.go

@@ -18,7 +18,6 @@ package docker
 import (
 	"context"
 	"encoding/hex"
-	"fmt"
 	"os"
 
 	"github.com/containerd/containerd/images"
@@ -30,11 +29,9 @@ import (
 	"github.com/docker/cli/cli/trust"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/registry"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
-	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/mudler/luet/pkg/api/core/bus"
 	"github.com/opencontainers/go-digest"
@@ -129,44 +126,8 @@ type UnpackEventData struct {
 	Dest  string
 }
 
-// UnarchiveLayers extract layers with archive.Untar from docker instead of containerd
-func UnarchiveLayers(temp string, img v1.Image, image, dest string, auth *types.AuthConfig, verify bool) (int64, error) {
-	layers, err := img.Layers()
-	if err != nil {
-		return 0, fmt.Errorf("reading layers from '%s' image failed: %v", image, err)
-	}
-	bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
-
-	var size int64
-	for _, l := range layers {
-		s, err := l.Size()
-		if err != nil {
-			return 0, fmt.Errorf("reading layer size from '%s' image failed: %v", image, err)
-		}
-		size += s
-
-		layerReader, err := l.Uncompressed()
-		if err != nil {
-			return 0, fmt.Errorf("reading uncompressed layer from '%s' image failed: %v", image, err)
-		}
-		defer layerReader.Close()
-
-		// Unpack the tarfile to the rootfs path.
-		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
-		if err := archive.Untar(layerReader, dest, &archive.TarOptions{
-			NoLchown:        false,
-			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
-		}); err != nil {
-			return 0, fmt.Errorf("extracting '%s' image to directory %s failed: %v", image, dest, err)
-		}
-	}
-	bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
-
-	return size, nil
-}
-
 // DownloadAndExtractDockerImage extracts a container image natively. It supports privileged/unprivileged mode
-func DownloadAndExtractDockerImage(ctx *luettypes.Context, image, dest string, auth *types.AuthConfig, verify bool) (*images.Image, error) {
+func DownloadAndExtractDockerImage(ctx luettypes.Context, image, dest string, auth *types.AuthConfig, verify bool) (*images.Image, error) {
 	if verify {
 		img, err := verifyImage(image, auth)
 		if err != nil {
@@ -213,7 +174,6 @@ func DownloadAndExtractDockerImage(ctx *luettypes.Context, image, dest string, a
 		ctx,
 		img,
 		dest,
-		true,
 		nil,
 	)
 	if err != nil {

pkg/helpers/terminal/terminal_check_bsd.go

@@ -1,12 +0,0 @@
-// +build darwin dragonfly freebsd netbsd openbsd
-
-package terminal
-
-import "golang.org/x/sys/unix"
-
-const ioctlReadTermios = unix.TIOCGETA
-
-func isTerminal(fd int) bool {
-	_, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
-	return err == nil
-}

pkg/helpers/terminal/terminal_check_general.go

@@ -1,17 +0,0 @@
-// +build !windows,!nacl,!plan9
-
-package terminal
-
-import (
-	"io"
-	"os"
-)
-
-func IsTerminal(w io.Writer) bool {
-	switch v := w.(type) {
-	case *os.File:
-		return isTerminal(int(v.Fd()))
-	default:
-		return false
-	}
-}

pkg/helpers/terminal/terminal_check_no_terminal.go

@@ -1,11 +0,0 @@
-// +build js nacl plan9
-
-package terminal
-
-import (
-	"io"
-)
-
-func IsTerminal(w io.Writer) bool {
-	return false
-}

pkg/helpers/terminal/terminal_check_solaris.go

@@ -1,11 +0,0 @@
-package terminal
-
-import (
-	"golang.org/x/sys/unix"
-)
-
-// IsTerminal returns true if the given file descriptor is a terminal.
-func isTerminal(fd int) bool {
-	_, err := unix.IoctlGetTermio(fd, unix.TCGETA)
-	return err == nil
-}

pkg/helpers/terminal/terminal_check_unix.go

@@ -1,12 +0,0 @@
-// +build linux aix zos
-
-package terminal
-
-import "golang.org/x/sys/unix"
-
-const ioctlReadTermios = unix.TCGETS
-
-func isTerminal(fd int) bool {
-	_, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
-	return err == nil
-}

pkg/helpers/terminal/terminal_check_windows.go

@@ -1,27 +0,0 @@
-// +build windows
-
-package terminal
-
-import (
-	"io"
-	"os"
-
-	"golang.org/x/sys/windows"
-)
-
-func IsTerminal(w io.Writer) bool {
-	switch v := w.(type) {
-	case *os.File:
-		handle := windows.Handle(v.Fd())
-		var mode uint32
-		if err := windows.GetConsoleMode(handle, &mode); err != nil {
-			return false
-		}
-		mode |= windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING
-		if err := windows.SetConsoleMode(handle, mode); err != nil {
-			return false
-		}
-		return true
-	}
-	return false
-}

pkg/installer/cli.go

@@ -88,7 +88,7 @@ func printMatches(artefacts map[string]ArtifactMatch) {
 	for _, m := range artefacts {
 		d = append(d, []string{
 			fmt.Sprintf("%s/%s", m.Package.GetCategory(), m.Package.GetName()),
-			pterm.LightGreen(m.Package.GetVersion()), m.Package.GetLicense(), m.Repository.Name})
+			pterm.LightGreen(m.Package.GetVersion()), m.Package.GetLicense(), m.Repository.GetName()})
 	}
 	pterm.DefaultTable.WithHasHeader().WithData(d).Render()
 	fmt.Println()

pkg/installer/client/docker.go

@@ -24,9 +24,9 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/go-units"
+	luettypes "github.com/mudler/luet/pkg/api/core/types"
 	"github.com/pkg/errors"
 
-	luetTypes "github.com/mudler/luet/pkg/api/core/types"
 	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/helpers"
 
@@ -42,17 +42,17 @@ type DockerClient struct {
 	RepoData RepoData
 	auth     *types.AuthConfig
 	Cache    *artifact.ArtifactCache
-	context  *luetTypes.Context
+	context  luettypes.Context
 }
 
-func NewDockerClient(r RepoData, ctx *luetTypes.Context) *DockerClient {
+func NewDockerClient(r RepoData, ctx luettypes.Context) *DockerClient {
 	auth := &types.AuthConfig{}
 
 	dat, _ := json.Marshal(r.Authentication)
 	json.Unmarshal(dat, auth)
 
 	return &DockerClient{RepoData: r, auth: auth,
-		Cache:   artifact.NewCache(ctx.Config.GetSystem().GetSystemPkgsCacheDirPath()),
+		Cache:   artifact.NewCache(ctx.GetConfig().System.PkgsCachePath),
 		context: ctx,
 	}
 }
@@ -64,11 +64,13 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
 	c.context.Spinner()
 	defer c.context.SpinnerStop()
 
-	resultingArtifact := a.ShallowCopy()
-	artifactName := path.Base(a.Path)
-
 	downloaded := false
 
+	resultingArtifact, err := c.CacheGet(a)
+	if err == nil {
+		return resultingArtifact, nil
+	}
+
 	// TODO:
 	// Files are in URI/packagename:version (GetPackageImageName() method)
 	// use downloadAndExtract .. and egenrate an archive to consume. Checksum should be already checked while downloading the image
@@ -79,71 +81,57 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
 	// We discard checksum, that are checked while during pull and unpack by containerd
 	resultingArtifact.Checksums = artifact.Checksums{}
 
-	// Check if file is already in cache
-	fileName, err := c.Cache.Get(resultingArtifact)
-	// Check if file is already in cache
-	if err == nil {
-		resultingArtifact = a
-		resultingArtifact.Path = fileName
-		resultingArtifact.Checksums = artifact.Checksums{}
-		c.context.Debug("Use artifact", artifactName, "from cache.")
-	} else {
+	temp, err := c.context.TempDir("image")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(temp)
 
-		temp, err := c.context.Config.GetSystem().TempDir("image")
+	tempArtifact, err := c.context.TempFile("artifact")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tempArtifact.Name())
+	for _, uri := range c.RepoData.Urls {
+
+		imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID())
+		c.context.Info("Downloading image", imageName)
+
+		// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
+		info, err := docker.DownloadAndExtractDockerImage(c.context, imageName, temp, c.auth, c.RepoData.Verify)
 		if err != nil {
-			return nil, err
+			c.context.Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
+			continue
 		}
-		defer os.RemoveAll(temp)
 
-		tempArtifact, err := c.context.Config.GetSystem().TempFile("artifact")
+		c.context.Info(
+			fmt.Sprintf("Image: %s. Pulled: %s. Size: %s",
+				imageName,
+				info.Target.Digest,
+				units.BytesSize(float64(info.Target.Size)),
+			),
+		)
+		c.context.Debug("\nCompressing result ", filepath.Join(temp), "to", tempArtifact.Name())
+
+		resultingArtifact.Path = tempArtifact.Name() // First set to cache file
+		err = resultingArtifact.Compress(temp, 1)
 		if err != nil {
-			return nil, err
-		}
-		defer os.RemoveAll(tempArtifact.Name())
-		for _, uri := range c.RepoData.Urls {
-
-			imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID())
-			c.context.Info("Downloading image", imageName)
-
-			// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
-			info, err := docker.DownloadAndExtractDockerImage(c.context, imageName, temp, c.auth, c.RepoData.Verify)
-			if err != nil {
-				c.context.Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
-				continue
-			}
-
-			c.context.Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
-			c.context.Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size))))
-			c.context.Debug("\nCompressing result ", filepath.Join(temp), "to", tempArtifact.Name())
-
-			resultingArtifact.Path = tempArtifact.Name() // First set to cache file
-			err = resultingArtifact.Compress(temp, 1)
-			if err != nil {
-				c.context.Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))
-				continue
-			}
-
-			_, _, err = c.Cache.Put(resultingArtifact)
-			if err != nil {
-				c.context.Error(fmt.Sprintf("Failed storing package %s from cache: %s", imageName, err.Error()))
-				continue
-			}
-
-			fileName, err := c.Cache.Get(resultingArtifact)
-			if err != nil {
-				c.context.Error(fmt.Sprintf("Failed getting package %s from cache: %s", imageName, err.Error()))
-				continue
-			}
-
-			resultingArtifact.Path = fileName // Cache is persistent. tempArtifact is not
-
-			downloaded = true
-			break
+			c.context.Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))
+			continue
 		}
 
-		if !downloaded {
-			return nil, errors.Wrap(err, "no image available from repositories")
+		_, _, err = c.Cache.Put(resultingArtifact)
+		if err != nil {
+			c.context.Error(fmt.Sprintf("Failed storing package %s from cache: %s", imageName, err.Error()))
+			continue
 		}
+		downloaded = true
+
+		return c.CacheGet(resultingArtifact)
+	}
+
+	if !downloaded {
+		return nil, errors.Wrap(err, "no image available from repositories")
 	}
 
 	return resultingArtifact, nil
@@ -156,13 +144,13 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 	// Files should be in URI/repository:<file>
 	ok := false
 
-	temp, err = c.context.Config.GetSystem().TempDir("tree")
+	temp, err = c.context.TempDir("tree")
 	if err != nil {
 		return "", err
 	}
 
 	for _, uri := range c.RepoData.Urls {
-		file, err = c.context.Config.GetSystem().TempFile("DockerClient")
+		file, err = c.context.TempFile("DockerClient")
 		if err != nil {
 			continue
 		}
@@ -194,3 +182,29 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 
 	return file.Name(), err
 }
+
+func (c *DockerClient) CacheGet(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
+	resultingArtifact := a.ShallowCopy()
+	// TODO:
+	// Files are in URI/packagename:version (GetPackageImageName() method)
+	// use downloadAndExtract .. and egenrate an archive to consume. Checksum should be already checked while downloading the image
+	// with the above functions, because Docker images already contain such metadata
+	// - Check how verification is done when calling DownloadArtifact outside, similarly we need to check DownloadFile, and how verification
+	// is done in such cases (see repository.go)
+
+	// We discard checksum, that are checked while during pull and unpack by containerd
+	resultingArtifact.Checksums = artifact.Checksums{}
+
+	// Check if file is already in cache
+	fileName, err := c.Cache.Get(resultingArtifact)
+	// Check if file is already in cache
+	if err == nil {
+		artifactName := path.Base(a.Path)
+		c.context.Debug("Use artifact", artifactName, "from cache.")
+		resultingArtifact = a
+		resultingArtifact.Path = fileName
+		resultingArtifact.Checksums = artifact.Checksums{}
+	}
+
+	return resultingArtifact, err
+}

pkg/installer/client/docker_test.go

@@ -20,7 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
@@ -38,7 +38,7 @@ import (
 // mount/unmount layers.
 var _ = Describe("Docker client", func() {
 	Context("With repository", func() {
-		ctx := types.NewContext()
+		ctx := context.NewContext()
 
 		repoImage := os.Getenv("UNIT_TEST_DOCKER_IMAGE")
 		var repoURL []string

pkg/installer/client/http.go

@@ -36,13 +36,13 @@ import (
 type HttpClient struct {
 	RepoData RepoData
 	Cache    *artifact.ArtifactCache
-	context  *types.Context
+	context  types.Context
 }
 
-func NewHttpClient(r RepoData, ctx *types.Context) *HttpClient {
+func NewHttpClient(r RepoData, ctx types.Context) *HttpClient {
 	return &HttpClient{
 		RepoData: r,
-		Cache:    artifact.NewCache(ctx.Config.GetSystem().GetSystemPkgsCacheDirPath()),
+		Cache:    artifact.NewCache(ctx.GetConfig().System.PkgsCachePath),
 		context:  ctx,
 	}
 }
@@ -85,16 +85,16 @@ func Round(input float64) float64 {
 func (c *HttpClient) DownloadFile(p string) (string, error) {
 	var file *os.File = nil
 	var downloaded bool
-	temp, err := c.context.Config.GetSystem().TempDir("download")
+	temp, err := c.context.TempDir("download")
 	if err != nil {
 		return "", err
 	}
 	defer os.RemoveAll(temp)
 
-	client := NewGrabClient(c.context.Config.General.HTTPTimeout)
+	client := NewGrabClient(c.context.GetConfig().General.HTTPTimeout)
 
 	for _, uri := range c.RepoData.Urls {
-		file, err = c.context.Config.GetSystem().TempFile("HttpClient")
+		file, err = c.context.TempFile("HttpClient")
 		if err != nil {
 			c.context.Debug("Failed downloading", p, "from", uri)
 
@@ -117,13 +117,12 @@ func (c *HttpClient) DownloadFile(p string) (string, error) {
 
 		// Initialize a progressbar only if we have one in the current context
 		var pb *pterm.ProgressbarPrinter
-		if c.context.ProgressBar != nil {
-			pb = pterm.DefaultProgressbar.WithTotal(int(resp.Size()))
-			if c.context.AreaPrinter != nil {
-				pb = pb.WithPrintTogether(c.context.AreaPrinter)
-			}
-			pb, _ = pb.WithTitle(filepath.Base(resp.Request.HTTPRequest.URL.RequestURI())).Start()
+		pbb := c.context.GetAnnotation("progressbar")
+		switch v := pbb.(type) {
+		case *pterm.ProgressbarPrinter:
+			pb, _ = v.WithTotal(int(resp.Size())).WithTitle(filepath.Base(resp.Request.HTTPRequest.URL.RequestURI())).Start()
 		}
+
 		// start download loop
 		t := time.NewTicker(500 * time.Millisecond)
 		defer t.Stop()
@@ -170,32 +169,33 @@ func (c *HttpClient) DownloadFile(p string) (string, error) {
 	return file.Name(), nil
 }
 
-func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
+func (c *HttpClient) CacheGet(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
 	newart := a.ShallowCopy()
-	artifactName := path.Base(a.Path)
 
 	fileName, err := c.Cache.Get(a)
+
+	newart.Path = fileName
+
+	return newart, err
+}
+
+func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
+	artifactName := path.Base(a.Path)
+
+	newart, err := c.CacheGet(a)
 	// Check if file is already in cache
 	if err == nil {
-		newart.Path = fileName
-		c.context.Debug("Use artifact", artifactName, "from cache.")
-	} else {
-		d, err := c.DownloadFile(artifactName)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed downloading %s", artifactName)
-		}
-
-		defer os.RemoveAll(d)
-		newart.Path = d
-		c.Cache.Put(newart)
-
-		fileName, err := c.Cache.Get(newart)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed getting file from cache %v", newart)
-		}
-
-		newart.Path = fileName
+		return newart, nil
 	}
 
-	return newart, nil
+	d, err := c.DownloadFile(artifactName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed downloading %s", artifactName)
+	}
+
+	defer os.RemoveAll(d)
+	newart.Path = d
+	c.Cache.Put(newart)
+
+	return c.CacheGet(newart)
 }

pkg/installer/client/http_test.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer/client"
@@ -32,7 +32,7 @@ import (
 
 var _ = Describe("Http client", func() {
 	Context("With repository", func() {
-		ctx := types.NewContext()
+		ctx := context.NewContext()
 
 		It("Downloads single files", func() {
 			// setup small staticfile webserver with content

pkg/installer/client/local.go

@@ -29,12 +29,12 @@ import (
 type LocalClient struct {
 	RepoData RepoData
 	Cache    *artifact.ArtifactCache
-	context  *types.Context
+	context  types.Context
 }
 
-func NewLocalClient(r RepoData, ctx *types.Context) *LocalClient {
+func NewLocalClient(r RepoData, ctx types.Context) *LocalClient {
 	return &LocalClient{
-		Cache:    artifact.NewCache(ctx.Config.GetSystem().GetSystemPkgsCacheDirPath()),
+		Cache:    artifact.NewCache(ctx.GetConfig().System.PkgsCachePath),
 		RepoData: r,
 		context:  ctx,
 	}
@@ -44,32 +44,32 @@ func (c *LocalClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.P
 	var err error
 
 	artifactName := path.Base(a.Path)
-	newart := a.ShallowCopy()
 
-	fileName, err := c.Cache.Get(a)
+	newart, err := c.CacheGet(a)
 	// Check if file is already in cache
 	if err == nil {
-		newart.Path = fileName
-		c.context.Debug("Use artifact", artifactName, "from cache.")
-	} else {
-		d, err := c.DownloadFile(artifactName)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed downloading %s", artifactName)
-		}
-		defer os.RemoveAll(d)
-
-		newart.Path = d
-		c.Cache.Put(newart)
-
-		fileName, err := c.Cache.Get(newart)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed getting file from cache %v", newart)
-		}
-
-		newart.Path = fileName
+		return newart, nil
 	}
 
-	return newart, nil
+	d, err := c.DownloadFile(artifactName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed downloading %s", artifactName)
+	}
+	defer os.RemoveAll(d)
+
+	newart.Path = d
+	c.Cache.Put(newart)
+
+	return c.CacheGet(newart)
+}
+
+func (c *LocalClient) CacheGet(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) {
+	newart := a.ShallowCopy()
+	fileName, err := c.Cache.Get(a)
+
+	newart.Path = fileName
+
+	return newart, err
 }
 
 func (c *LocalClient) DownloadFile(name string) (string, error) {
@@ -78,11 +78,8 @@ func (c *LocalClient) DownloadFile(name string) (string, error) {
 
 	rootfs := ""
 
-	if !c.context.Config.ConfigFromHost {
-		rootfs, err = c.context.Config.GetSystem().GetRootFsAbs()
-		if err != nil {
-			return "", err
-		}
+	if !c.context.GetConfig().ConfigFromHost {
+		rootfs = c.context.GetConfig().System.Rootfs
 	}
 
 	ok := false
@@ -91,7 +88,7 @@ func (c *LocalClient) DownloadFile(name string) (string, error) {
 		uri = filepath.Join(rootfs, uri)
 
 		c.context.Info("Copying file", name, "from", uri)
-		file, err = c.context.Config.GetSystem().TempFile("localclient")
+		file, err = c.context.TempFile("localclient")
 		if err != nil {
 			continue
 		}

pkg/installer/client/local_test.go

@@ -20,7 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/installer/client"
@@ -30,7 +30,7 @@ import (
 
 var _ = Describe("Local client", func() {
 	Context("With repository", func() {
-		ctx := types.NewContext()
+		ctx := context.NewContext()
 
 		It("Downloads single files", func() {
 			tmpdir, err := ioutil.TempDir("", "test")

pkg/installer/finalizer.go

@@ -32,7 +32,7 @@ type LuetFinalizer struct {
 	Uninstall []string `json:"uninstall"` // TODO: Where to store?
 }
 
-func (f *LuetFinalizer) RunInstall(ctx *types.Context, s *System) error {
+func (f *LuetFinalizer) RunInstall(ctx types.Context, s *System) error {
 	var cmd string
 	var args []string
 	if len(f.Shell) == 0 {
@@ -51,14 +51,14 @@ func (f *LuetFinalizer) RunInstall(ctx *types.Context, s *System) error {
 		ctx.Info(":shell: Executing finalizer on ", s.Target, cmd, toRun)
 		if s.Target == string(os.PathSeparator) {
 			cmd := exec.Command(cmd, toRun...)
-			cmd.Env = ctx.Config.GetFinalizerEnvs()
+			cmd.Env = ctx.GetConfig().FinalizerEnvs.Slice()
 			stdoutStderr, err := cmd.CombinedOutput()
 			if err != nil {
 				return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
 			}
 			ctx.Info(string(stdoutStderr))
 		} else {
-			b := box.NewBox(cmd, toRun, []string{}, ctx.Config.GetFinalizerEnvs(), s.Target, false, true, true)
+			b := box.NewBox(cmd, toRun, []string{}, ctx.GetConfig().FinalizerEnvs.Slice(), s.Target, false, true, true)
 			err := b.Run()
 			if err != nil {
 				return errors.Wrap(err, "Failed running command ")
@@ -69,7 +69,7 @@ func (f *LuetFinalizer) RunInstall(ctx *types.Context, s *System) error {
 }
 
 // TODO: We don't store uninstall finalizers ?!
-func (f *LuetFinalizer) RunUnInstall(ctx *types.Context) error {
+func (f *LuetFinalizer) RunUnInstall(ctx types.Context) error {
 	for _, c := range f.Uninstall {
 		ctx.Debug("finalizer:", "sh", "-c", c)
 		cmd := exec.Command("sh", "-c", c)

pkg/installer/installer.go

@@ -25,11 +25,11 @@ import (
 	"sync"
 
 	"github.com/mudler/luet/pkg/api/core/config"
+	"github.com/mudler/luet/pkg/api/core/logger"
 
 	"github.com/mudler/luet/pkg/api/core/bus"
 	"github.com/mudler/luet/pkg/api/core/types"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
-	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	"github.com/mudler/luet/pkg/helpers/match"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -53,8 +53,9 @@ type LuetInstallerOptions struct {
 	DownloadOnly                                                   bool
 	Relaxed                                                        bool
 	PackageRepositories                                            types.LuetRepositories
+	AutoOSCheck                                                    bool
 
-	Context *types.Context
+	Context types.Context
 }
 
 type LuetInstaller struct {
@@ -64,7 +65,7 @@ type LuetInstaller struct {
 type ArtifactMatch struct {
 	Package    pkg.Package
 	Artifact   *artifact.PackageArtifact
-	Repository *LuetSystemRepository
+	Repository Repository
 }
 
 func NewLuetInstaller(opts LuetInstallerOptions) *LuetInstaller {
@@ -267,13 +268,16 @@ func (l *LuetInstaller) swap(o Option, syncedRepos Repositories, toRemove pkg.Pa
 		return nil
 	}
 
-	ops := l.getOpsWithOptions(toRemove, match, Option{
+	ops, err := l.generateRunOps(toRemove, match, Option{
 		Force:              o.Force,
 		NoDeps:             false,
 		OnlyDeps:           o.OnlyDeps,
 		RunFinalizers:      false,
 		CheckFileConflicts: false,
-	}, o, syncedRepos, packages, assertions, allRepos)
+	}, o, syncedRepos, packages, assertions, allRepos, s)
+	if err != nil {
+		return errors.Wrap(err, "failed computing installer options")
+	}
 
 	err = l.runOps(ops, s)
 	if err != nil {
@@ -317,8 +321,8 @@ type installOperation struct {
 // installerOp is the operation that is sent to the
 // upgradeWorker's channel (todo)
 type installerOp struct {
-	Uninstall operation
-	Install   installOperation
+	Uninstall []operation
+	Install   []installOperation
 }
 
 func (l *LuetInstaller) runOps(ops []installerOp, s *System) error {
@@ -349,13 +353,32 @@ func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, systemLock
 
 	for p := range c {
 
-		if p.Uninstall.Package != nil {
-			l.Options.Context.Debug("Replacing package inplace")
-			toUninstall, uninstall, err := l.generateUninstallFn(p.Uninstall.Option, s, p.Uninstall.Package)
+		installedFiles := map[string]interface{}{}
+		for _, pp := range p.Install {
+			artMatch := pp.Matches[pp.Package.GetFingerPrint()]
+			art, err := l.getPackage(artMatch, l.Options.Context)
 			if err != nil {
-				l.Options.Context.Error("Failed to generate Uninstall function for" + err.Error())
+				installedFiles = map[string]interface{}{}
+				break
+			}
+
+			l, err := art.FileList()
+			if err != nil {
+				installedFiles = map[string]interface{}{}
+				break
+			}
+			for _, f := range l {
+				installedFiles[f] = nil
+			}
+		}
+
+		for _, pp := range p.Uninstall {
+
+			l.Options.Context.Debug("Replacing package inplace")
+			toUninstall, uninstall, err := l.generateUninstallFn(pp.Option, s, installedFiles, pp.Package)
+			if err != nil {
+				l.Options.Context.Debug("Skipping uninstall, fail to generate uninstall function, error: " + err.Error())
 				continue
-				//return errors.Wrap(err, "while computing uninstall")
 			}
 			systemLock.Lock()
 			err = uninstall()
@@ -364,22 +387,21 @@ func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, systemLock
 			if err != nil {
 				l.Options.Context.Error("Failed uninstall for ", packsToList(toUninstall))
 				continue
-				//return errors.Wrap(err, "uninstalling "+packsToList(toUninstall))
 			}
 		}
-		if p.Install.Package != nil {
-			artMatch := p.Install.Matches[p.Install.Package.GetFingerPrint()]
-			ass := p.Install.Assertions.Search(p.Install.Package.GetFingerPrint())
-			packageToInstall, _ := p.Install.Packages.Find(p.Install.Package.GetPackageName())
+		for _, pp := range p.Install {
+			artMatch := pp.Matches[pp.Package.GetFingerPrint()]
+			ass := pp.Assertions.Search(pp.Package.GetFingerPrint())
+			packageToInstall, _ := pp.Packages.Find(pp.Package.GetPackageName())
 
 			systemLock.Lock()
 			err := l.install(
-				p.Install.Option,
-				p.Install.Reposiories,
-				map[string]ArtifactMatch{p.Install.Package.GetFingerPrint(): artMatch},
+				pp.Option,
+				pp.Reposiories,
+				map[string]ArtifactMatch{pp.Package.GetFingerPrint(): artMatch},
 				pkg.Packages{packageToInstall},
 				solver.PackagesAssertions{*ass},
-				p.Install.Database,
+				pp.Database,
 				s,
 			)
 			systemLock.Unlock()
@@ -393,65 +415,41 @@ func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, systemLock
 }
 
 // checks wheter we can uninstall and install in place and compose installer worker ops
-func (l *LuetInstaller) getOpsWithOptions(
+func (l *LuetInstaller) generateRunOps(
 	toUninstall pkg.Packages, installMatch map[string]ArtifactMatch, installOpt, uninstallOpt Option,
-	syncedRepos Repositories, toInstall pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase) []installerOp {
-	resOps := []installerOp{}
-	for _, match := range installMatch {
-		if pack, err := toUninstall.Find(match.Package.GetPackageName()); err == nil {
-			resOps = append(resOps, installerOp{
-				Uninstall: operation{Package: pack, Option: uninstallOpt},
-				Install: installOperation{
-					operation: operation{
-						Package: match.Package,
-						Option:  installOpt,
-					},
-					Matches:     installMatch,
-					Packages:    toInstall,
-					Reposiories: syncedRepos,
-					Assertions:  solution,
-					Database:    allRepos,
-				},
-			})
-		} else {
-			resOps = append(resOps, installerOp{
-				Install: installOperation{
-					operation:   operation{Package: match.Package, Option: installOpt},
-					Matches:     installMatch,
-					Reposiories: syncedRepos,
-					Packages:    toInstall,
-					Assertions:  solution,
-					Database:    allRepos,
-				},
-			})
-		}
+	syncedRepos Repositories, toInstall pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) (resOps []installerOp, err error) {
+
+	uOpts := []operation{}
+	for _, u := range toUninstall {
+		uOpts = append(uOpts, operation{Package: u, Option: uninstallOpt})
 	}
-
-	for _, p := range toUninstall {
-		found := false
-
-		for _, match := range installMatch {
-			if match.Package.GetPackageName() == p.GetPackageName() {
-				found = true
-			}
-
-		}
-		if !found {
-			resOps = append(resOps, installerOp{
-				Uninstall: operation{Package: p, Option: uninstallOpt},
-			})
-		}
+	iOpts := []installOperation{}
+	for _, u := range installMatch {
+		iOpts = append(iOpts, installOperation{
+			operation: operation{
+				Package: u.Package,
+				Option:  installOpt,
+			},
+			Matches:     installMatch,
+			Packages:    toInstall,
+			Reposiories: syncedRepos,
+			Assertions:  solution,
+			Database:    allRepos,
+		})
 	}
-	return resOps
+	resOps = append(resOps, installerOp{
+		Uninstall: uOpts,
+		Install:   iOpts,
+	})
+
+	return resOps, nil
 }
 
 func (l *LuetInstaller) checkAndUpgrade(r Repositories, s *System) error {
-	//	Spinner(32)
 	uninstall, toInstall, err := l.computeUpgrade(r, s)
 	if err != nil {
 		return errors.Wrap(err, "failed computing upgrade")
 	}
-	//	SpinnerStop()
 
 	if len(toInstall) == 0 && len(uninstall) == 0 {
 		l.Options.Context.Info("Nothing to upgrade")
@@ -487,7 +485,34 @@ func (l *LuetInstaller) checkAndUpgrade(r Repositories, s *System) error {
 		}
 	}
 
-	return l.swap(o, r, uninstall, toInstall, s)
+	bus.Manager.Publish(bus.EventPreUpgrade, struct{ Uninstall, Install pkg.Packages }{Uninstall: uninstall, Install: toInstall})
+
+	err = l.swap(o, r, uninstall, toInstall, s)
+
+	bus.Manager.Publish(bus.EventPostUpgrade, struct {
+		Error              error
+		Uninstall, Install pkg.Packages
+	}{Uninstall: uninstall, Install: toInstall, Error: err})
+
+	if err != nil {
+		return err
+	}
+
+	if l.Options.AutoOSCheck {
+		l.Options.Context.Info("Performing automatic oscheck")
+		packs := s.OSCheck(l.Options.Context)
+		if len(packs) > 0 {
+			p := ""
+			for _, r := range packs {
+				p += " " + r.HumanReadableString()
+			}
+			l.Options.Context.Info("Following packages requires reinstallation: " + p)
+			return l.swap(o, r, packs, packs, s)
+		}
+		l.Options.Context.Info("OSCheck done")
+	}
+
+	return err
 }
 
 func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
@@ -566,6 +591,21 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 
 func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string]ArtifactMatch) error {
 
+	// Don't attempt to download stuff that is already in cache
+	missArtifacts := false
+	for _, m := range toDownload {
+		c := m.Repository.Client(l.Options.Context)
+		_, err := c.CacheGet(m.Artifact)
+		if err != nil {
+			missArtifacts = true
+		}
+	}
+
+	if !missArtifacts {
+		l.Options.Context.Debug("Packages already in cache, skipping download")
+		return nil
+	}
+
 	// Download packages into cache in parallel.
 	all := make(chan ArtifactMatch)
 
@@ -575,27 +615,28 @@ func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string
 
 	// Check if the terminal is big enough to display a progress bar
 	// https://github.com/pterm/pterm/blob/4c725e56bfd9eb38e1c7b9dec187b50b93baa8bd/progressbar_printer.go#L190
-	w, _, err := ctx.GetTerminalSize()
-	if ctx.IsTerminal && err == nil && w > 100 {
-		area, _ := pterm.DefaultArea.Start()
-		p, _ := pterm.DefaultProgressbar.WithPrintTogether(area).WithTotal(len(toDownload)).WithTitle("Downloading packages").Start()
+	w, _, err := logger.GetTerminalSize()
+
+	var pb *pterm.ProgressbarPrinter
+	if logger.IsTerminal() && err == nil && w > 100 {
+		area, _ := pterm.DefaultArea.Start()
+		pb = pterm.DefaultProgressbar.WithPrintTogether(area).WithTotal(len(toDownload)).WithTitle("Downloading packages")
+		pb, _ = pb.Start()
+		ctx.SetAnnotation("progressbar", pb)
 
-		ctx.AreaPrinter = area
-		ctx.ProgressBar = p
 		defer area.Stop()
 	}
 
 	// Download
 	for i := 0; i < l.Options.Concurrency; i++ {
 		wg.Add(1)
-		go l.downloadWorker(i, wg, all, ctx)
+		go l.downloadWorker(i, wg, pb, all, ctx)
 	}
 	for _, c := range toDownload {
 		all <- c
 	}
 	close(all)
 	wg.Wait()
-
 	return nil
 }
 
@@ -784,9 +825,11 @@ func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, c
 	l.Options.Context.Info("Checking for file conflicts..")
 	defer s.Clean() // Release memory
 
-	filesToInstall := []string{}
+	filesToInstall := map[string]interface{}{}
 	for _, m := range toInstall {
-		a, err := l.downloadPackage(m, l.Options.Context)
+		l.Options.Context.Debug("Checking file conflicts for", m.Package.HumanReadableString())
+
+		a, err := l.getPackage(m, l.Options.Context)
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Failed downloading package")
 		}
@@ -796,7 +839,7 @@ func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, c
 		}
 
 		for _, f := range files {
-			if helpers.Contains(filesToInstall, f) {
+			if _, ok := filesToInstall[f]; ok {
 				return fmt.Errorf(
 					"file conflict between packages to be installed",
 				)
@@ -815,9 +858,10 @@ func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, c
 					)
 				}
 			}
+			filesToInstall[f] = nil
 		}
-		filesToInstall = append(filesToInstall, files...)
 	}
+	l.Options.Context.Info("Done checking for file conflicts..")
 
 	return nil
 }
@@ -882,11 +926,10 @@ func (l *LuetInstaller) install(o Option, syncedRepos Repositories, toInstall ma
 	return s.ExecuteFinalizers(l.Options.Context, toFinalize)
 }
 
-func (l *LuetInstaller) downloadPackage(a ArtifactMatch, ctx *types.Context) (*artifact.PackageArtifact, error) {
-
+func (l *LuetInstaller) getPackage(a ArtifactMatch, ctx types.Context) (artifact *artifact.PackageArtifact, err error) {
 	cli := a.Repository.Client(ctx)
 
-	artifact, err := cli.DownloadArtifact(a.Artifact)
+	artifact, err = cli.DownloadArtifact(a.Artifact)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error on download artifact")
 	}
@@ -900,7 +943,7 @@ func (l *LuetInstaller) downloadPackage(a ArtifactMatch, ctx *types.Context) (*a
 
 func (l *LuetInstaller) installPackage(m ArtifactMatch, s *System) error {
 
-	a, err := l.downloadPackage(m, l.Options.Context)
+	a, err := l.getPackage(m, l.Options.Context)
 	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Failed downloading package")
 	}
@@ -920,20 +963,20 @@ func (l *LuetInstaller) installPackage(m ArtifactMatch, s *System) error {
 	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: m.Package.GetFingerPrint(), Files: files})
 }
 
-func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, ctx *types.Context) error {
+func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, pb *pterm.ProgressbarPrinter, c <-chan ArtifactMatch, ctx types.Context) error {
 	defer wg.Done()
 
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
-		_, err := l.downloadPackage(p, ctx)
+		_, err := l.getPackage(p, ctx)
 		if err != nil {
 			l.Options.Context.Error("Failed downloading package "+p.Package.GetName(), err.Error())
 			return errors.Wrap(err, "Failed downloading package "+p.Package.GetName())
 		} else {
 			l.Options.Context.Success(":package: Package ", p.Package.HumanReadableString(), "downloaded")
 		}
-		if ctx.ProgressBar != nil {
-			ctx.ProgressBar.Increment()
+		if pb != nil {
+			pb.Increment()
 		}
 	}
 
@@ -950,7 +993,7 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, installLock *
 		installLock.Unlock()
 		if err != nil && !l.Options.Force {
 			//TODO: Uninstall, rollback.
-			l.Options.Context.Fatal("Failed installing package "+p.Package.GetName(), err.Error())
+			l.Options.Context.Error("Failed installing package "+p.Package.GetName(), err.Error())
 			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
 		}
 		if err == nil {
@@ -963,7 +1006,7 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, installLock *
 	return nil
 }
 
-func checkAndPrunePath(ctx *types.Context, target, path string) {
+func checkAndPrunePath(ctx types.Context, target, path string) {
 	// check if now the target path is empty
 	targetPath := filepath.Dir(path)
 
@@ -995,7 +1038,7 @@ func checkAndPrunePath(ctx *types.Context, target, path string) {
 }
 
 // We will try to cleanup every path from the file, if the folders left behind are empty
-func pruneEmptyFilePath(ctx *types.Context, target string, path string) {
+func pruneEmptyFilePath(ctx types.Context, target string, path string) {
 	checkAndPrunePath(ctx, target, path)
 
 	// A path is for e.g. /usr/bin/bar
@@ -1020,16 +1063,54 @@ func pruneEmptyFilePath(ctx *types.Context, target string, path string) {
 	}
 }
 
-func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
+func (l *LuetInstaller) pruneFile(f string, s *System, cp *config.ConfigProtect) {
+	target := filepath.Join(s.Target, f)
+
+	if !l.Options.Context.GetConfig().ConfigProtectSkip && cp.Protected(f) {
+		l.Options.Context.Debug("Preserving protected file:", f)
+		return
+	}
+
+	l.Options.Context.Debug("Removing", target)
+	if l.Options.PreserveSystemEssentialData &&
+		strings.HasPrefix(f, l.Options.Context.GetConfig().System.PkgsCachePath) ||
+		strings.HasPrefix(f, l.Options.Context.GetConfig().System.DatabasePath) {
+		l.Options.Context.Warning("Preserve ", f, " which is required by luet ( you have to delete it manually if you really need to)")
+		return
+	}
+
+	fi, err := os.Lstat(target)
+	if err != nil {
+		l.Options.Context.Debug("File not found (it was before?) ", err.Error())
+		return
+	}
+	switch mode := fi.Mode(); {
+	case mode.IsDir():
+		files, err := ioutil.ReadDir(target)
+		if err != nil {
+			l.Options.Context.Debug("Failed reading folder", target, err.Error())
+		}
+		if len(files) != 0 {
+			l.Options.Context.Debug("Preserving not-empty folder", target)
+			return
+		}
+	}
+
+	if err = os.Remove(target); err != nil {
+		l.Options.Context.Debug("Failed removing file (maybe not present in the system target anymore ?)", target, err.Error())
+	} else {
+		l.Options.Context.Debug("Removed", target)
+	}
+
+	pruneEmptyFilePath(l.Options.Context, s.Target, target)
+}
+
+func (l *LuetInstaller) configProtectForPackage(p pkg.Package, s *System, files []string) *config.ConfigProtect {
+
 	var cp *config.ConfigProtect
 	annotationDir := ""
 
-	files, err := s.Database.GetPackageFiles(p)
-	if err != nil {
-		return errors.Wrap(err, "Failed getting installed files")
-	}
-
-	if !l.Options.Context.Config.ConfigProtectSkip {
+	if !l.Options.Context.GetConfig().ConfigProtectSkip {
 
 		if p.HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
 			dir, ok := p.GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
@@ -1039,72 +1120,43 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		}
 
 		cp = config.NewConfigProtect(annotationDir)
-		cp.Map(files, l.Options.Context.Config.GetConfigProtectConfFiles())
+		cp.Map(files, l.Options.Context.GetConfig().ConfigProtectConfFiles)
 	}
 
+	return cp
+}
+
+func (l *LuetInstaller) pruneFiles(files []string, cp *config.ConfigProtect, s *System) {
+
 	toRemove, notPresent := fileHelper.OrderFiles(s.Target, files)
 
 	// Remove from target
-	for _, f := range toRemove {
-		target := filepath.Join(s.Target, f)
+	for _, f := range append(toRemove, notPresent...) {
+		l.pruneFile(f, s, cp)
+	}
+}
 
-		if !l.Options.Context.Config.ConfigProtectSkip && cp.Protected(f) {
-			l.Options.Context.Debug("Preserving protected file:", f)
-			continue
-		}
-
-		l.Options.Context.Debug("Removing", target)
-		if l.Options.PreserveSystemEssentialData &&
-			strings.HasPrefix(f, l.Options.Context.Config.GetSystem().GetSystemPkgsCacheDirPath()) ||
-			strings.HasPrefix(f, l.Options.Context.Config.GetSystem().GetSystemRepoDatabaseDirPath()) {
-			l.Options.Context.Warning("Preserve ", f, " which is required by luet ( you have to delete it manually if you really need to)")
-			continue
-		}
-
-		fi, err := os.Lstat(target)
-		if err != nil {
-			l.Options.Context.Warning("File not found (it was before?) ", err.Error())
-			continue
-		}
-		switch mode := fi.Mode(); {
-		case mode.IsDir():
-			files, err := ioutil.ReadDir(target)
-			if err != nil {
-				l.Options.Context.Warning("Failed reading folder", target, err.Error())
-			}
-			if len(files) != 0 {
-				l.Options.Context.Debug("Preserving not-empty folder", target)
-				continue
-			}
-		}
-
-		if err = os.Remove(target); err != nil {
-			l.Options.Context.Warning("Failed removing file (maybe not present in the system target anymore ?)", target, err.Error())
-		} else {
-			l.Options.Context.Debug("Removed", target)
-		}
-
-		pruneEmptyFilePath(l.Options.Context, s.Target, target)
+func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
+	files, err := s.Database.GetPackageFiles(p)
+	if err != nil {
+		return errors.Wrap(err, "Failed getting installed files")
 	}
 
-	for _, f := range notPresent {
-		target := filepath.Join(s.Target, f)
+	cp := l.configProtectForPackage(p, s, files)
 
-		if !l.Options.Context.Config.ConfigProtectSkip && cp.Protected(f) {
-			l.Options.Context.Debug("Preserving protected file:", f)
-			continue
-		}
+	l.pruneFiles(files, cp, s)
 
-		if err = os.Remove(target); err != nil {
-			l.Options.Context.Debug("Failed removing file (not present in the system target)", target, err.Error())
-		} else {
-			l.Options.Context.Debug("Removed", target)
-		}
-
-		pruneEmptyFilePath(l.Options.Context, s.Target, target)
+	err = l.removePackage(p, s)
+	if err != nil {
+		return errors.Wrap(err, "Failed removing package files from database")
 	}
 
-	err = s.Database.RemovePackageFiles(p)
+	l.Options.Context.Info(":recycle: ", p.HumanReadableString(), "Removed :heavy_check_mark:")
+	return nil
+}
+
+func (l *LuetInstaller) removePackage(p pkg.Package, s *System) error {
+	err := s.Database.RemovePackageFiles(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed removing package files from database")
 	}
@@ -1114,8 +1166,6 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 	}
 
 	bus.Manager.Publish(bus.EventPackageUnInstall, p)
-
-	l.Options.Context.Info(":recycle: ", p.HumanReadableString(), "Removed :heavy_check_mark:")
 	return nil
 }
 
@@ -1163,7 +1213,7 @@ func (l *LuetInstaller) computeUninstall(o Option, s *System, packs ...pkg.Packa
 	return toUninstall, nil
 }
 
-func (l *LuetInstaller) generateUninstallFn(o Option, s *System, packs ...pkg.Package) (pkg.Packages, func() error, error) {
+func (l *LuetInstaller) generateUninstallFn(o Option, s *System, filesToInstall map[string]interface{}, packs ...pkg.Package) (pkg.Packages, func() error, error) {
 	for _, p := range packs {
 		if packs, _ := s.Database.FindPackages(p); len(packs) == 0 {
 			return nil, nil, errors.New(fmt.Sprintf("Package %s not found in the system", p.HumanReadableString()))
@@ -1177,9 +1227,34 @@ func (l *LuetInstaller) generateUninstallFn(o Option, s *System, packs ...pkg.Pa
 
 	uninstall := func() error {
 		for _, p := range toUninstall {
-			err := l.uninstall(p, s)
-			if err != nil && !o.Force {
-				return errors.Wrap(err, "Uninstall failed")
+			if len(filesToInstall) == 0 {
+				err := l.uninstall(p, s)
+				if err != nil && !o.Force {
+					return errors.Wrap(err, "Uninstall failed")
+				}
+
+			} else {
+				files, err := s.Database.GetPackageFiles(p)
+				if err != nil && !o.Force {
+					return errors.Wrap(err, "Failed getting installed files")
+				}
+
+				cp := l.configProtectForPackage(p, s, files)
+
+				toPrune := []string{}
+				for _, f := range files {
+					if _, exists := filesToInstall[f]; !exists {
+						toPrune = append(toPrune, f)
+					}
+				}
+				l.Options.Context.Debug("calculated files for removal", toPrune)
+				l.pruneFiles(toPrune, cp, s)
+
+				err = l.removePackage(p, s)
+				if err != nil && !o.Force {
+					return errors.Wrap(err, "Failed removing package")
+				}
+
 			}
 		}
 		return nil
@@ -1198,7 +1273,7 @@ func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
 		CheckConflicts:     l.Options.CheckConflicts,
 		FullCleanUninstall: l.Options.FullCleanUninstall,
 	}
-	toUninstall, uninstall, err := l.generateUninstallFn(o, s, packs...)
+	toUninstall, uninstall, err := l.generateUninstallFn(o, s, map[string]interface{}{}, packs...)
 	if err != nil {
 		return errors.Wrap(err, "while computing uninstall")
 	}

pkg/installer/installer_test.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 
 	//	. "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/api/core/types"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
@@ -45,16 +46,16 @@ func stubRepo(tmpdir, tree string) (*LuetSystemRepository, error) {
 		WithPriority(1),
 		WithSource(tmpdir),
 		WithTree(tree),
-		WithContext(types.NewContext()),
+		WithContext(context.NewContext()),
 		WithDatabase(pkg.NewInMemoryDatabase(false)),
 	)
 }
 
 var _ = Describe("Installer", func() {
-	ctx := types.NewContext()
+	ctx := context.NewContext()
 
 	BeforeEach(func() {
-		ctx = types.NewContext()
+		ctx = context.NewContext()
 	})
 
 	Context("Writes a repository definition", func() {
@@ -654,6 +655,7 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			inst := NewLuetInstaller(LuetInstallerOptions{
 				Concurrency: 1, Context: ctx,
+				Relaxed:             true,
 				PackageRepositories: types.LuetRepositories{*repo2.LuetRepository},
 			})
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
@@ -702,6 +704,254 @@ urls:
 
 		})
 
+		It("Compute the correct upgrade order", func() {
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/upgrade_complex")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec4, err := c.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec5, err := c.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+			spec4.SetOutputPath(tmpdir)
+			spec5.SetOutputPath(tmpdir)
+
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec4, spec5))
+
+			Expect(errs).To(BeEmpty())
+
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_complex")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(ctx, tmpdir, false, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+enable: true
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			inst := NewLuetInstaller(LuetInstallerOptions{
+				Concurrency: 1, Context: ctx,
+				Relaxed:             true,
+				PackageRepositories: types.LuetRepositories{*repo2.LuetRepository},
+			})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.GetPackages())).To(Equal(1))
+			p, err := system.Database.GetPackage(system.Database.GetPackages()[0])
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetName()).To(Equal("b"))
+
+			files, err := systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(files).To(Equal([]string{"test5", "test6"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			files, err = systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"})
+			Expect(files).To(Equal([]string{"test3", "test4"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test4"))).To(BeTrue())
+
+			err = inst.Upgrade(system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+
+		})
+
+		It("Compute the correct upgrade order with a package replacing multiple ones", func() {
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/upgrade_complex_multiple")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(6))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec3, err := c.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec4, err := c.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec5, err := c.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+			spec6, err := c.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+			spec4.SetOutputPath(tmpdir)
+			spec5.SetOutputPath(tmpdir)
+			spec3.SetOutputPath(tmpdir)
+			spec6.SetOutputPath(tmpdir)
+
+			_, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3, spec4, spec5, spec6))
+
+			Expect(errs).To(BeEmpty())
+
+			repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_complex_multiple")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(ctx, tmpdir, false, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+enable: true
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			inst := NewLuetInstaller(LuetInstallerOptions{
+				Concurrency: 1, Context: ctx,
+				Relaxed:             true,
+				PackageRepositories: types.LuetRepositories{*repo2.LuetRepository},
+			})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			bolt, err := ioutil.TempDir("", "db")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(bolt) // clean up
+
+			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
+			system := &System{Database: systemDB, Target: fakeroot}
+
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.GetPackages())).To(Equal(1))
+			p, err := system.Database.GetPackage(system.Database.GetPackages()[0])
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetName()).To(Equal("b"))
+
+			files, err := systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(files).To(Equal([]string{"test5", "test6"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.1"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test1"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test2"))).To(BeTrue())
+
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			files, err = systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"})
+			Expect(files).To(Equal([]string{"test3", "test4"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test4"))).To(BeTrue())
+
+			err = inst.Upgrade(system)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test1"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test2"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test3"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test4"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+
+		})
+
 		It("Handles package drops", func() {
 			//repo:=NewLuetSystemRepository()
 

pkg/installer/interface.go

@@ -16,13 +16,22 @@
 package installer
 
 import (
+	"github.com/mudler/luet/pkg/api/core/types"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
+	"github.com/mudler/luet/pkg/tree"
 	//"github.com/mudler/luet/pkg/solver"
 )
 
 type Client interface {
 	DownloadArtifact(*artifact.PackageArtifact) (*artifact.PackageArtifact, error)
 	DownloadFile(string) (string, error)
+	CacheGet(*artifact.PackageArtifact) (*artifact.PackageArtifact, error)
 }
 
 type Repositories []*LuetSystemRepository
+
+type Repository interface {
+	GetTree() tree.Builder
+	Client(types.Context) Client
+	GetName() string
+}

pkg/installer/repository.go

@@ -74,7 +74,7 @@ type LuetSystemRepository struct {
 	PushImages      bool                          `json:"-"`
 	ForcePush       bool                          `json:"-"`
 
-	imagePrefix string
+	imagePrefix, snapshotID string
 }
 
 type LuetSystemRepositoryMetadata struct {
@@ -115,11 +115,11 @@ func SystemRepositories(t types.LuetRepositories) Repositories {
 }
 
 // LoadBuildTree loads to the tree the compilation specs from the system repositories
-func LoadBuildTree(t tree.Builder, db pkg.PackageDatabase, ctx *types.Context) error {
+func LoadBuildTree(t tree.Builder, db pkg.PackageDatabase, ctx types.Context) error {
 	var reserr error
-	repos := SystemRepositories(ctx.Config.SystemRepositories)
+	repos := SystemRepositories(ctx.GetConfig().SystemRepositories)
 	for _, r := range repos {
-		repodir, err := ctx.Config.GetSystem().TempDir(r.Name)
+		repodir, err := ctx.TempDir(r.Name)
 		if err != nil {
 			reserr = multierr.Append(reserr, err)
 		}
@@ -378,8 +378,8 @@ func (r *LuetSystemRepository) SetPriority(n int) {
 	r.LuetRepository.Priority = n
 }
 
-func (r *LuetSystemRepository) initialize(ctx *types.Context, src string) error {
-	generator, err := r.getGenerator(ctx)
+func (r *LuetSystemRepository) initialize(ctx types.Context, src string) error {
+	generator, err := r.getGenerator(ctx, r.snapshotID)
 	if err != nil {
 		return errors.Wrap(err, "while constructing repository generator")
 	}
@@ -430,6 +430,11 @@ func (r *LuetSystemRepository) SetType(p string) {
 	r.LuetRepository.Type = p
 }
 
+// Sets snapshot ID
+func (r *LuetSystemRepository) SetSnapshotID(i string) {
+	r.snapshotID = i
+}
+
 func (r *LuetSystemRepository) GetVerify() bool {
 	return r.LuetRepository.Verify
 }
@@ -521,12 +526,12 @@ func (r *LuetSystemRepository) BumpRevision(repospec string, resetRevision bool)
 
 // AddMetadata adds the repository serialized content into the metadata key of the repository
 // It writes the serialized content to repospec, and writes the repository.meta.yaml file into dst
-func (r *LuetSystemRepository) AddMetadata(ctx *types.Context, repospec, dst string) (*artifact.PackageArtifact, error) {
+func (r *LuetSystemRepository) AddMetadata(ctx types.Context, repospec, dst string) (*artifact.PackageArtifact, error) {
 	// Create Metadata struct and serialized repository
 	meta, serialized := r.Serialize()
 
 	// Create metadata file and repository file
-	metaTmpDir, err := ctx.Config.GetSystem().TempDir("metadata")
+	metaTmpDir, err := ctx.TempDir("metadata")
 	defer os.RemoveAll(metaTmpDir) // clean up
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for metadata")
@@ -559,9 +564,9 @@ func (r *LuetSystemRepository) AddMetadata(ctx *types.Context, repospec, dst str
 // AddTree adds a tree.Builder with the given key to the repository.
 // It will generate an artifact which will be then embedded in the repository manifest
 // It returns the generated artifacts and an error
-func (r *LuetSystemRepository) AddTree(ctx *types.Context, t tree.Builder, dst, key string, defaults LuetRepositoryFile) (*artifact.PackageArtifact, error) {
+func (r *LuetSystemRepository) AddTree(ctx types.Context, t tree.Builder, dst, key string, defaults LuetRepositoryFile) (*artifact.PackageArtifact, error) {
 	// Create tree and repository file
-	archive, err := ctx.Config.GetSystem().TempDir("archive")
+	archive, err := ctx.TempDir("archive")
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for archive")
 	}
@@ -705,11 +710,15 @@ type RepositoryGenerator interface {
 	Initialize(string, pkg.PackageDatabase) ([]*artifact.PackageArtifact, error)
 }
 
-func (r *LuetSystemRepository) getGenerator(ctx *types.Context) (RepositoryGenerator, error) {
+func (r *LuetSystemRepository) getGenerator(ctx types.Context, snapshotID string) (RepositoryGenerator, error) {
+	if snapshotID == "" {
+		snapshotID = time.Now().Format("20060102150405")
+	}
+
 	var rg RepositoryGenerator
 	switch r.GetType() {
 	case DiskRepositoryType, HttpRepositoryType:
-		rg = &localRepositoryGenerator{context: ctx}
+		rg = &localRepositoryGenerator{context: ctx, snapshotID: snapshotID}
 	case DockerRepositoryType:
 		rg = &dockerRepositoryGenerator{
 			b:           r.Backend,
@@ -717,6 +726,7 @@ func (r *LuetSystemRepository) getGenerator(ctx *types.Context) (RepositoryGener
 			imagePush:   r.PushImages,
 			force:       r.ForcePush,
 			context:     ctx,
+			snapshotID:  snapshotID,
 		}
 	default:
 		return nil, errors.New("invalid repository type")
@@ -725,8 +735,8 @@ func (r *LuetSystemRepository) getGenerator(ctx *types.Context) (RepositoryGener
 }
 
 // Write writes the repository metadata to the supplied destination
-func (r *LuetSystemRepository) Write(ctx *types.Context, dst string, resetRevision, force bool) error {
-	rg, err := r.getGenerator(ctx)
+func (r *LuetSystemRepository) Write(ctx types.Context, dst string, resetRevision, force bool) error {
+	rg, err := r.getGenerator(ctx, r.snapshotID)
 	if err != nil {
 		return err
 	}
@@ -734,7 +744,7 @@ func (r *LuetSystemRepository) Write(ctx *types.Context, dst string, resetRevisi
 	return rg.Generate(r, dst, resetRevision)
 }
 
-func (r *LuetSystemRepository) Client(ctx *types.Context) Client {
+func (r *LuetSystemRepository) Client(ctx types.Context) Client {
 	switch r.GetType() {
 	case DiskRepositoryType:
 		return client.NewLocalClient(client.RepoData{Urls: r.GetUrls()}, ctx)
@@ -793,7 +803,7 @@ func (r *LuetSystemRepository) getRepoFile(c Client, key string) (*artifact.Pack
 
 }
 
-func (r *LuetSystemRepository) SyncBuildMetadata(ctx *types.Context, path string) error {
+func (r *LuetSystemRepository) SyncBuildMetadata(ctx types.Context, path string) error {
 
 	repo, err := r.Sync(ctx, false)
 	if err != nil {
@@ -842,11 +852,24 @@ func (r *LuetSystemRepository) referenceID() string {
 	return repositoryReferenceID
 }
 
-func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystemRepository, error) {
+func (r *LuetSystemRepository) Sync(ctx types.Context, force bool) (*LuetSystemRepository, error) {
 	var repoUpdated bool = false
 	var treefs, metafs string
 
+	repobasedir := ctx.GetConfig().System.GetRepoDatabaseDirPath(r.GetName())
+
+	toTimeSync := false
+	dat, err := ioutil.ReadFile(filepath.Join(repobasedir, "SYNCTIME"))
+	if err == nil {
+		parsed, _ := time.Parse(time.RFC3339, string(dat))
+		if time.Now().After(parsed.Add(24 * time.Hour)) {
+			toTimeSync = true
+			ctx.Debug(r.Name, "is old, refresh is suggested")
+		}
+	}
+
 	ctx.Debug("Sync of the repository", r.Name, "in progress...")
+
 	c := r.Client(ctx)
 	if c == nil {
 		return nil, errors.New("no client could be generated from repository")
@@ -854,20 +877,29 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 
 	repositoryReferenceID := r.referenceID()
 
-	// Retrieve remote repository.yaml for retrieve revision and date
-	file, err := c.DownloadFile(repositoryReferenceID)
-	if err != nil {
-		return nil, errors.Wrap(err, "while downloading "+repositoryReferenceID)
-	}
+	var downloadedRepoMeta *LuetSystemRepository
+	var file string
+	repoFile := filepath.Join(repobasedir, repositoryReferenceID)
 
-	repobasedir := ctx.Config.GetSystem().GetRepoDatabaseDirPath(r.GetName())
-	downloadedRepoMeta, err := r.ReadSpecFile(file)
-	if err != nil {
-		return nil, err
+	_, repoExistsErr := os.Stat(repoFile)
+	if toTimeSync || force || os.IsNotExist(repoExistsErr) {
+		// Retrieve remote repository.yaml for retrieve revision and date
+		file, err = c.DownloadFile(repositoryReferenceID)
+		if err != nil {
+			return nil, errors.Wrap(err, "while downloading "+repositoryReferenceID)
+		}
+		downloadedRepoMeta, err = r.ReadSpecFile(file)
+		if err != nil {
+			return nil, err
+		}
+		defer os.RemoveAll(file)
+	} else {
+		downloadedRepoMeta, err = r.ReadSpecFile(repoFile)
+		if err != nil {
+			return nil, err
+		}
+		repoUpdated = true
 	}
-	// Remove temporary file that contains repository.yaml
-	// Example: /tmp/HttpClient236052003
-	defer os.RemoveAll(file)
 
 	if r.Cached {
 		if !force {
@@ -891,11 +923,11 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 		}
 
 	} else {
-		treefs, err = ctx.Config.GetSystem().TempDir("treefs")
+		treefs, err = ctx.TempDir("treefs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 		}
-		metafs, err = ctx.Config.GetSystem().TempDir("metafs")
+		metafs, err = ctx.TempDir("metafs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met whilte creating tempdir for metafs")
 		}
@@ -958,8 +990,8 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 				downloadedRepoMeta.GetRevision(),
 				time.Unix(tsec, 0).String()))
 
-	} else {
-		ctx.Info("Repository", downloadedRepoMeta.GetName(), "is already up to date.")
+		now := time.Now().Format(time.RFC3339)
+		ioutil.WriteFile(filepath.Join(repobasedir, "SYNCTIME"), []byte(now), os.ModePerm)
 	}
 
 	meta, err := NewLuetSystemRepositoryMetadata(
@@ -983,11 +1015,14 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 	// e.g. locally we can override the type (disk), or priority
 	// while remotely it could be advertized differently
 	r.fill(downloadedRepoMeta)
-	ctx.Info(
-		fmt.Sprintf(":information_source: Repository: %s Priority: %d Type: %s",
-			downloadedRepoMeta.GetName(),
-			downloadedRepoMeta.GetPriority(),
-			downloadedRepoMeta.GetType()))
+
+	if !repoUpdated {
+		ctx.Info(
+			fmt.Sprintf(":information_source: Repository: %s Priority: %d Type: %s",
+				downloadedRepoMeta.GetName(),
+				downloadedRepoMeta.GetPriority(),
+				downloadedRepoMeta.GetType()))
+	}
 	return downloadedRepoMeta, nil
 }
 

pkg/installer/repository_docker.go

@@ -37,10 +37,10 @@ import (
 )
 
 type dockerRepositoryGenerator struct {
-	b                compiler.CompilerBackend
-	imagePrefix      string
-	imagePush, force bool
-	context          *types.Context
+	b                       compiler.CompilerBackend
+	imagePrefix, snapshotID string
+	imagePush, force        bool
+	context                 types.Context
 }
 
 func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
@@ -92,8 +92,8 @@ func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDataba
 		} else {
 			l.context.Info("Generating final image", packageImage,
 				"for package ", a.CompileSpec.GetPackage().HumanReadableString())
-			if opts, err := a.GenerateFinalImage(l.context, packageImage, l.b, true); err != nil {
-				return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+			if err := a.GenerateFinalImage(l.context, packageImage, l.b, true); err != nil {
+				return errors.Wrap(err, "Failed generating metadata tree"+packageImage)
 			}
 		}
 		if l.imagePush {
@@ -115,7 +115,7 @@ func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDataba
 	return art, nil
 }
 
-func pushImage(ctx *types.Context, b compiler.CompilerBackend, image string, force bool) error {
+func pushImage(ctx types.Context, b compiler.CompilerBackend, image string, force bool) error {
 	if b.ImageAvailable(image) && !force {
 		ctx.Debug("Image", image, "already present, skipping")
 		return nil
@@ -125,8 +125,8 @@ func pushImage(ctx *types.Context, b compiler.CompilerBackend, image string, for
 
 func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string) error {
 	d.context.Debug("Generating image", imageTree)
-	if opts, err := a.GenerateFinalImage(d.context, imageTree, d.b, false); err != nil {
-		return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
+	if err := a.GenerateFinalImage(d.context, imageTree, d.b, false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree "+imageTree)
 	}
 	if d.imagePush {
 		if err := pushImage(d.context, d.b, imageTree, true); err != nil {
@@ -138,7 +138,7 @@ func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArti
 
 func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec, tag string, r *LuetSystemRepository) error {
 	// create temp dir for metafile
-	metaDir, err := d.context.Config.GetSystem().TempDir("metadata")
+	metaDir, err := d.context.TempDir("metadata")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for metadata")
 	}
@@ -184,7 +184,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 
 	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
 
-	repoTemp, err := d.context.Config.GetSystem().TempDir("repo")
+	repoTemp, err := d.context.TempDir("repo")
 	if err != nil {
 		return errors.Wrap(err, "error met while creating tempdir for repository")
 	}
@@ -196,7 +196,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 		if err != nil {
 			return errors.Wrapf(err, "while downloading '%s'", imageRepository)
 		}
-		img, err := r.GetBackend().ImageReference(imageRepository)
+		img, err := r.GetBackend().ImageReference(imageRepository, true)
 		if err != nil {
 			return errors.Wrapf(err, "while downloading '%s'", imageRepository)
 		}
@@ -204,7 +204,6 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 			d.context,
 			img,
 			repoTemp,
-			d.context.Config.GetGeneral().SameOwner,
 			nil,
 		)
 		if err != nil {
@@ -271,8 +270,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 	// Create a named snapshot and push it.
 	// It edits the metadata pointing at the repository files associated with the snapshot
 	// And copies the new files
-	id := time.Now().Format("20060102150405")
-	artifacts, snapshotRepoFile, err := r.Snapshot(id, repoTemp)
+	artifacts, snapshotRepoFile, err := r.Snapshot(d.snapshotID, repoTemp)
 	if err != nil {
 		return errors.Wrap(err, "while creating snapshot")
 	}

pkg/installer/repository_local.go

@@ -32,13 +32,16 @@ import (
 	"github.com/pkg/errors"
 )
 
-type localRepositoryGenerator struct{ context *types.Context }
+type localRepositoryGenerator struct {
+	context    types.Context
+	snapshotID string
+}
 
 func (l *localRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
 	return buildPackageIndex(l.context, path, db)
 }
 
-func buildPackageIndex(ctx *types.Context, path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
+func buildPackageIndex(ctx types.Context, path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
 
 	var art []*artifact.PackageArtifact
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
@@ -124,7 +127,7 @@ func (g *localRepositoryGenerator) Generate(r *LuetSystemRepository, dst string,
 	// Create named snapshot.
 	// It edits the metadata pointing at the repository files associated with the snapshot
 	// And copies the new files
-	if _, _, err := r.Snapshot(time.Now().Format("20060102150405"), dst); err != nil {
+	if _, _, err := r.Snapshot(g.snapshotID, dst); err != nil {
 		return errors.Wrap(err, "while creating snapshot")
 	}
 

pkg/installer/repository_options.go

@@ -33,7 +33,7 @@ type RepositoryConfig struct {
 	CompilerBackend         compiler.CompilerBackend
 	ImagePrefix             string
 
-	context                                         *types.Context
+	context                                         types.Context
 	PushImages, Force, FromRepository, FromMetadata bool
 }
 
@@ -51,7 +51,7 @@ func (cfg *RepositoryConfig) Apply(opts ...RepositoryOption) error {
 	return nil
 }
 
-func WithContext(c *types.Context) func(cfg *RepositoryConfig) error {
+func WithContext(c types.Context) func(cfg *RepositoryConfig) error {
 	return func(cfg *RepositoryConfig) error {
 		cfg.context = c
 		return nil

pkg/installer/repository_test.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/api/core/types"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler"
@@ -49,16 +50,16 @@ func dockerStubRepo(tmpdir, tree, image string, push, force bool) (*LuetSystemRe
 		WithSource(tmpdir),
 		WithTree(tree),
 		WithDatabase(pkg.NewInMemoryDatabase(false)),
-		WithCompilerBackend(backend.NewSimpleDockerBackend(types.NewContext())),
+		WithCompilerBackend(backend.NewSimpleDockerBackend(context.NewContext())),
 		WithImagePrefix(image),
 		WithPushImages(push),
-		WithContext(types.NewContext()),
+		WithContext(context.NewContext()),
 		WithForce(force))
 }
 
 var _ = Describe("Repository", func() {
 	Context("Generation", func() {
-		ctx := types.NewContext()
+		ctx := context.NewContext()
 		It("Generate repository metadata", func() {
 
 			tmpdir, err := ioutil.TempDir("", "tree")
@@ -139,11 +140,11 @@ var _ = Describe("Repository", func() {
 			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(ctx), generalRecipe2.GetDatabase(), options.WithContext(types.NewContext()))
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(ctx), generalRecipe2.GetDatabase(), options.WithContext(context.NewContext()))
 			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(context.NewContext()))
 
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
@@ -466,12 +467,12 @@ urls:
 	})
 	Context("Docker repository", func() {
 		repoImage := os.Getenv("UNIT_TEST_DOCKER_IMAGE_REPOSITORY")
-		ctx := types.NewContext()
+		ctx := context.NewContext()
 		BeforeEach(func() {
 			if repoImage == "" {
 				Skip("UNIT_TEST_DOCKER_IMAGE_REPOSITORY not specified")
 			}
-			ctx = types.NewContext()
+			ctx = context.NewContext()
 		})
 
 		It("generates images", func() {
@@ -619,11 +620,8 @@ urls:
 
 			Expect(a.Unpack(ctx, extracted, false)).ToNot(HaveOccurred())
 
-			Expect(fileHelper.DirectoryIsEmpty(extracted)).To(BeFalse())
-			content, err := ioutil.ReadFile(filepath.Join(extracted, ".virtual"))
-			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.DirectoryIsEmpty(extracted)).To(BeTrue())
 
-			Expect(string(content)).To(Equal(""))
 		})
 
 		It("Searches files", func() {

pkg/installer/system.go

@@ -1,6 +1,8 @@
 package installer
 
 import (
+	"os"
+	"path/filepath"
 	"sync"
 
 	"github.com/hashicorp/go-multierror"
@@ -12,9 +14,10 @@ import (
 )
 
 type System struct {
-	Database  pkg.PackageDatabase
-	Target    string
-	fileIndex map[string]pkg.Package
+	Database          pkg.PackageDatabase
+	Target            string
+	fileIndex         map[string]pkg.Package
+	fileIndexPackages map[string]pkg.Package
 	sync.Mutex
 }
 
@@ -22,7 +25,24 @@ func (s *System) World() (pkg.Packages, error) {
 	return s.Database.World(), nil
 }
 
-func (s *System) ExecuteFinalizers(ctx *types.Context, packs []pkg.Package) error {
+func (s *System) OSCheck(ctx types.Context) (notFound pkg.Packages) {
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	for f, p := range s.fileIndex {
+		targetFile := filepath.Join(s.Target, f)
+		if _, err := os.Lstat(targetFile); err != nil {
+			if _, err := s.Database.FindPackage(p); err == nil {
+				ctx.Debugf("Missing file '%s' from '%s'", targetFile, p.HumanReadableString())
+				notFound = append(notFound, p)
+			}
+		}
+	}
+	notFound = notFound.Unique()
+	return
+}
+
+func (s *System) ExecuteFinalizers(ctx types.Context, packs []pkg.Package) error {
 	var errs error
 	executedFinalizer := map[string]bool{}
 	for _, p := range packs {
@@ -56,16 +76,27 @@ func (s *System) ExecuteFinalizers(ctx *types.Context, packs []pkg.Package) erro
 }
 
 func (s *System) buildFileIndex() {
+	// XXX: Replace with cache
 	s.Lock()
 	defer s.Unlock()
-	// Check if cache is empty or if it got modified
-	if s.fileIndex == nil { //|| len(s.Database.GetPackages()) != len(s.fileIndex) {
+
+	if s.fileIndex == nil {
 		s.fileIndex = make(map[string]pkg.Package)
+	}
+
+	if s.fileIndexPackages == nil {
+		s.fileIndexPackages = make(map[string]pkg.Package)
+	}
+
+	// Check if cache is empty or if it got modified
+	if len(s.Database.GetPackages()) != len(s.fileIndexPackages) {
+		s.fileIndexPackages = make(map[string]pkg.Package)
 		for _, p := range s.Database.World() {
 			files, _ := s.Database.GetPackageFiles(p)
 			for _, f := range files {
 				s.fileIndex[f] = p
 			}
+			s.fileIndexPackages[p.GetPackageName()] = p
 		}
 	}
 }
@@ -76,6 +107,13 @@ func (s *System) Clean() {
 	s.fileIndex = nil
 }
 
+func (s *System) FileIndex() map[string]pkg.Package {
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	return s.fileIndex
+}
+
 func (s *System) ExistsPackageFile(file string) (bool, pkg.Package, error) {
 	s.buildFileIndex()
 	s.Lock()

pkg/installer/system_test.go

@@ -19,6 +19,11 @@ import (
 
 	//	. "github.com/mudler/luet/pkg/installer"
 
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/api/core/context"
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -31,6 +36,7 @@ var _ = Describe("System", func() {
 		var s *System
 		var db pkg.PackageDatabase
 		var a, b *pkg.DefaultPackage
+		ctx := context.NewContext()
 
 		BeforeEach(func() {
 			db = pkg.NewInMemoryDatabase(false)
@@ -66,5 +72,18 @@ var _ = Describe("System", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p).To(Equal(b))
 		})
+
+		It("detect missing files", func() {
+			dir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(dir)
+			s.Target = dir
+			notfound := s.OSCheck(ctx)
+			Expect(len(notfound)).To(Equal(2))
+			ioutil.WriteFile(filepath.Join(dir, "f"), []byte{}, os.ModePerm)
+			ioutil.WriteFile(filepath.Join(dir, "foo"), []byte{}, os.ModePerm)
+			notfound = s.OSCheck(ctx)
+			Expect(len(notfound)).To(Equal(1))
+		})
 	})
 })

pkg/tree/compiler_recipe.go

@@ -174,6 +174,7 @@ func (r *CompilerRecipe) Load(path string) error {
 								filepath.Dir(currentpath))
 					}
 					pack.Requires(packbuild.GetRequires())
+
 					pack.Conflicts(packbuild.GetConflicts())
 				}
 

scripts/ginkgo.coverage.sh

@@ -23,7 +23,7 @@ coveragetxt="coverage.txt"
 
 
 generate_cover_data() {
-  ginkgo -flakeAttempts=3 -failFast -cover -r .
+  ginkgo -flakeAttempts=3 -failFast -race -cover -r .
   echo "" > ${coveragetxt}
   find . -type f -name "*.coverprofile" | while read -r file;  do cat "$file" >> ${coveragetxt} && mv "$file" "${coverdir}"; done
   echo "mode: $covermode" >"$profile"

tests/fixtures/caps/pkgC/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "empty"
+version: "0.1"

tests/fixtures/extra_perms/pkgA/0.1/build.yaml

@@ -0,0 +1,18 @@
+image: "alpine"
+unpack: true
+includes:
+  - /foo
+  - /foo/bar
+  - /foo/bar/suid
+  - /foo/bar/sticky
+  - /foo/bar/sgid
+steps:
+- mkdir -p /foo/bar
+- touch /foo/bar/suid
+- touch /foo/bar/sgid
+- touch /foo/bar/sticky
+- chown 100:100 /foo/bar
+- chown 101:101 /foo/bar/suid
+- chmod u+s /foo/bar/suid
+- chmod u-s,g+s /foo/bar/sgid
+- chmod +t /foo/bar/sticky

tests/fixtures/extra_perms/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "extra-perms"
+version: "0.1"

tests/fixtures/join_complex/c/c1/build.yaml

@@ -6,4 +6,8 @@ requires:
     category: "test"
     version: ">=0"
 
-requires_final_images: true
+requires_final_images: true
+
+steps:
+- | 
+   /bin/sh -c "if [ -e /usr/bin/generate.sh ]; then ls -liah /usr/bin && exit 1; fi"

tests/fixtures/join_complex/cat/b-1.1/build.yaml

@@ -6,8 +6,9 @@ requires:
 prelude:
   - echo foo > /test
   - echo bar > /test2
+  - cp -rf generate.sh /usr/bin/
 steps:
   - echo artifact5 > /newc
   - echo artifact6 > /newnewc
   - chmod +x generate.sh
-  - ./generate.sh
+  - ./generate.sh

tests/fixtures/upgrade_complex/cat/a/a-1.2/build.yaml

@@ -0,0 +1,4 @@
+image: "alpine"
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6

tests/fixtures/upgrade_complex/cat/a/a-1.2/definition.yaml

@@ -0,0 +1,4 @@
+category: "test"
+name: "a"
+version: "1.2"
+

tests/fixtures/upgrade_complex/cat/a/a/build.yaml

@@ -0,0 +1,4 @@
+image: "alpine"
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/upgrade_complex/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.1"