🆕 Tag 0.20.13

It is safer, and plays better with buildx

CONTRIBUTING.md

@@ -17,7 +17,7 @@ Join us in [slack](https://luet.slack.com/join/shared_invite/enQtOTQxMjcyNDQ0MDU
 ## All Code Changes Happen Through Pull Requests
 Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests:
 
-1. Fork the repo you want to contribute to and create your branch from `develop`.
+1. Fork the repo you want to contribute to and create your branch from `master`.
 2. If you've added code that should be tested, add tests.
 3. If you've changed APIs, update the [documentation](https://github.com/Luet-lab/docs).
 4. Ensure the test suite passes.

cmd/build.go

@@ -110,6 +110,9 @@ Build packages specifying multiple definition trees:
 		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
 		full, _ := cmd.Flags().GetBool("full")
 		rebuild, _ := cmd.Flags().GetBool("rebuild")
+		pushFinalImages, _ := cmd.Flags().GetBool("push-final-images")
+		pushFinalImagesRepository, _ := cmd.Flags().GetString("push-final-images-repository")
+		pushFinalImagesForce, _ := cmd.Flags().GetBool("push-final-images-force")
 
 		var results Results
 		backendArgs := viper.GetStringSlice("backend-args")
@@ -159,8 +162,7 @@ Build packages specifying multiple definition trees:
 
 		opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
 
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
-			options.NoDeps(nodeps),
+		compileropts := []options.Option{options.NoDeps(nodeps),
 			options.WithBackendType(backendType),
 			options.PushImages(push),
 			options.WithBuildValues(values),
@@ -177,8 +179,21 @@ Build packages specifying multiple definition trees:
 			options.WithContext(util.DefaultContext),
 			options.BackendArgs(backendArgs),
 			options.Concurrency(concurrency),
-			options.WithCompressionType(compression.Implementation(compressionType)),
-		)
+			options.WithCompressionType(compression.Implementation(compressionType))}
+
+		if pushFinalImages {
+			compileropts = append(compileropts, options.EnablePushFinalImages)
+			if pushFinalImagesForce {
+				compileropts = append(compileropts, options.ForcePushFinalImages)
+			}
+			if pushFinalImagesRepository != "" {
+				compileropts = append(compileropts, options.WithFinalRepository(pushFinalImagesRepository))
+			} else if imageRepository != "" {
+				compileropts = append(compileropts, options.WithFinalRepository(imageRepository))
+			}
+		}
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), compileropts...)
 
 		if full {
 			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
@@ -302,6 +317,11 @@ func init() {
 	buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
+
+	buildCmd.Flags().Bool("push-final-images", false, "Push final images while building")
+	buildCmd.Flags().Bool("push-final-images-force", false, "Override existing images")
+	buildCmd.Flags().String("push-final-images-repository", "", "Repository where to push final images to")
+
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
 	buildCmd.Flags().StringSlice("values", []string{}, "Build values file to interpolate with each package")
 	buildCmd.Flags().StringSliceP("backend-args", "a", []string{}, "Backend args")

cmd/oscheck.go

@@ -0,0 +1,140 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	installer "github.com/mudler/luet/pkg/installer"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/mudler/luet/cmd/util"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var osCheckCmd = &cobra.Command{
+	Use:   "oscheck",
+	Short: "Checks packages integrity",
+	Long: `List packages that are installed in the system which files are missing in the system.
+
+	$ luet oscheck
+	
+To reinstall packages in the list:
+	
+	$ luet oscheck --reinstall
+`,
+	Aliases: []string{"i"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+
+		force := viper.GetBool("force")
+		onlydeps := viper.GetBool("onlydeps")
+		yes := viper.GetBool("yes")
+
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+		util.SetSystemConfig(util.DefaultContext)
+
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		packs := system.OSCheck()
+		if !util.DefaultContext.Config.General.Quiet {
+			if len(packs) == 0 {
+				util.DefaultContext.Success("All good!")
+				os.Exit(0)
+			} else {
+				util.DefaultContext.Info("Following packages are missing files or are incomplete:")
+				for _, p := range packs {
+					util.DefaultContext.Info(p.HumanReadableString())
+				}
+			}
+		} else {
+			var s []string
+			for _, p := range packs {
+				s = append(s, p.HumanReadableString())
+			}
+			fmt.Println(strings.Join(s, " "))
+		}
+
+		reinstall, _ := cmd.Flags().GetBool("reinstall")
+		if reinstall {
+
+			// Strip version for reinstall
+			toInstall := pkg.Packages{}
+			for _, p := range packs {
+				new := p.Clone()
+				new.SetVersion(">=0")
+				toInstall = append(toInstall, new)
+			}
+
+			util.SetSolverConfig(util.DefaultContext)
+
+			util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+			util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+
+			// Load config protect configs
+			util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+				Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+				SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+				NoDeps:                      true,
+				Force:                       force,
+				OnlyDeps:                    onlydeps,
+				PreserveSystemEssentialData: true,
+				Ask:                         !yes,
+				DownloadOnly:                downloadOnly,
+				Context:                     util.DefaultContext,
+				PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			})
+
+			err := inst.Swap(packs, toInstall, system)
+			if err != nil {
+				util.DefaultContext.Fatal("Error: " + err.Error())
+			}
+		}
+	},
+}
+
+func init() {
+	osCheckCmd.Flags().String("system-dbpath", "", "System db path")
+	osCheckCmd.Flags().String("system-target", "", "System rootpath")
+	osCheckCmd.Flags().String("system-engine", "", "System DB engine")
+	osCheckCmd.Flags().Bool("reinstall", false, "reinstall")
+
+	osCheckCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	osCheckCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	osCheckCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	osCheckCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	osCheckCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	osCheckCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	osCheckCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	osCheckCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	osCheckCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(osCheckCmd)
+}

cmd/root.go

@@ -30,7 +30,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.20.5"
+	LuetCLIVersion = "0.20.13"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/search.go

@@ -371,6 +371,10 @@ Search can also return results in the terminal in different ways: as terminal ou
 		default:
 			if tableMode {
 				t.Render()
+			} else if util.DefaultContext.Config.General.Quiet {
+				for _, tt := range results.Packages {
+					fmt.Printf("%s/%s-%s\n", tt.Category, tt.Name, tt.Version)
+				}
 			} else {
 				l.Render()
 			}

cmd/util.go

@@ -19,9 +19,14 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/go-units"
+	"github.com/mudler/luet/pkg/api/core/image"
+	luettypes "github.com/mudler/luet/pkg/api/core/types"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	"github.com/pkg/errors"
 
 	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/helpers/docker"
@@ -29,6 +34,55 @@ import (
 	"github.com/spf13/cobra"
 )
 
+func pack(ctx *luettypes.Context, p, dst, imageName, arch, OS string) error {
+
+	tempimage, err := ctx.Config.GetSystem().TempFile("tempimage")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for "+p)
+	}
+	defer os.RemoveAll(tempimage.Name()) // clean up
+
+	if err := image.CreateTar(p, tempimage.Name(), imageName, arch, OS); err != nil {
+		return errors.Wrap(err, "could not create image from tar")
+	}
+
+	return fileHelper.CopyFile(tempimage.Name(), dst)
+}
+
+func NewPackCommand() *cobra.Command {
+
+	c := &cobra.Command{
+		Use:   "pack image src.tar dst.tar",
+		Short: "Pack a standard tar archive as a container image",
+		Long: `Pack creates a tar which can be loaded as an image from a standard flat tar archive, for e.g. with docker load. 
+It doesn't need the docker daemon to run, and allows to override default os/arch:
+		
+	luet util pack --os arm64 image:tag src.tar dst.tar
+`,
+		Args: cobra.MinimumNArgs(3),
+		Run: func(cmd *cobra.Command, args []string) {
+
+			image := args[0]
+			src := args[1]
+			dst := args[2]
+
+			arch, _ := cmd.Flags().GetString("arch")
+			os, _ := cmd.Flags().GetString("os")
+
+			err := pack(util.DefaultContext, src, dst, image, arch, os)
+			if err != nil {
+				util.DefaultContext.Fatal(err.Error())
+			}
+			util.DefaultContext.Info("Image packed as", image)
+		},
+	}
+
+	c.Flags().String("arch", runtime.GOARCH, "Image architecture")
+	c.Flags().String("os", runtime.GOOS, "Image OS")
+
+	return c
+}
+
 func NewUnpackCommand() *cobra.Command {
 
 	c := &cobra.Command{
@@ -102,5 +156,6 @@ func init() {
 
 	utilGroup.AddCommand(
 		NewUnpackCommand(),
+		NewPackCommand(),
 	)
 }

cmd/util/cli.go

@@ -156,11 +156,15 @@ func DisplayVersionBanner(c *types.Context, banner func(), version func() string
 		}
 	}
 	if display {
-		banner()
-		pterm.DefaultCenter.Print(version())
-		for _, l := range license {
-			pterm.DefaultCenter.Print(l)
-
+		if c.Config.General.Quiet {
+			pterm.Info.Printf("Luet %s\n", version())
+			pterm.Info.Println(strings.Join(license, "\n"))
+		} else {
+			banner()
+			pterm.DefaultCenter.Print(version())
+			for _, l := range license {
+				pterm.DefaultCenter.Print(l)
+			}
 		}
 	}
 }

cmd/util/config.go

@@ -120,6 +120,7 @@ func setDefaults(viper *viper.Viper) {
 
 	viper.SetDefault("general.concurrency", runtime.NumCPU())
 	viper.SetDefault("general.debug", false)
+	viper.SetDefault("general.quiet", false)
 	viper.SetDefault("general.show_build_output", false)
 	viper.SetDefault("general.fatal_warnings", false)
 	viper.SetDefault("general.http_timeout", 360)
@@ -159,7 +160,8 @@ func InitViper(ctx *types.Context, RootCmd *cobra.Command) {
 	cobra.OnInitialize(initConfig)
 	pflags := RootCmd.PersistentFlags()
 	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
-	pflags.BoolP("debug", "d", false, "verbose output")
+	pflags.BoolP("debug", "d", false, "debug output")
+	pflags.BoolP("quiet", "q", false, "quiet output")
 	pflags.Bool("fatal", false, "Enables Warnings to exit")
 	pflags.Bool("enable-logfile", false, "Enable log to file")
 	pflags.Bool("no-spinner", false, "Disable spinner.")
@@ -188,6 +190,7 @@ func InitViper(ctx *types.Context, RootCmd *cobra.Command) {
 
 	viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
 	viper.BindPFlag("general.debug", pflags.Lookup("debug"))
+	viper.BindPFlag("general.quiet", pflags.Lookup("quiet"))
 	viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
 	viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
 	viper.BindPFlag("plugin", pflags.Lookup("plugin"))

pkg/api/core/image/create.go

@@ -19,6 +19,7 @@ import (
 	"io"
 	"os"
 
+	containerdCompression "github.com/containerd/containerd/archive/compression"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
@@ -27,18 +28,31 @@ import (
 	"github.com/pkg/errors"
 )
 
-func imageFromTar(imagename string, r io.Reader) (name.Reference, v1.Image, error) {
+func imageFromTar(imagename, architecture, OS string, opener func() (io.ReadCloser, error)) (name.Reference, v1.Image, error) {
 	newRef, err := name.ParseReference(imagename)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	layer, err := tarball.LayerFromReader(r)
+	layer, err := tarball.LayerFromOpener(opener)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	img, err := mutate.Append(empty.Image, mutate.Addendum{
+	baseImage := empty.Image
+	cfg, err := baseImage.ConfigFile()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfg.Architecture = architecture
+	cfg.OS = OS
+
+	baseImage, err = mutate.ConfigFile(baseImage, cfg)
+	if err != nil {
+		return nil, nil, err
+	}
+	img, err := mutate.Append(baseImage, mutate.Addendum{
 		Layer: layer,
 		History: v1.History{
 			CreatedBy: "luet",
@@ -53,29 +67,31 @@ func imageFromTar(imagename string, r io.Reader) (name.Reference, v1.Image, erro
 }
 
 // CreateTar a imagetarball from a standard tarball
-func CreateTar(srctar, dstimageTar, imagename string) error {
-	f, err := os.Open(srctar)
-	if err != nil {
-		return errors.Wrap(err, "Cannot open "+srctar)
-	}
-	defer f.Close()
+func CreateTar(srctar, dstimageTar, imagename, architecture, OS string) error {
 
-	return CreateTarReader(f, dstimageTar, imagename)
-}
-
-// CreateTarReader a imagetarball from a standard tarball
-func CreateTarReader(r io.Reader, dstimageTar, imagename string) error {
 	dstFile, err := os.Create(dstimageTar)
 	if err != nil {
 		return errors.Wrap(err, "Cannot create "+dstimageTar)
 	}
 	defer dstFile.Close()
 
-	newRef, img, err := imageFromTar(imagename, r)
+	newRef, img, err := imageFromTar(imagename, architecture, OS, func() (io.ReadCloser, error) {
+		f, err := os.Open(srctar)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot open "+srctar)
+		}
+		decompressed, err := containerdCompression.DecompressStream(f)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot open "+srctar)
+		}
+
+		return decompressed, nil
+	})
 	if err != nil {
 		return err
 	}
 
 	// NOTE: We might also stream that back to the daemon with daemon.Write(tag, img)
 	return tarball.Write(newRef, img, dstFile)
+
 }

pkg/api/core/image/create_test.go

@@ -18,6 +18,7 @@ package image_test
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 
 	. "github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types"
@@ -46,7 +47,7 @@ var _ = Describe("Create", func() {
 			img, err := b.ImageReference("alpine", false)
 			Expect(err).ToNot(HaveOccurred())
 
-			_, dir, err := Extract(ctx, img, false, nil)
+			_, dir, err := Extract(ctx, img, nil)
 			Expect(err).ToNot(HaveOccurred())
 
 			defer os.RemoveAll(dir)
@@ -58,7 +59,7 @@ var _ = Describe("Create", func() {
 			a.Compress(dir, 1)
 
 			// Unfortunately there is no other easy way to test this
-			err = CreateTar(srcTar.Name(), dst.Name(), "testimage")
+			err = CreateTar(srcTar.Name(), dst.Name(), "testimage", runtime.GOARCH, runtime.GOOS)
 			Expect(err).ToNot(HaveOccurred())
 
 			b.LoadImage(dst.Name())
@@ -68,7 +69,7 @@ var _ = Describe("Create", func() {
 			img, err = b.ImageReference("testimage", false)
 			Expect(err).ToNot(HaveOccurred())
 
-			_, dir, err = Extract(ctx, img, false, nil)
+			_, dir, err = Extract(ctx, img, nil)
 			Expect(err).ToNot(HaveOccurred())
 
 			defer os.RemoveAll(dir)

pkg/api/core/image/delta_test.go

@@ -74,7 +74,6 @@ var _ = Describe("Delta", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
 					f,
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -94,7 +93,6 @@ var _ = Describe("Delta", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
 					f,
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -109,7 +107,6 @@ var _ = Describe("Delta", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
 					f,
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -124,7 +121,6 @@ var _ = Describe("Delta", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
 					f,
 				)
 				Expect(err).ToNot(HaveOccurred())

pkg/api/core/image/extract.go

@@ -22,10 +22,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"syscall"
 
 	containerdarchive "github.com/containerd/containerd/archive"
-	"github.com/docker/docker/pkg/system"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/mudler/luet/pkg/api/core/types"
@@ -195,7 +193,7 @@ func ExtractFiles(
 // ExtractReader perform the extracting action over the io.ReadCloser
 // it extracts the files over output. Accepts a filter as an option
 // and additional containerd Options
-func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	defer reader.Close()
 
 	// If no filter is specified, grab all.
@@ -203,34 +201,7 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keep
 		filter = func(h *tar.Header) (bool, error) { return true, nil }
 	}
 
-	// Keep records of permissions as we walk the tar
-	type permData struct {
-		PAX, Xattrs map[string]string
-		Uid, Gid    int
-		Name        string
-	}
-
-	permstore, err := ctx.Config.System.TempDir("permstore")
-	if err != nil {
-		return 0, "", err
-	}
-	perms := NewCache(permstore, 50*1024*1024, 10000)
-
-	f := func(h *tar.Header) (bool, error) {
-		res, err := filter(h)
-		if res {
-			perms.SetValue(h.Name, permData{
-				PAX: h.PAXRecords,
-				Uid: h.Uid, Gid: h.Gid,
-				Xattrs: h.Xattrs,
-				Name:   h.Name,
-			})
-			//perms = append(perms, })
-		}
-		return res, err
-	}
-
-	opts = append(opts, containerdarchive.WithFilter(f))
+	opts = append(opts, containerdarchive.WithFilter(filter))
 
 	// Handle the extraction
 	c, err := containerdarchive.Apply(context.Background(), output, reader, opts...)
@@ -238,42 +209,19 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keep
 		return 0, "", err
 	}
 
-	// Reconstruct permissions
-	if keepPerms {
-		ctx.Debug("Reconstructing permissions")
-		perms.All(func(cr CacheResult) {
-			p := &permData{}
-			cr.Unmarshal(p)
-			ff := filepath.Join(output, p.Name)
-			if _, err := os.Lstat(ff); err == nil {
-				if err := os.Lchown(ff, p.Uid, p.Gid); err != nil {
-					ctx.Warning(err, "failed chowning file")
-				}
-			}
-			for _, attrs := range []map[string]string{p.Xattrs, p.PAX} {
-				for k, attr := range attrs {
-					if err := system.Lsetxattr(ff, k, []byte(attr), 0); err != nil {
-						if errors.Is(err, syscall.ENOTSUP) {
-							ctx.Debug("ignored xattr %s in archive", ff)
-						}
-					}
-				}
-			}
-		})
-	}
 	return c, output, nil
 }
 
 // Extract is just syntax sugar around ExtractReader. It extracts an image into a dir
-func Extract(ctx *types.Context, img v1.Image, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
 	if err != nil {
 		return 0, "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
-	return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, keepPerms, filter, opts...)
+	return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, filter, opts...)
 }
 
 // ExtractTo is just syntax sugar around ExtractReader
-func ExtractTo(ctx *types.Context, img v1.Image, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
-	return ExtractReader(ctx, mutate.Extract(img), output, keepPerms, filter, opts...)
+func ExtractTo(ctx *types.Context, img v1.Image, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+	return ExtractReader(ctx, mutate.Extract(img), output, filter, opts...)
 }

pkg/api/core/image/extract_test.go

@@ -58,7 +58,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "", []string{}, []string{}),
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -72,7 +71,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "/usr", []string{}, []string{}),
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -86,7 +84,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "/usr", []string{"bin"}, []string{"sbin"}),
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -101,7 +98,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "", []string{"/usr|/usr/bin"}, []string{"^/bin"}),
 				)
 				Expect(err).ToNot(HaveOccurred())

pkg/api/core/types/artifact/artifact.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 
 	"github.com/docker/docker/pkg/pools"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -69,8 +70,8 @@ type PackageArtifact struct {
 	Runtime           *pkg.DefaultPackage               `json:"runtime,omitempty"`
 }
 
-func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, keepPerms bool, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
-	_, tmpdiffs, err := image.Extract(ctx, img, keepPerms, filter)
+func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
+	_, tmpdiffs, err := image.Extract(ctx, img, filter)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
@@ -210,16 +211,6 @@ type ImageBuilder interface {
 
 // GenerateFinalImage takes an artifact and builds a Docker image with its content
 func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) error {
-	archiveFile, err := os.Open(a.Path)
-	if err != nil {
-		return errors.Wrap(err, "Cannot open "+a.Path)
-	}
-	defer archiveFile.Close()
-
-	decompressed, err := containerdCompression.DecompressStream(archiveFile)
-	if err != nil {
-		return errors.Wrap(err, "Cannot open "+a.Path)
-	}
 
 	tempimage, err := ctx.Config.GetSystem().TempFile("tempimage")
 	if err != nil {
@@ -227,7 +218,7 @@ func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName strin
 	}
 	defer os.RemoveAll(tempimage.Name()) // clean up
 
-	if err := image.CreateTarReader(decompressed, tempimage.Name(), imageName); err != nil {
+	if err := image.CreateTar(a.Path, tempimage.Name(), imageName, runtime.GOARCH, runtime.GOOS); err != nil {
 		return errors.Wrap(err, "could not create image from tar")
 	}
 
@@ -569,7 +560,7 @@ func (a *PackageArtifact) Unpack(ctx *types.Context, dst string, keepPerms bool)
 	// 	//	tarModifier.Modifier()
 	// 	return true, nil
 	// },
-	_, _, err = image.ExtractReader(ctx, replacerArchive, dst, ctx.Config.GetGeneral().SameOwner, nil)
+	_, _, err = image.ExtractReader(ctx, replacerArchive, dst, nil)
 	return err
 }
 

pkg/api/core/types/artifact/artifact_test.go

@@ -161,7 +161,6 @@ RUN echo bar > /test2`))
 				ctx,
 				img,
 				result,
-				false,
 				nil,
 			)
 			Expect(err).ToNot(HaveOccurred())
@@ -210,7 +209,6 @@ RUN echo bar > /test2`))
 				ctx,
 				img,
 				result,
-				false,
 				nil,
 			)
 			Expect(err).ToNot(HaveOccurred())

pkg/api/core/types/config.go

@@ -61,6 +61,7 @@ type LuetGeneralConfig struct {
 	ShowBuildOutput bool `yaml:"show_build_output,omitempty" mapstructure:"show_build_output"`
 	FatalWarns      bool `yaml:"fatal_warnings,omitempty" mapstructure:"fatal_warnings"`
 	HTTPTimeout     int  `yaml:"http_timeout,omitempty" mapstructure:"http_timeout"`
+	Quiet           bool `yaml:"quiet" mapstructure:"quiet"`
 }
 
 type LuetSolverOptions struct {

pkg/api/core/types/context.go

@@ -48,11 +48,11 @@ type Context struct {
 	Config     *LuetConfig
 	IsTerminal bool
 	NoSpinner  bool
+	name       string
 
 	s           *pterm.SpinnerPrinter
 	spinnerLock *sync.Mutex
 	z           *zap.Logger
-	AreaPrinter *pterm.AreaPrinter
 	ProgressBar *pterm.ProgressbarPrinter
 }
 
@@ -73,6 +73,12 @@ func NewContext() *Context {
 	}
 }
 
+func (c *Context) WithName(name string) *Context {
+	newc := c.Copy()
+	newc.name = name
+	return newc
+}
+
 func (c *Context) Copy() *Context {
 
 	configCopy := *c.Config
@@ -113,6 +119,11 @@ func (c *Context) Init() (err error) {
 		c.NoColor()
 	}
 
+	if c.Config.General.Quiet {
+		c.NoColor()
+		pterm.DisableStyling()
+	}
+
 	c.Debug("Colors", c.Config.GetLogging().Color)
 	c.Debug("Logging level", c.Config.GetLogging().Level)
 	c.Debug("Debug mode", c.Config.GetGeneral().Debug)
@@ -307,6 +318,10 @@ func (c *Context) Msg(level LogLevel, ln bool, msg ...interface{}) {
 		levelMsg = re.ReplaceAllString(levelMsg, "")
 	}
 
+	if c.name != "" {
+		levelMsg = fmt.Sprintf("[%s] %s", c.name, levelMsg)
+	}
+
 	if c.z != nil {
 		c.log2File(level, message)
 	}

pkg/compiler/compiler.go

@@ -239,23 +239,9 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
 		return nil, err
 	}
 
-	// artifact.ImageToArtifact(
-	// 	cs.Options.Context,
-	// 	img,
-	// 	cs.Options.CompressionType,
-	// 	p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"),
-	// 	image.ExtractFiles(
-	// 		cs.Options.Context,
-	// 		strings.TrimLeft(p.GetPackageDir(), "/"),
-	// 		p.GetIncludes(),
-	// 		p.GetExcludes(),
-	// 	),
-	// )
-	// TODO: Trim includes/excludes from "/" ?
 	_, rootfs, err := image.Extract(
 		cs.Options.Context,
 		img,
-		keepPermissions,
 		image.ExtractFiles(
 			cs.Options.Context,
 			p.GetPackageDir(),
@@ -287,11 +273,11 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
 
 func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, builderOpts, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
 
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	rootfs, err := cs.Options.Context.Config.System.TempDir("rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
 	}
-	defer os.RemoveAll(rootfs) // clean up
+	defer os.RemoveAll(rootfs)
 
 	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
 	if cs.Options.PullFirst {
@@ -338,7 +324,6 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co
 		ref2,
 		cs.Options.CompressionType,
 		p.Rel(fmt.Sprintf("%s%s", p.GetPackage().GetFingerPrint(), ".package.tar")),
-		keepPermissions,
 		filter,
 	)
 	if err != nil {
@@ -386,7 +371,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	}
 
 	// First we create the builder image
-	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")); err != nil {
+	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().ImageID()+"-builder.dockerfile")); err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
 	}
 
@@ -401,21 +386,21 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	// steps in prelude are == 0 those are equivalent.
 
 	// Then we write the step image, which uses the builder one
-	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")); err != nil {
+	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().ImageID()+".dockerfile")); err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
 	}
 
 	builderOpts = backend.Options{
 		ImageName:      buildertaggedImage,
 		SourcePath:     buildDir,
-		DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
+		DockerFileName: p.GetPackage().ImageID() + "-builder.dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 		BackendArgs:    cs.Options.BackendArgs,
 	}
 	runnerOpts = backend.Options{
 		ImageName:      packageImage,
 		SourcePath:     buildDir,
-		DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
+		DockerFileName: p.GetPackage().ImageID() + ".dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 		BackendArgs:    cs.Options.BackendArgs,
 	}
@@ -473,11 +458,11 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	if p.EmptyPackage() {
 		fakePackage := p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")
 
-		rootfs, err = ioutil.TempDir(p.GetOutputPath(), "rootfs")
+		rootfs, err = cs.Options.Context.Config.System.TempDir("rootfs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not create tempdir")
 		}
-		defer os.RemoveAll(rootfs) // clean up
+		defer os.RemoveAll(rootfs)
 
 		a := artifact.NewPackageArtifact(fakePackage)
 		a.CompressionType = cs.Options.CompressionType
@@ -488,12 +473,16 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 
 		a.CompileSpec = p
 		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
-
 		err = a.WriteYAML(p.GetOutputPath())
 		if err != nil {
 			return a, errors.Wrap(err, "Failed while writing metadata file")
 		}
 		cs.Options.Context.Success(pkgTag, "   :white_check_mark: done (empty virtual package)")
+		if cs.Options.PushFinalImages {
+			if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil {
+				return nil, err
+			}
+		}
 		return a, nil
 	}
 
@@ -523,11 +512,55 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	if err != nil {
 		return a, errors.Wrap(err, "Failed while writing metadata file")
 	}
-	cs.Options.Context.Success(pkgTag, "   :white_check_mark: Done")
+	cs.Options.Context.Success(pkgTag, "   :white_check_mark: Done building")
+
+	if cs.Options.PushFinalImages {
+		if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil {
+			return nil, err
+		}
+	}
 
 	return a, nil
 }
 
+// TODO: A small readaptation of repository_docker.go pushImageFromArtifact()
+//       Move this to a common place
+func (cs *LuetCompiler) pushFinalArtifact(a *artifact.PackageArtifact, p *compilerspec.LuetCompilationSpec, keepPermissions bool) error {
+	cs.Options.Context.Info("Pushing final image for", a.CompileSpec.Package.HumanReadableString())
+	imageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, a.CompileSpec.Package.ImageID())
+
+	// First push the package image
+	if !cs.Backend.ImageAvailable(imageID) || cs.Options.PushFinalImagesForce {
+		cs.Options.Context.Info("Generating and pushing final image for", a.CompileSpec.Package.HumanReadableString(), "as", imageID)
+
+		if err := a.GenerateFinalImage(cs.Options.Context, imageID, cs.GetBackend(), true); err != nil {
+			return errors.Wrap(err, "while creating final image")
+		}
+		if err := cs.Backend.Push(backend.Options{ImageName: imageID}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", imageID)
+		}
+	}
+
+	// Then the image ID
+	metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, helpers.SanitizeImageString(a.CompileSpec.GetPackage().GetMetadataFilePath()))
+	if !cs.Backend.ImageAvailable(metadataImageID) || cs.Options.PushFinalImagesForce {
+		cs.Options.Context.Info("Generating metadata image for", a.CompileSpec.Package.HumanReadableString(), metadataImageID)
+
+		a := artifact.NewPackageArtifact(filepath.Join(p.GetOutputPath(), a.CompileSpec.GetPackage().GetMetadataFilePath()))
+		metadataArchive, err := artifact.CreateArtifactForFile(cs.Options.Context, a.Path)
+		if err != nil {
+			return errors.Wrap(err, "failed generating checksums for tree")
+		}
+		if err := metadataArchive.GenerateFinalImage(cs.Options.Context, metadataImageID, cs.Backend, keepPermissions); err != nil {
+			return errors.Wrap(err, "Failed generating metadata tree "+metadataImageID)
+		}
+		if err = cs.Backend.Push(backend.Options{ImageName: metadataImageID}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", metadataImageID)
+		}
+	}
+	return nil
+}
+
 func (cs *LuetCompiler) waitForImages(images []string) {
 	if cs.Options.PullFirst && cs.Options.Wait {
 		available, _ := oneOfImagesAvailable(images, cs.Backend)
@@ -906,11 +939,11 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 	}
 
 	// otherwise, generate it and push it aside
-	joinDir, err := ioutil.TempDir(p.GetOutputPath(), "join")
+	joinDir, err := cs.Options.Context.Config.System.TempDir("join")
 	if err != nil {
 		return errors.Wrap(err, "could not create tempdir for joining images")
 	}
-	defer os.RemoveAll(joinDir) // clean up
+	defer os.RemoveAll(joinDir)
 
 	for _, p := range fromPackages {
 		cs.Options.Context.Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
@@ -945,11 +978,11 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 		}
 	}
 
-	artifactDir, err := ioutil.TempDir(p.GetOutputPath(), "artifact")
+	artifactDir, err := cs.Options.Context.Config.System.TempDir("join")
 	if err != nil {
 		return errors.Wrap(err, "could not create tempdir for final artifact")
 	}
-	defer os.RemoveAll(joinDir) // clean up
+	defer os.RemoveAll(artifactDir)
 
 	cs.Options.Context.Info(joinTag, ":droplet: generating artifact for source image of", p.GetPackage().HumanReadableString())
 
@@ -1044,7 +1077,34 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 
 	ht := NewHashTree(cs.Database)
 
-	packageHashTree, err := ht.Query(cs, p)
+	// When computing the hash tree, we need to take into consideration
+	// that packages that require final images have to be seen as packages without deps
+	// This is because we don't really want to calculate the deptree of them as
+	// as it is handled already when we are creating the images in resolveFinalImages().
+	c := *cs
+	copy := &c
+	memDB := pkg.NewInMemoryDatabase(false)
+	// Create a copy to avoid races
+	dbCopy := pkg.NewInMemoryDatabase(false)
+	err := cs.Database.Clone(dbCopy)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed cloning db")
+	}
+	for _, p := range dbCopy.World() {
+		copy := p.Clone()
+		spec, err := cs.FromPackage(p)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed getting compile spec for package "+p.HumanReadableString())
+		}
+		if spec.RequiresFinalImages {
+			copy.Requires([]*pkg.DefaultPackage{})
+		}
+
+		memDB.CreatePackage(copy)
+	}
+	copy.Database = memDB
+
+	packageHashTree, err := ht.Query(copy, p)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed querying hashtree")
 	}
@@ -1102,6 +1162,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 	buildTarget := !cs.Options.OnlyDeps
 
 	if buildDeps {
+
 		cs.Options.Context.Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
 		for _, assertion := range dependencies { //highly dependent on the order
 			depsN++
@@ -1117,6 +1178,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
 			}
 			compileSpec.BuildOptions.PullImageRepository = append(compileSpec.BuildOptions.PullImageRepository, p.BuildOptions.PullImageRepository...)
+
 			cs.Options.Context.Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
 
 			compileSpec.SetOutputPath(p.GetOutputPath())
@@ -1273,11 +1335,12 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
 	} else {
 		bv := cs.Options.BuildValuesFile
 		if len(vals) > 0 {
-			valuesdir, err := ioutil.TempDir("", "genvalues")
+			valuesdir, err := cs.Options.Context.Config.System.TempDir("genvalues")
 			if err != nil {
 				return nil, errors.Wrap(err, "Could not create tempdir")
 			}
-			defer os.RemoveAll(valuesdir) // clean up
+			defer os.RemoveAll(valuesdir)
+
 			for _, b := range vals {
 				out, err := yaml.Marshal(b)
 				if err != nil {

pkg/compiler/compiler_test.go

@@ -16,10 +16,15 @@
 package compiler_test
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
+	helpers "github.com/mudler/luet/tests/helpers"
+
+	"github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types"
 	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	. "github.com/mudler/luet/pkg/compiler"
@@ -853,6 +858,67 @@ var _ = Describe("Compiler", func() {
 			Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
+
+		It("Pushes final images along", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			randString := strings.ToLower(helpers.String(10))
+			imageName := fmt.Sprintf("ttl.sh/%s", randString)
+			b := sd.NewSimpleDockerBackend(ctx)
+
+			err := generalRecipe.Load("../../tests/fixtures/packagelayers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(b, generalRecipe.GetDatabase(),
+				options.EnablePushFinalImages, options.ForcePushFinalImages, options.WithFinalRepository(imageName))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "build", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(2))
+			//Expect(len(artifacts[0].Dependencies)).To(Equal(1))
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.ImageID()))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.GetMetadataFilePath()))).To(BeTrue())
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.ImageID()))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.GetMetadataFilePath()))).To(BeTrue())
+
+			img, err := b.ImageReference(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.ImageID()), true)
+			Expect(err).ToNot(HaveOccurred())
+			_, path, err := image.Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(path) // clean up
+
+			Expect(fileHelper.Exists(filepath.Join(path, "bin/busybox"))).To(BeTrue())
+
+			img, err = b.ImageReference(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.GetMetadataFilePath()), true)
+			Expect(err).ToNot(HaveOccurred())
+			_, path, err = image.Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(path) // clean up
+
+			meta := filepath.Join(path, artifacts[1].Runtime.GetMetadataFilePath())
+			Expect(fileHelper.Exists(meta)).To(BeTrue())
+
+			d, err := ioutil.ReadFile(meta)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(d)).To(ContainSubstring(artifacts[1].CompileSpec.GetPackage().GetName()))
+		})
 	})
 
 	Context("Packages which conents are a package folder", func() {

pkg/compiler/types/options/compiler_options.go

@@ -47,6 +47,12 @@ type Compiler struct {
 	// TemplatesFolder. should default to tree/templates
 	TemplatesFolder []string
 
+	// Tells wether to push final container images after building
+	PushFinalImages      bool
+	PushFinalImagesForce bool
+	// Image repository to push to
+	PushFinalImagesRepository string
+
 	Context *types.Context
 }
 
@@ -85,6 +91,25 @@ func WithOptions(opt *Compiler) func(cfg *Compiler) error {
 	}
 }
 
+// WithFinalRepository Sets the final repository where to push
+// images of built artifacts
+func WithFinalRepository(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PushFinalImagesRepository = r
+		return nil
+	}
+}
+
+func EnablePushFinalImages(cfg *Compiler) error {
+	cfg.PushFinalImages = true
+	return nil
+}
+
+func ForcePushFinalImages(cfg *Compiler) error {
+	cfg.PushFinalImagesForce = true
+	return nil
+}
+
 func WithBackendType(r string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		cfg.BackendType = r
@@ -113,6 +138,8 @@ func WithPullRepositories(r []string) func(cfg *Compiler) error {
 	}
 }
 
+// WithPushRepository Sets the image reference where to push
+// cache images
 func WithPushRepository(r string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		if len(cfg.PullImageRepository) == 0 {

pkg/helpers/docker/docker.go

@@ -174,7 +174,6 @@ func DownloadAndExtractDockerImage(ctx *luettypes.Context, image, dest string, a
 		ctx,
 		img,
 		dest,
-		true,
 		nil,
 	)
 	if err != nil {

pkg/installer/client/http.go

@@ -118,11 +118,7 @@ func (c *HttpClient) DownloadFile(p string) (string, error) {
 		// Initialize a progressbar only if we have one in the current context
 		var pb *pterm.ProgressbarPrinter
 		if c.context.ProgressBar != nil {
-			pb = pterm.DefaultProgressbar.WithTotal(int(resp.Size()))
-			if c.context.AreaPrinter != nil {
-				pb = pb.WithPrintTogether(c.context.AreaPrinter)
-			}
-			pb, _ = pb.WithTitle(filepath.Base(resp.Request.HTTPRequest.URL.RequestURI())).Start()
+			pb, _ = c.context.ProgressBar.WithTotal(int(resp.Size())).WithTitle(filepath.Base(resp.Request.HTTPRequest.URL.RequestURI())).Start()
 		}
 		// start download loop
 		t := time.NewTicker(500 * time.Millisecond)

pkg/installer/installer.go

@@ -576,19 +576,19 @@ func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string
 	// Check if the terminal is big enough to display a progress bar
 	// https://github.com/pterm/pterm/blob/4c725e56bfd9eb38e1c7b9dec187b50b93baa8bd/progressbar_printer.go#L190
 	w, _, err := ctx.GetTerminalSize()
+
+	var pb *pterm.ProgressbarPrinter
 	if ctx.IsTerminal && err == nil && w > 100 {
 		area, _ := pterm.DefaultArea.Start()
-		p, _ := pterm.DefaultProgressbar.WithPrintTogether(area).WithTotal(len(toDownload)).WithTitle("Downloading packages").Start()
-
-		ctx.AreaPrinter = area
-		ctx.ProgressBar = p
+		ctx.ProgressBar = pterm.DefaultProgressbar.WithPrintTogether(area).WithTotal(len(toDownload)).WithTitle("Downloading packages")
+		pb, _ = ctx.ProgressBar.Start()
 		defer area.Stop()
 	}
 
 	// Download
 	for i := 0; i < l.Options.Concurrency; i++ {
 		wg.Add(1)
-		go l.downloadWorker(i, wg, all, ctx)
+		go l.downloadWorker(i, wg, pb, all, ctx)
 	}
 	for _, c := range toDownload {
 		all <- c
@@ -920,7 +920,7 @@ func (l *LuetInstaller) installPackage(m ArtifactMatch, s *System) error {
 	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: m.Package.GetFingerPrint(), Files: files})
 }
 
-func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, ctx *types.Context) error {
+func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, pb *pterm.ProgressbarPrinter, c <-chan ArtifactMatch, ctx *types.Context) error {
 	defer wg.Done()
 
 	for p := range c {
@@ -932,8 +932,8 @@ func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan Artif
 		} else {
 			l.Options.Context.Success(":package: Package ", p.Package.HumanReadableString(), "downloaded")
 		}
-		if ctx.ProgressBar != nil {
-			ctx.ProgressBar.Increment()
+		if pb != nil {
+			pb.Increment()
 		}
 	}
 

pkg/installer/repository_docker.go

@@ -204,7 +204,6 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 			d.context,
 			img,
 			repoTemp,
-			d.context.Config.GetGeneral().SameOwner,
 			nil,
 		)
 		if err != nil {

pkg/installer/system.go

@@ -1,6 +1,8 @@
 package installer
 
 import (
+	"os"
+	"path/filepath"
 	"sync"
 
 	"github.com/hashicorp/go-multierror"
@@ -22,6 +24,19 @@ func (s *System) World() (pkg.Packages, error) {
 	return s.Database.World(), nil
 }
 
+func (s *System) OSCheck() (notFound pkg.Packages) {
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	for f, p := range s.fileIndex {
+		if _, err := os.Lstat(filepath.Join(s.Target, f)); err != nil {
+			notFound = append(notFound, p)
+		}
+	}
+	notFound = notFound.Unique()
+	return
+}
+
 func (s *System) ExecuteFinalizers(ctx *types.Context, packs []pkg.Package) error {
 	var errs error
 	executedFinalizer := map[string]bool{}
@@ -56,6 +71,7 @@ func (s *System) ExecuteFinalizers(ctx *types.Context, packs []pkg.Package) erro
 }
 
 func (s *System) buildFileIndex() {
+	// XXX: Replace with cache
 	s.Lock()
 	defer s.Unlock()
 	// Check if cache is empty or if it got modified

pkg/installer/system_test.go

@@ -19,6 +19,10 @@ import (
 
 	//	. "github.com/mudler/luet/pkg/installer"
 
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -66,5 +70,18 @@ var _ = Describe("System", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p).To(Equal(b))
 		})
+
+		It("detect missing files", func() {
+			dir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(dir)
+			s.Target = dir
+			notfound := s.OSCheck()
+			Expect(len(notfound)).To(Equal(2))
+			ioutil.WriteFile(filepath.Join(dir, "f"), []byte{}, os.ModePerm)
+			ioutil.WriteFile(filepath.Join(dir, "foo"), []byte{}, os.ModePerm)
+			notfound = s.OSCheck()
+			Expect(len(notfound)).To(Equal(1))
+		})
 	})
 })

pkg/tree/compiler_recipe.go

@@ -174,6 +174,7 @@ func (r *CompilerRecipe) Load(path string) error {
 								filepath.Dir(currentpath))
 					}
 					pack.Requires(packbuild.GetRequires())
+
 					pack.Conflicts(packbuild.GetConflicts())
 				}
 

tests/fixtures/extra_perms/pkgA/0.1/build.yaml

@@ -0,0 +1,18 @@
+image: "alpine"
+unpack: true
+includes:
+  - /foo
+  - /foo/bar
+  - /foo/bar/suid
+  - /foo/bar/sticky
+  - /foo/bar/sgid
+steps:
+- mkdir -p /foo/bar
+- touch /foo/bar/suid
+- touch /foo/bar/sgid
+- touch /foo/bar/sticky
+- chown 100:100 /foo/bar
+- chown 101:101 /foo/bar/suid
+- chmod u+s /foo/bar/suid
+- chmod u-s,g+s /foo/bar/sgid
+- chmod +t /foo/bar/sticky

tests/fixtures/extra_perms/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "extra-perms"
+version: "0.1"

tests/fixtures/join_complex/c/c1/build.yaml

@@ -6,4 +6,8 @@ requires:
     category: "test"
     version: ">=0"
 
-requires_final_images: true
+requires_final_images: true
+
+steps:
+- | 
+   /bin/sh -c "if [ -e /usr/bin/generate.sh ]; then ls -liah /usr/bin && exit 1; fi"

tests/fixtures/join_complex/cat/b-1.1/build.yaml

@@ -6,8 +6,9 @@ requires:
 prelude:
   - echo foo > /test
   - echo bar > /test2
+  - cp -rf generate.sh /usr/bin/
 steps:
   - echo artifact5 > /newc
   - echo artifact6 > /newnewc
   - chmod +x generate.sh
-  - ./generate.sh
+  - ./generate.sh

tests/integration/36_extra_perm_bits.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testbuild
+    luet build -d --tree "$ROOT_DIR/tests/fixtures/extra_perms" --same-owner=true --destination $tmpdir/testbuild --compression gzip --full
+    buildst=$?
+    assertTrue 'create package perms 0.1' "[ -e '$tmpdir/testbuild/extra-perms-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/extra_perms" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    $ROOT_DIR/tests/integration/bin/luet install -y --config $tmpdir/luet.yaml test/extra-perms
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+   
+    tree $tmpdir/testrootfs/foo/bar
+    assertTrue 'package installed bar' "[ -d '$tmpdir/testrootfs/foo/bar' ]"
+
+    assertContains 'perms2' "$(stat -c %u:%g $tmpdir/testrootfs/foo/bar)" "100:100"
+    assertContains 'suid' "$(stat -c %a $tmpdir/testrootfs/foo/bar/suid)" "4644"
+    assertContains 'sgid' "$(stat -c %a $tmpdir/testrootfs/foo/bar/sgid)" "2644"
+    assertContains 'sticky' "$(stat -c %a $tmpdir/testrootfs/foo/bar/sticky)" "1644"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+