Tag 0.14.7

Add unit tests, consume imagehashtree in compiler and cleanup

Fixes: #203

cmd/build.go

@@ -247,11 +247,12 @@ Build packages specifying multiple definition trees:
 			}
 
 			for _, sp := range toCalculate {
-				packs, err := luetCompiler.ComputeDepTree(sp)
+				ht := compiler.NewHashTree(generalRecipe.GetDatabase())
+				hashTree, err := ht.Query(luetCompiler, sp)
 				if err != nil {
 					errs = append(errs, err)
 				}
-				for _, p := range packs {
+				for _, p := range hashTree.Dependencies {
 					results.Packages = append(results.Packages,
 						PackageResult{
 							Name:       p.Package.GetName(),

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.14.0"
+	LuetCLIVersion = "0.14.7"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -97,7 +97,7 @@ To build a package, from a tree definition:
 
 		plugin := viper.GetStringSlice("plugin")
 
-		bus.Manager.Load(plugin...).Register()
+		bus.Manager.Initialize(plugin...)
 		if len(bus.Manager.Plugins) != 0 {
 			Info(":lollipop:Enabled plugins:")
 			for _, p := range bus.Manager.Plugins {

go.mod

@@ -16,7 +16,6 @@ require (
 	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
-	github.com/fsouza/go-dockerclient v1.6.4
 	github.com/genuinetools/img v0.5.11
 	github.com/ghodss/yaml v1.0.0
 	github.com/google/go-containerregistry v0.2.1
@@ -37,7 +36,7 @@ require (
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+	github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3

go.sum

@@ -188,8 +188,6 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d h1:UKAt78F1OvM4ceTn1VvXuYuatXohsFU1eSI2IBtTw9g=
@@ -760,8 +758,8 @@ github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d h1:fKh+rvw
 github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d/go.mod h1:puRUWSwyecW2V355tKncwPVPRAjQBduPsFjG0mrV/Nw=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87 h1:mGz7T8KvmHH0gLWPI5tQne8xl2cO3T8wrrb6Aa16Jxo=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87/go.mod h1:1w4zI1LYXDeiUXqedPcrT5eQJnmKR6dbg5iJMgSIP/Y=
-github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82 h1:Hkefw2tzoKATVUTFsCtDlUnY180+OE851qGbq45ATxk=
-github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82/go.mod h1:4P/ULate+2QxoAQtojaRjyO5VGMhV0KLnSdAS8nuBbo=
+github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af h1:jixIxEgLSqu24eMiyzfCI+roa5IaOUhF546ePSFyHeY=
+github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290 h1:426hFyXMpXeqIeGJn2cGAW9ogvM2Jf+Jv23gtVPvBLM=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290/go.mod h1:uP5BBgFxq2wNWo7n1vnY5SSbgL0WDshVJrOO12tZ/lA=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=

pkg/bus/events.go

@@ -1,6 +1,8 @@
 package bus
 
 import (
+	. "github.com/mudler/luet/pkg/logger"
+
 	"github.com/mudler/go-pluggable"
 )
 
@@ -47,21 +49,47 @@ var (
 )
 
 // Manager is the bus instance manager, which subscribes plugins to events emitted by Luet
-var Manager *pluggable.Manager = pluggable.NewManager(
-	[]pluggable.EventType{
-		EventPackageInstall,
-		EventPackageUnInstall,
-		EventPackagePreBuild,
-		EventPackagePreBuildArtifact,
-		EventPackagePostBuildArtifact,
-		EventPackagePostBuild,
-		EventRepositoryPreBuild,
-		EventRepositoryPostBuild,
-		EventImagePreBuild,
-		EventImagePrePull,
-		EventImagePrePush,
-		EventImagePostBuild,
-		EventImagePostPull,
-		EventImagePostPush,
-	},
-)
+var Manager *Bus = &Bus{
+	Manager: pluggable.NewManager(
+		[]pluggable.EventType{
+			EventPackageInstall,
+			EventPackageUnInstall,
+			EventPackagePreBuild,
+			EventPackagePreBuildArtifact,
+			EventPackagePostBuildArtifact,
+			EventPackagePostBuild,
+			EventRepositoryPreBuild,
+			EventRepositoryPostBuild,
+			EventImagePreBuild,
+			EventImagePrePull,
+			EventImagePrePush,
+			EventImagePostBuild,
+			EventImagePostPull,
+			EventImagePostPush,
+		},
+	),
+}
+
+type Bus struct {
+	*pluggable.Manager
+}
+
+func (b *Bus) Initialize(plugin ...string) {
+	b.Manager.Load(plugin...).Register()
+
+	for _, e := range b.Manager.Events {
+		b.Manager.Response(e, func(p *pluggable.Plugin, r *pluggable.EventResponse) {
+			if r.Errored() {
+				Fatal("Plugin", p.Name, "at", p.Executable, "Error", r.Error)
+			}
+			Debug(
+				"plugin_event",
+				"received from",
+				p.Name,
+				"at",
+				p.Executable,
+				r,
+			)
+		})
+	}
+}

pkg/compiler/backend/simpledocker.go

@@ -25,7 +25,6 @@ import (
 
 	bus "github.com/mudler/luet/pkg/bus"
 
-	docker "github.com/fsouza/go-dockerclient"
 	capi "github.com/mudler/docker-companion/api"
 
 	"github.com/mudler/luet/pkg/helpers"
@@ -56,24 +55,6 @@ func (*SimpleDocker) BuildImage(opts Options) error {
 
 	Info(":whale: Building image " + name + " done")
 
-	if os.Getenv("DOCKER_SQUASH") == "true" {
-		Info(":whale: Squashing image " + name)
-		var client *docker.Client
-
-		Spinner(22)
-		defer SpinnerStop()
-		client, err = docker.NewClientFromEnv()
-		if err != nil {
-			return errors.Wrap(err, "could not connect to the Docker daemon")
-		}
-		err = capi.Squash(client, name, name)
-		if err != nil {
-			return errors.Wrap(err, "Failed squashing image")
-		}
-
-		Info(":whale: Squashing image " + name + " done")
-	}
-
 	bus.Manager.Publish(bus.EventImagePostBuild, opts)
 
 	return nil

pkg/compiler/compiler.go

@@ -146,11 +146,6 @@ func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps *compilerspec.L
 	}
 
 	for _, p := range ps.All() {
-		asserts, err := cs.ComputeDepTree(p)
-		if err != nil {
-			panic(err)
-		}
-		p.SetSourceAssertion(asserts)
 		all <- p
 	}
 
@@ -295,17 +290,6 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co
 	return artifact, nil
 }
 
-func (cs *LuetCompiler) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
-	// Use packageImage as salt into the fp being used
-	// so the hash is unique also in cases where
-	// some package deps does have completely different
-	// depgraphs
-	// TODO: We should use the image tag, or pass by the package assertion hash which is unique
-	// and identifies the deptree of the package.
-	return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(helpers.StripRegistryFromImage(packageImage)))
-
-}
-
 func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
 	concurrency int, keepPermissions bool,
 	p *compilerspec.LuetCompilationSpec) (backend.Options, backend.Options, error) {
@@ -680,33 +664,7 @@ func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst s
 	}
 }
 
-// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
-func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
-	// Generate a set with all the deps of the provided specs
-	// we will use that set to remove the deps from the list of provided compilation specs
-	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
-	result := []*compilerspec.LuetCompilationSpec{}
-	for _, spec := range p {
-		ass, err := cs.ComputeDepTree(spec)
-		if err != nil {
-			return result, errors.Wrap(err, "computin specs deptree")
-		}
-
-		allDependencies = append(allDependencies, ass.Drop(spec.GetPackage())...)
-	}
-
-	for _, spec := range p {
-		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
-			result = append(result, spec)
-		}
-	}
-	return result, nil
-}
-
-// ComputeDepTree computes the dependency tree of a compilation spec and returns solver assertions
-// in order to be able to compile the spec.
 func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
-
 	s := solver.NewResolver(cs.Options.SolverOptions.Options, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
 
 	solution, err := s.Install(pkg.Packages{p.GetPackage()})
@@ -718,31 +676,34 @@ func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (sol
 	if err != nil {
 		return nil, errors.Wrap(err, "While order a solution for "+p.GetPackage().HumanReadableString())
 	}
+	return dependencies, nil
+}
 
-	assertions := solver.PackagesAssertions{}
-	for _, assertion := range dependencies { //highly dependent on the order
-		if assertion.Value {
-			nthsolution := dependencies.Cut(assertion.Package)
-			assertion.Hash = solver.PackageHash{
-				BuildHash:   nthsolution.HashFrom(assertion.Package),
-				PackageHash: nthsolution.AssertionHash(),
-			}
-			assertion.Package.SetTreeDir(p.Package.GetTreeDir())
-			assertions = append(assertions, assertion)
+// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
+func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
+	// Generate a set with all the deps of the provided specs
+	// we will use that set to remove the deps from the list of provided compilation specs
+	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
+	result := []*compilerspec.LuetCompilationSpec{}
+	for _, spec := range p {
+		sol, err := cs.ComputeDepTree(spec)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed querying hashtree")
+		}
+		allDependencies = append(allDependencies, sol.Drop(spec.GetPackage())...)
+	}
+
+	for _, spec := range p {
+		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
+			result = append(result, spec)
 		}
 	}
-	p.SetSourceAssertion(assertions)
-	return assertions, nil
+	return result, nil
 }
 
 // Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
 // an artifact
 func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	asserts, err := cs.ComputeDepTree(p)
-	if err != nil {
-		return nil, err
-	}
-	p.SetSourceAssertion(asserts)
 	return cs.compile(cs.Options.Concurrency, keepPermissions, p)
 }
 
@@ -773,11 +734,6 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS
 }
 
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	// TODO: Racy, remove it
-	// Inherit build options from compilation specs metadata
-	// orig := cs.Options.PullImageRepository
-	// defer func() { cs.Options.PullImageRepository = orig }()
-
 	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
 	Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
@@ -789,36 +745,42 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 			)
 	}
 
-	targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
+	ht := NewHashTree(cs.Database)
+
+	packageHashTree, err := ht.Query(cs, p)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed querying hashtree")
+	}
+
+	// This is in order to have the metadata in the yaml
+	p.SetSourceAssertion(packageHashTree.Solution)
+	targetAssertion := packageHashTree.Target
 
 	bus.Manager.Publish(bus.EventPackagePreBuild, struct {
-		CompileSpec *compilerspec.LuetCompilationSpec
-		Assert      solver.PackageAssert
+		CompileSpec     *compilerspec.LuetCompilationSpec
+		Assert          solver.PackageAssert
+		PackageHashTree *PackageImageHashTree
 	}{
-		CompileSpec: p,
-		Assert:      *targetAssertion,
+		CompileSpec:     p,
+		Assert:          *targetAssertion,
+		PackageHashTree: packageHashTree,
 	})
 
 	// Update compilespec build options - it will be then serialized into the compilation metadata file
-	//p.SetBuildOptions(cs.Options)
 	p.BuildOptions.PushImageRepository = cs.Options.PushImageRepository
-	//p.BuildOptions.BuildValues = cs.Options.BuildValues
-	//p.BuildOptions.BuildValuesFile = cs.Options.BuildValuesFile
 
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		return cs.compileWithImage(p.GetImage(), cs.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
+		return cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
 	// on how we compute the resolvable tree.
 	// This means to recursively build all the build-images needed to reach that tree part.
 	// - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image.
-
-	dependencies := p.GetSourceAssertion().Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
-	departifacts := []*artifact.PackageArtifact{}               // TODO: Return this somehow
-	var lastHash string
+	dependencies := packageHashTree.Dependencies  // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
+	departifacts := []*artifact.PackageArtifact{} // TODO: Return this somehow
 	depsN := 0
 	currentN := 0
 
@@ -845,8 +807,6 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 			Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
 
 			compileSpec.SetOutputPath(p.GetOutputPath())
-			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash)
-			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
 
 			bus.Manager.Publish(bus.EventPackagePreBuild, struct {
 				CompileSpec *compilerspec.LuetCompilationSpec
@@ -856,29 +816,43 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 				Assert:      assertion,
 			})
 
-			lastHash = assertion.Hash.PackageHash
-			// for the source instead, pick an image and a buildertaggedImage from hashes if they exists.
-			// otherways fallback to the pushed repo
-			// Resolve images from the hashtree
-			resolvedBuildImage := cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec)
-			if compileSpec.GetImage() != "" {
-				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
-
-				a, err := cs.compileWithImage(compileSpec.GetImage(), assertion.Hash.BuildHash, assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps)
-				if err != nil {
-					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
-				}
-				departifacts = append(departifacts, a)
-				Info(pkgTag, ":white_check_mark: Done")
-				continue
+			buildHash, err := packageHashTree.DependencyBuildImage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed looking for dependency in hashtree")
 			}
 
-			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
-			a, err := cs.compileWithImage(resolvedBuildImage, cs.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash), assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps)
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash)
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
+
+			var sourceImage string
+
+			if compileSpec.GetImage() != "" {
+				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
+				sourceImage = compileSpec.GetImage()
+			} else {
+				// for the source instead, pick an image and a buildertaggedImage from hashes if they exists.
+				// otherways fallback to the pushed repo
+				// Resolve images from the hashtree
+				sourceImage = cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec)
+				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
+			}
+
+			a, err := cs.compileWithImage(
+				sourceImage,
+				buildHash,
+				assertion.Hash.PackageHash,
+				concurrency,
+				keepPermissions,
+				cs.Options.KeepImg,
+				compileSpec,
+				packageDeps,
+			)
 			if err != nil {
 				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 			}
 
+			Info(pkgTag, ":white_check_mark: Done")
+
 			bus.Manager.Publish(bus.EventPackagePostBuild, struct {
 				CompileSpec *compilerspec.LuetCompilationSpec
 				Artifact    *artifact.PackageArtifact
@@ -888,18 +862,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 			})
 
 			departifacts = append(departifacts, a)
-			Info(pkgTag, ":white_check_mark: Done")
 		}
-
-	} else if len(dependencies) > 0 {
-		lastHash = dependencies[len(dependencies)-1].Hash.PackageHash
 	}
 
 	if buildTarget {
-		resolvedBuildImage := cs.resolveExistingImageHash(lastHash, p)
+		resolvedSourceImage := cs.resolveExistingImageHash(packageHashTree.SourceHash, p)
 		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
-		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedBuildImage)
-		a, err := cs.compileWithImage(resolvedBuildImage, cs.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
+		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedSourceImage)
+		a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
 		if err != nil {
 			return a, err
 		}

pkg/compiler/imagehashtree.go

@@ -0,0 +1,126 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"fmt"
+
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	"github.com/mudler/luet/pkg/config"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	"github.com/pkg/errors"
+)
+
+type ImageHashTree struct {
+	Database      pkg.PackageDatabase
+	SolverOptions config.LuetSolverOptions
+}
+
+type PackageImageHashTree struct {
+	Target                       *solver.PackageAssert
+	Dependencies                 solver.PackagesAssertions
+	Solution                     solver.PackagesAssertions
+	dependencyBuilderImageHashes map[string]string
+	SourceHash                   string
+	BuilderImageHash             string
+}
+
+func NewHashTree(db pkg.PackageDatabase) *ImageHashTree {
+	return &ImageHashTree{
+		Database: db,
+	}
+}
+
+func (ht *PackageImageHashTree) DependencyBuildImage(p pkg.Package) (string, error) {
+	found, ok := ht.dependencyBuilderImageHashes[p.GetFingerPrint()]
+	if !ok {
+		return "", errors.New("package hash not found")
+	}
+	return found, nil
+}
+
+// TODO: ___ When computing the hash per package (and evaluating the sat solver solution tree part)
+// we should use the hash of each package  + its fingerprint  instead as a salt.
+// That's because the hash will be salted with its `build.yaml`.
+// In this way, we trigger recompilations if some dep of a target changes
+// a build.yaml, without touching the version
+func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) {
+	assertions, err := ht.resolve(cs, p)
+	if err != nil {
+		return nil, err
+	}
+	targetAssertion := assertions.Search(p.GetPackage().GetFingerPrint())
+
+	dependencies := assertions.Drop(p.GetPackage())
+	var sourceHash string
+	imageHashes := map[string]string{}
+	for _, assertion := range dependencies {
+		var depbuildImageTag string
+		compileSpec, err := cs.FromPackage(assertion.Package)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+		}
+		if compileSpec.GetImage() != "" {
+			depbuildImageTag = assertion.Hash.BuildHash
+		} else {
+			depbuildImageTag = ht.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash)
+		}
+		imageHashes[assertion.Package.GetFingerPrint()] = depbuildImageTag
+		sourceHash = assertion.Hash.PackageHash
+	}
+
+	return &PackageImageHashTree{
+		Dependencies:                 dependencies,
+		Target:                       targetAssertion,
+		SourceHash:                   sourceHash,
+		BuilderImageHash:             ht.genBuilderImageTag(p, targetAssertion.Hash.PackageHash),
+		dependencyBuilderImageHashes: imageHashes,
+		Solution:                     assertions,
+	}, nil
+}
+
+func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(packageImage))
+}
+
+// resolve computes the dependency tree of a compilation spec and returns solver assertions
+// in order to be able to compile the spec.
+func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
+	dependencies, err := cs.ComputeDepTree(p)
+	if err != nil {
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
+	}
+
+	assertions := solver.PackagesAssertions{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			nthsolution := dependencies.Cut(assertion.Package)
+			assertion.Hash = solver.PackageHash{
+				BuildHash:   nthsolution.HashFrom(assertion.Package),
+				PackageHash: nthsolution.AssertionHash(),
+			}
+			assertion.Package.SetTreeDir(p.Package.GetTreeDir())
+			assertions = append(assertions, assertion)
+		}
+	}
+
+	return assertions, nil
+}

pkg/compiler/imagehashtree_test.go

@@ -0,0 +1,94 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler_test
+
+import (
+	. "github.com/mudler/luet/pkg/compiler"
+	sd "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ImageHashTree", func() {
+	generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+	compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+	hashtree := NewHashTree(generalRecipe.GetDatabase())
+	Context("Simple package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			err := generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+
+		It("Calculates the hash correctly", func() {
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(packageHash.Target.Hash.BuildHash).To(Equal("6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-79462b60bf899ad79db63f194a3c9c2a"))
+		})
+	})
+
+	Context("complex package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
+			Expect(packageHash.SourceHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-37f4d05ba8a39525742ca364f69b4090"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("bb1d9a99c0c309a297c75b436504e664a42121fadbb4e035bda403cd418117aa"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+			Expect(assertionA.Hash.PackageHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+			Expect(assertionB.Hash.PackageHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hashB).To(Equal("6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"))
+		})
+	})
+
+})

pkg/compiler/types/artifact/artifact.go

@@ -19,6 +19,8 @@ import (
 	"archive/tar"
 	"bufio"
 	"bytes"
+	"crypto/sha1"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -27,7 +29,6 @@ import (
 	"path/filepath"
 	"regexp"
 
-	system "github.com/docker/docker/pkg/system"
 	zstd "github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 
@@ -317,7 +318,6 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	default:
 		return helpers.Tar(src, a.getCompressedName())
 	}
-	return errors.New("Compression type must be supplied")
 }
 
 func (a *PackageArtifact) getCompressedName() string {
@@ -340,6 +340,28 @@ func (a *PackageArtifact) GetUncompressedName() string {
 	return a.Path
 }
 
+func hashContent(bv []byte) string {
+	hasher := sha1.New()
+	hasher.Write(bv)
+	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	return sha
+}
+
+func hashFileContent(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	h := sha1.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+
+	return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil
+}
+
 func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
 	// If the destination path already exists I rename target file name with postfix.
 	var destPath string
@@ -351,6 +373,7 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return nil, nil, err
 		}
 	}
+	tarHash := hashContent(buffer.Bytes())
 
 	// If file is not present on archive but is defined on mods
 	// I receive the callback. Prevent nil exception.
@@ -363,8 +386,21 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return header, buffer.Bytes(), nil
 		}
 
+		existingHash := ""
+		f, err := os.Lstat(destPath)
+		if err == nil {
+			Debug("File exists already, computing hash for", destPath)
+			hash, herr := hashFileContent(destPath)
+			if herr == nil {
+				existingHash = hash
+			}
+		}
+
+		Debug("Existing file hash: ", existingHash, "Tar file hashsum: ", tarHash)
+		// We want to protect file only if the hash of the files are differing OR the file size are
+		differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
 		// Check if exists
-		if helpers.Exists(destPath) {
+		if helpers.Exists(destPath) && differs {
 			for i := 1; i < 1000; i++ {
 				name := filepath.Join(filepath.Join(filepath.Dir(path),
 					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
@@ -584,47 +620,16 @@ type CopyJob struct {
 	Artifact string
 }
 
-func copyXattr(srcPath, dstPath, attr string) error {
-	data, err := system.Lgetxattr(srcPath, attr)
-	if err != nil {
-		return err
-	}
-	if data != nil {
-		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func doCopyXattrs(srcPath, dstPath string) error {
-	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
-		return err
-	}
-
-	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
-}
-
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
 	for job := range s {
-		//Info("#"+strconv.Itoa(i), "copying", job.Src, "to", job.Dst)
-		// if dir, err := helpers.IsDirectory(job.Src); err == nil && dir {
-		// 	err = helpers.CopyDir(job.Src, job.Dst)
-		// 	if err != nil {
-		// 		Warning("Error copying dir", job, err)
-		// 	}
-		// 	continue
-		// }
-
 		_, err := os.Lstat(job.Dst)
 		if err != nil {
 			Debug("Copying ", job.Src)
-			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
+			if err := helpers.DeepCopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
-			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }

pkg/compiler/types/spec/spec.go

@@ -16,6 +16,7 @@
 package compilerspec
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
 
@@ -85,6 +86,13 @@ func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
 	return &newSpecs
 }
 
+type CopyField struct {
+	Package     *pkg.DefaultPackage `json:"package"`
+	Image       string              `json:"image"`
+	Source      string              `json:"src"`
+	Destination string              `json:"dst"`
+}
+
 type LuetCompilationSpec struct {
 	Steps           []string                  `json:"steps"` // Are run inside a container and the result layer diff is saved
 	Env             []string                  `json:"env"`
@@ -103,6 +111,8 @@ type LuetCompilationSpec struct {
 	Excludes   []string `json:"excludes"`
 
 	BuildOptions *options.Compiler `json:"build_options"`
+
+	Copy []CopyField `json:"copy"`
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
@@ -256,6 +266,12 @@ ADD ` + s + ` /luetbuild/`
 		}
 	}
 
+	for _, c := range cs.Copy {
+		if c.Image != "" {
+			spec = spec + fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
+		}
+	}
+
 	for _, s := range cs.Env {
 		spec = spec + `
 ENV ` + s

pkg/helpers/archive.go

@@ -90,8 +90,7 @@ func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modi
 	}
 
 	if sameOwner {
-		// PRE: i have root privileged.
-
+		// we do have root permissions, so we can extract keeping the same permissions.
 		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
 
 		opts := &archive.TarOptions{

pkg/helpers/file.go

@@ -27,6 +27,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/docker/docker/pkg/system"
 	"github.com/google/renameio"
 	copy "github.com/otiai10/copy"
 	"github.com/pkg/errors"
@@ -167,9 +168,27 @@ func Read(file string) (string, error) {
 	return string(dat), nil
 }
 
+func EnsureDirPerm(src, dst string) {
+	if info, err := os.Lstat(filepath.Dir(src)); err == nil {
+		if _, err := os.Lstat(filepath.Dir(dst)); os.IsNotExist(err) {
+			err := os.MkdirAll(filepath.Dir(dst), info.Mode().Perm())
+			if err != nil {
+				fmt.Println("warning: failed creating", filepath.Dir(dst), err.Error())
+			}
+			if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+				if err := os.Lchown(filepath.Dir(dst), int(stat.Uid), int(stat.Gid)); err != nil {
+					fmt.Println("warning: failed chowning", filepath.Dir(dst), err.Error())
+				}
+			}
+		}
+	} else {
+		EnsureDir(dst)
+	}
+}
+
 func EnsureDir(fileName string) error {
 	dirName := filepath.Dir(fileName)
-	if _, serr := os.Stat(dirName); serr != nil {
+	if _, serr := os.Stat(dirName); os.IsNotExist(serr) {
 		merr := os.MkdirAll(dirName, os.ModePerm) // FIXME: It should preserve permissions from src to dst instead
 		if merr != nil {
 			return merr
@@ -178,12 +197,39 @@ func EnsureDir(fileName string) error {
 	return nil
 }
 
-// CopyFile copies the contents of the file named src to the file named
+func CopyFile(src, dst string) (err error) {
+	return copy.Copy(src, dst, copy.Options{
+		Sync:      true,
+		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
+}
+
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
+// DeepCopyFile copies the contents of the file named src to the file named
 // by dst. The file will be created if it does not already exist. If the
 // destination file exists, all it's contents will be replaced by the contents
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
-func CopyFile(src, dst string) (err error) {
+func DeepCopyFile(src, dst string) (err error) {
 	// Workaround for https://github.com/otiai10/copy/issues/47
 	fi, err := os.Lstat(src)
 	if err != nil {
@@ -193,7 +239,7 @@ func CopyFile(src, dst string) (err error) {
 	fm := fi.Mode()
 	switch {
 	case fm&os.ModeNamedPipe != 0:
-		EnsureDir(dst)
+		EnsureDirPerm(src, dst)
 		if err := syscall.Mkfifo(dst, uint32(fi.Mode())); err != nil {
 			return errors.Wrap(err, "failed creating pipe")
 		}
@@ -205,6 +251,9 @@ func CopyFile(src, dst string) (err error) {
 		return nil
 	}
 
+	//filepath.Dir(src)
+	EnsureDirPerm(src, dst)
+
 	err = copy.Copy(src, dst, copy.Options{
 		Sync:      true,
 		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
@@ -216,7 +265,8 @@ func CopyFile(src, dst string) (err error) {
 			fmt.Println("warning: failed chowning", dst, err.Error())
 		}
 	}
-	return err
+
+	return doCopyXattrs(src, dst)
 }
 
 func IsDirectory(path string) (bool, error) {

pkg/helpers/match.go

@@ -17,9 +17,18 @@
 package helpers
 
 import (
+	"reflect"
 	"regexp"
 )
 
+func ReverseAny(s interface{}) {
+	n := reflect.ValueOf(s).Len()
+	swap := reflect.Swapper(s)
+	for i, j := 0, n-1; i < j; i, j = i+1, j-1 {
+		swap(i, j)
+	}
+}
+
 func MapMatchRegex(m *map[string]string, r *regexp.Regexp) bool {
 	ans := false
 

pkg/installer/installer.go

@@ -128,6 +128,8 @@ func packsToList(p pkg.Packages) string {
 	for _, pp := range p {
 		packs = append(packs, pp.HumanReadableString())
 	}
+
+	sort.Strings(packs)
 	return strings.Join(packs, " ")
 }
 
@@ -137,6 +139,7 @@ func matchesToList(artefacts map[string]ArtifactMatch) string {
 	for fingerprint, match := range artefacts {
 		packs = append(packs, fmt.Sprintf("%s (%s)", fingerprint, match.Repository.GetName()))
 	}
+	sort.Strings(packs)
 	return strings.Join(packs, " ")
 }
 
@@ -194,11 +197,19 @@ func (l *LuetInstaller) Swap(toRemove pkg.Packages, toInstall pkg.Packages, s *S
 			toRemoveFinal = append(toRemoveFinal, pp)
 		}
 	}
+	o := Option{
+		FullUninstall:      false,
+		Force:              true,
+		CheckConflicts:     false,
+		FullCleanUninstall: false,
+		NoDeps:             l.Options.NoDeps,
+		OnlyDeps:           false,
+	}
 
-	return l.swap(syncedRepos, toRemoveFinal, toInstall, s, false)
+	return l.swap(o, syncedRepos, toRemoveFinal, toInstall, s)
 }
 
-func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
+func (l *LuetInstaller) computeSwap(o Option, syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
 
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
@@ -213,8 +224,8 @@ func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packa
 
 	systemAfterChanges := &System{Database: installedtmp}
 
-	packs, err := l.computeUninstall(systemAfterChanges, toRemove...)
-	if err != nil && !l.Options.Force {
+	packs, err := l.computeUninstall(o, systemAfterChanges, toRemove...)
+	if err != nil && !o.Force {
 		Error("Failed computing uninstall for ", packsToList(toRemove))
 		return nil, nil, nil, nil, errors.Wrap(err, "computing uninstall "+packsToList(toRemove))
 	}
@@ -225,30 +236,16 @@ func (l *LuetInstaller) computeSwap(syncedRepos Repositories, toRemove pkg.Packa
 		}
 	}
 
-	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, toInstall, systemAfterChanges)
+	match, packages, assertions, allRepos, err := l.computeInstall(o, syncedRepos, toInstall, systemAfterChanges)
 	for _, p := range toInstall {
 		assertions = append(assertions, solver.PackageAssert{Package: p.(*pkg.DefaultPackage), Value: true})
 	}
 	return match, packages, assertions, allRepos, err
 }
 
-func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System, forceNodeps bool) error {
-	forced := l.Options.Force
-	nodeps := l.Options.NoDeps
+func (l *LuetInstaller) swap(o Option, syncedRepos Repositories, toRemove pkg.Packages, toInstall pkg.Packages, s *System) error {
 
-	// We don't want any conflict with the installed to raise during the upgrade.
-	// In this way we both force uninstalls and we avoid to check with conflicts
-	// against the current system state which is pending to deletion
-	// E.g. you can't check for conflicts for an upgrade of a new version of A
-	// if the old A results installed in the system. This is due to the fact that
-	// now the solver enforces the constraints and explictly denies two packages
-	// of the same version installed.
-	l.Options.Force = true
-	if forceNodeps {
-		l.Options.NoDeps = true
-	}
-
-	match, packages, assertions, allRepos, err := l.computeSwap(syncedRepos, toRemove, toInstall, s)
+	match, packages, assertions, allRepos, err := l.computeSwap(o, syncedRepos, toRemove, toInstall, s)
 	if err != nil {
 		return errors.Wrap(err, "failed computing package replacement")
 	}
@@ -278,15 +275,173 @@ func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, to
 		return nil
 	}
 
-	err = l.Uninstall(s, toRemove...)
-	if err != nil && !l.Options.Force {
-		Error("Failed uninstall for ", packsToList(toRemove))
-		return errors.Wrap(err, "uninstalling "+packsToList(toRemove))
+	ops := l.getOpsWithOptions(toRemove, match, Option{
+		Force:         o.Force,
+		NoDeps:        false,
+		OnlyDeps:      o.OnlyDeps,
+		RunFinalizers: false,
+	}, o, syncedRepos, packages, assertions, allRepos)
+
+	err = l.runOps(ops, s)
+	if err != nil {
+		return errors.Wrap(err, "failed running installer options")
 	}
 
-	l.Options.Force = forced
-	l.Options.NoDeps = nodeps
-	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+	toFinalize, err := l.getFinalizers(allRepos, assertions, match, o.NoDeps)
+	if err != nil {
+		return errors.Wrap(err, "failed getting package to finalize")
+	}
+
+	return s.ExecuteFinalizers(toFinalize)
+}
+
+type Option struct {
+	Force              bool
+	NoDeps             bool
+	CheckConflicts     bool
+	FullUninstall      bool
+	FullCleanUninstall bool
+	OnlyDeps           bool
+	RunFinalizers      bool
+}
+
+type operation struct {
+	Option  Option
+	Package pkg.Package
+}
+
+type installOperation struct {
+	operation
+	Reposiories Repositories
+	Packages    pkg.Packages
+	Assertions  solver.PackagesAssertions
+	Database    pkg.PackageDatabase
+	Matches     map[string]ArtifactMatch
+}
+
+// installerOp is the operation that is sent to the
+// upgradeWorker's channel (todo)
+type installerOp struct {
+	Uninstall operation
+	Install   installOperation
+}
+
+func (l *LuetInstaller) runOps(ops []installerOp, s *System) error {
+	all := make(chan installerOp)
+
+	wg := new(sync.WaitGroup)
+
+	// Do the real install
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.installerOpWorker(i, wg, all, s)
+	}
+
+	for _, c := range ops {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
+
+	return nil
+}
+
+// TODO: use installerOpWorker in place of all the other workers.
+// This one is general enough to read a list of operations and execute them.
+func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, c <-chan installerOp, s *System) error {
+	defer wg.Done()
+
+	for p := range c {
+		if p.Uninstall.Package != nil {
+			Debug("Replacing package inplace")
+			toUninstall, uninstall, err := l.generateUninstallFn(p.Uninstall.Option, s, p.Uninstall.Package)
+			if err != nil {
+				Error("Failed to generate Uninstall function for" + err.Error())
+				continue
+				//return errors.Wrap(err, "while computing uninstall")
+			}
+
+			err = uninstall()
+			if err != nil {
+				Error("Failed uninstall for ", packsToList(toUninstall))
+				continue
+				//return errors.Wrap(err, "uninstalling "+packsToList(toUninstall))
+			}
+		}
+		if p.Install.Package != nil {
+			artMatch := p.Install.Matches[p.Install.Package.GetFingerPrint()]
+			ass := p.Install.Assertions.Search(p.Install.Package.GetFingerPrint())
+			packageToInstall, _ := p.Install.Packages.Find(p.Install.Package.GetPackageName())
+
+			err := l.install(
+				p.Install.Option,
+				p.Install.Reposiories,
+				map[string]ArtifactMatch{p.Install.Package.GetFingerPrint(): artMatch},
+				pkg.Packages{packageToInstall},
+				solver.PackagesAssertions{*ass},
+				p.Install.Database,
+				s,
+			)
+			if err != nil {
+				Error(err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// checks wheter we can uninstall and install in place and compose installer worker ops
+func (l *LuetInstaller) getOpsWithOptions(
+	toUninstall pkg.Packages, installMatch map[string]ArtifactMatch, installOpt, uninstallOpt Option,
+	syncedRepos Repositories, toInstall pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase) []installerOp {
+	resOps := []installerOp{}
+	for _, match := range installMatch {
+		if pack, err := toUninstall.Find(match.Package.GetPackageName()); err == nil {
+			resOps = append(resOps, installerOp{
+				Uninstall: operation{Package: pack, Option: uninstallOpt},
+				Install: installOperation{
+					operation: operation{
+						Package: match.Package,
+						Option:  installOpt,
+					},
+					Matches:     installMatch,
+					Packages:    toInstall,
+					Reposiories: syncedRepos,
+					Assertions:  solution,
+					Database:    allRepos,
+				},
+			})
+		} else {
+			resOps = append(resOps, installerOp{
+				Install: installOperation{
+					operation:   operation{Package: match.Package, Option: installOpt},
+					Matches:     installMatch,
+					Reposiories: syncedRepos,
+					Packages:    toInstall,
+					Assertions:  solution,
+					Database:    allRepos,
+				},
+			})
+		}
+	}
+
+	for _, p := range toUninstall {
+		found := false
+
+		for _, match := range installMatch {
+			if match.Package.GetPackageName() == p.GetPackageName() {
+				found = true
+			}
+
+		}
+		if !found {
+			resOps = append(resOps, installerOp{
+				Uninstall: operation{Package: p, Option: uninstallOpt},
+			})
+		}
+	}
+	return resOps
 }
 
 func (l *LuetInstaller) checkAndUpgrade(r Repositories, s *System) error {
@@ -310,19 +465,33 @@ func (l *LuetInstaller) checkAndUpgrade(r Repositories, s *System) error {
 		return nil
 	}
 
+	// We don't want any conflict with the installed to raise during the upgrade.
+	// In this way we both force uninstalls and we avoid to check with conflicts
+	// against the current system state which is pending to deletion
+	// E.g. you can't check for conflicts for an upgrade of a new version of A
+	// if the old A results installed in the system. This is due to the fact that
+	// now the solver enforces the constraints and explictly denies two packages
+	// of the same version installed.
+	o := Option{
+		FullUninstall:      false,
+		Force:              true,
+		CheckConflicts:     false,
+		FullCleanUninstall: false,
+		NoDeps:             true,
+		OnlyDeps:           false,
+	}
+
 	if l.Options.Ask {
 		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
 		if Ask() {
 			l.Options.Ask = false // Don't prompt anymore
-			return l.swap(r, uninstall, toInstall, s, true)
+			return l.swap(o, r, uninstall, toInstall, s)
 		} else {
 			return errors.New("Aborted by user")
 		}
 	}
 
-	Spinner(32)
-	defer SpinnerStop()
-	return l.swap(r, uninstall, toInstall, s, true)
+	return l.swap(o, r, uninstall, toInstall, s)
 }
 
 func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
@@ -338,7 +507,13 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 		}
 	}
 
-	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, cp, s)
+	o := Option{
+		NoDeps:        l.Options.NoDeps,
+		Force:         l.Options.Force,
+		OnlyDeps:      l.Options.OnlyDeps,
+		RunFinalizers: true,
+	}
+	match, packages, assertions, allRepos, err := l.computeInstall(o, syncedRepos, cp, s)
 	if err != nil {
 		return err
 	}
@@ -375,12 +550,12 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 		Info("By going forward, you are also accepting the licenses of the packages that you are going to install in your system.")
 		if Ask() {
 			l.Options.Ask = false // Don't prompt anymore
-			return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+			return l.install(o, syncedRepos, match, packages, assertions, allRepos, s)
 		} else {
 			return errors.New("Aborted by user")
 		}
 	}
-	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
+	return l.install(o, syncedRepos, match, packages, assertions, allRepos, s)
 }
 
 func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string]ArtifactMatch) error {
@@ -455,7 +630,7 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 	return nil
 }
 
-func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
+func (l *LuetInstaller) computeInstall(o Option, syncedRepos Repositories, cp pkg.Packages, s *System) (map[string]ArtifactMatch, pkg.Packages, solver.PackagesAssertions, pkg.PackageDatabase, error) {
 	var p pkg.Packages
 	toInstall := map[string]ArtifactMatch{}
 	allRepos := pkg.NewInMemoryDatabase(false)
@@ -488,11 +663,11 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 	var packagesToInstall pkg.Packages
 	var err error
 
-	if !l.Options.NoDeps {
+	if !o.NoDeps {
 		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		solution, err = solv.Install(p)
 		/// TODO: PackageAssertions needs to be a map[fingerprint]pack so lookup is in O(1)
-		if err != nil && !l.Options.Force {
+		if err != nil && !o.Force {
 			return toInstall, p, solution, allRepos, errors.Wrap(err, "Failed solving solution for package")
 		}
 		// Gathers things to install
@@ -505,7 +680,7 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 				packagesToInstall = append(packagesToInstall, assertion.Package)
 			}
 		}
-	} else if !l.Options.OnlyDeps {
+	} else if !o.OnlyDeps {
 		for _, currentPack := range p {
 			if _, err := s.Database.FindPackage(currentPack); err == nil {
 				// skip matching if it is installed already
@@ -540,7 +715,47 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 	return toInstall, p, solution, allRepos, nil
 }
 
-func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
+func (l *LuetInstaller) getFinalizers(allRepos pkg.PackageDatabase, solution solver.PackagesAssertions, toInstall map[string]ArtifactMatch, nodeps bool) ([]pkg.Package, error) {
+	var toFinalize []pkg.Package
+	if !nodeps {
+		// TODO: Lower those errors as warning
+		for _, w := range toInstall {
+			// Finalizers needs to run in order and in sequence.
+			ordered, err := solution.Order(allRepos, w.Package.GetFingerPrint())
+			if err != nil {
+				return toFinalize, errors.Wrap(err, "While order a solution for "+w.Package.HumanReadableString())
+			}
+		ORDER:
+			for _, ass := range ordered {
+				if ass.Value {
+					installed, ok := toInstall[ass.Package.GetFingerPrint()]
+					if !ok {
+						// It was a dep already installed in the system, so we can skip it safely
+						continue ORDER
+					}
+					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
+					if err != nil {
+						return toFinalize, errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
+					}
+
+					toFinalize = append(toFinalize, treePackage)
+				}
+			}
+
+		}
+	} else {
+		for _, c := range toInstall {
+			treePackage, err := c.Repository.GetTree().GetDatabase().FindPackage(c.Package)
+			if err != nil {
+				return toFinalize, errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
+			}
+			toFinalize = append(toFinalize, treePackage)
+		}
+	}
+	return toFinalize, nil
+}
+
+func (l *LuetInstaller) install(o Option, syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
 	// Install packages into rootfs in parallel.
 	if err := l.download(syncedRepos, toInstall); err != nil {
 		return errors.Wrap(err, "Downloading packages")
@@ -569,46 +784,19 @@ func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]A
 	for _, c := range toInstall {
 		// Annotate to the system that the package was installed
 		_, err := s.Database.CreatePackage(c.Package)
-		if err != nil && !l.Options.Force {
+		if err != nil && !o.Force {
 			return errors.Wrap(err, "Failed creating package")
 		}
 		bus.Manager.Publish(bus.EventPackageInstall, c)
 	}
-	var toFinalize []pkg.Package
-	if !l.Options.NoDeps {
-		// TODO: Lower those errors as warning
-		for _, w := range p {
-			// Finalizers needs to run in order and in sequence.
-			ordered, err := solution.Order(allRepos, w.GetFingerPrint())
-			if err != nil {
-				return errors.Wrap(err, "While order a solution for "+w.HumanReadableString())
-			}
-		ORDER:
-			for _, ass := range ordered {
-				if ass.Value {
-					installed, ok := toInstall[ass.Package.GetFingerPrint()]
-					if !ok {
-						// It was a dep already installed in the system, so we can skip it safely
-						continue ORDER
-					}
-					treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
-					if err != nil {
-						return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
-					}
 
-					toFinalize = append(toFinalize, treePackage)
-				}
-			}
+	if !o.RunFinalizers {
+		return nil
+	}
 
-		}
-	} else {
-		for _, c := range toInstall {
-			treePackage, err := c.Repository.GetTree().GetDatabase().FindPackage(c.Package)
-			if err != nil {
-				return errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
-			}
-			toFinalize = append(toFinalize, treePackage)
-		}
+	toFinalize, err := l.getFinalizers(allRepos, solution, toInstall, o.NoDeps)
+	if err != nil {
+		return errors.Wrap(err, "failed getting package to finalize")
 	}
 
 	return s.ExecuteFinalizers(toFinalize)
@@ -688,6 +876,51 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 	return nil
 }
 
+func checkAndPrunePath(path string) {
+	// check if now the target path is empty
+	targetPath := filepath.Dir(path)
+
+	fi, err := os.Lstat(targetPath)
+	if err != nil {
+		//	Warning("Dir not found (it was before?) ", err.Error())
+		return
+	}
+
+	switch mode := fi.Mode(); {
+	case mode.IsDir():
+		files, err := ioutil.ReadDir(targetPath)
+		if err != nil {
+			Warning("Failed reading folder", targetPath, err.Error())
+		}
+		if len(files) != 0 {
+			Debug("Preserving not-empty folder", targetPath)
+			return
+		}
+	}
+	if err = os.Remove(targetPath); err != nil {
+		Warning("Failed removing file (maybe not present in the system target anymore ?)", targetPath, err.Error())
+	}
+}
+
+// We will try to cleanup every path from the file, if the folders left behind are empty
+func pruneEmptyFilePath(path string) {
+	checkAndPrunePath(path)
+
+	// A path is for e.g. /usr/bin/bar
+	// we want to create an array as "/usr", "/usr/bin", "/usr/bin/bar"
+	paths := strings.Split(path, string(os.PathSeparator))
+	currentPath := filepath.Join(string(os.PathSeparator), paths[0])
+	allPaths := []string{currentPath}
+	for _, p := range paths[1:] {
+		currentPath = filepath.Join(currentPath, p)
+		allPaths = append(allPaths, currentPath)
+	}
+	helpers.ReverseAny(allPaths)
+	for _, p := range allPaths {
+		checkAndPrunePath(p)
+	}
+}
+
 func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 	var cp *config.ConfigProtect
 	annotationDir := ""
@@ -749,6 +982,8 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		if err = os.Remove(target); err != nil {
 			Warning("Failed removing file (maybe not present in the system target anymore ?)", target, err.Error())
 		}
+
+		pruneEmptyFilePath(target)
 	}
 
 	for _, f := range notPresent {
@@ -762,6 +997,8 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 		if err = os.Remove(target); err != nil {
 			Debug("Failed removing file (not present in the system target)", target, err.Error())
 		}
+
+		pruneEmptyFilePath(target)
 	}
 
 	err = s.Database.RemovePackageFiles(p)
@@ -779,17 +1016,17 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 	return nil
 }
 
-func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.Packages, error) {
+func (l *LuetInstaller) computeUninstall(o Option, s *System, packs ...pkg.Package) (pkg.Packages, error) {
 
 	var toUninstall pkg.Packages
 	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) TODO - mark the uninstallation in db
 	// Get installed definition
-	checkConflicts := l.Options.CheckConflicts
-	full := l.Options.FullUninstall
-	if l.Options.Force == true { // IF forced, we want to remove the package and all its requires
-		checkConflicts = false
-		full = false
-	}
+	checkConflicts := o.CheckConflicts
+	full := o.FullUninstall
+	// if o.Force == true { // IF forced, we want to remove the package and all its requires
+	// 	checkConflicts = false
+	// 	full = false
+	// }
 
 	// Create a temporary DB with the installed packages
 	// so the solver is much faster finding the deptree
@@ -799,11 +1036,11 @@ func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.P
 		return toUninstall, errors.Wrap(err, "Failed create temporary in-memory db")
 	}
 
-	if !l.Options.NoDeps {
+	if !o.NoDeps {
 		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		var solution pkg.Packages
 		var err error
-		if l.Options.FullCleanUninstall {
+		if o.FullCleanUninstall {
 			solution, err = solv.UninstallUniverse(packs)
 			if err != nil {
 				return toUninstall, errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
@@ -824,32 +1061,47 @@ func (l *LuetInstaller) computeUninstall(s *System, packs ...pkg.Package) (pkg.P
 
 	return toUninstall, nil
 }
-func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
 
+func (l *LuetInstaller) generateUninstallFn(o Option, s *System, packs ...pkg.Package) (pkg.Packages, func() error, error) {
 	for _, p := range packs {
 		if packs, _ := s.Database.FindPackages(p); len(packs) == 0 {
-			return errors.New("Package not found in the system")
+			return nil, nil, errors.New("Package not found in the system")
 		}
-
 	}
 
-	Spinner(32)
-	toUninstall, err := l.computeUninstall(s, packs...)
+	toUninstall, err := l.computeUninstall(o, s, packs...)
 	if err != nil {
-		return errors.Wrap(err, "while computing uninstall")
+		return nil, nil, errors.Wrap(err, "while computing uninstall")
 	}
-	SpinnerStop()
 
 	uninstall := func() error {
 		for _, p := range toUninstall {
 			err := l.uninstall(p, s)
-			if err != nil && !l.Options.Force {
+			if err != nil && !o.Force {
 				return errors.Wrap(err, "Uninstall failed")
 			}
 		}
 		return nil
 	}
 
+	return toUninstall, uninstall, nil
+}
+
+func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error {
+
+	Spinner(32)
+	o := Option{
+		FullUninstall:      l.Options.FullUninstall,
+		Force:              l.Options.Force,
+		CheckConflicts:     l.Options.CheckConflicts,
+		FullCleanUninstall: l.Options.FullCleanUninstall,
+	}
+	toUninstall, uninstall, err := l.generateUninstallFn(o, s, packs...)
+	if err != nil {
+		return errors.Wrap(err, "while computing uninstall")
+	}
+	SpinnerStop()
+
 	if len(toUninstall) == 0 {
 		Info("Nothing to do")
 		return nil

pkg/package/package.go

@@ -645,6 +645,16 @@ func (set Packages) Best(v version.Versioner) Package {
 	return versionsMap[sorted[len(sorted)-1]]
 }
 
+func (set Packages) Find(packageName string) (Package, error) {
+	for _, p := range set {
+		if p.GetPackageName() == packageName {
+			return p, nil
+		}
+	}
+
+	return &DefaultPackage{}, errors.New("package not found")
+}
+
 func (set Packages) Unique() Packages {
 	var result Packages
 	uniq := make(map[string]Package)

tests/fixtures/perms/pkgA/0.1/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+unpack: true
+includes:
+  - /foo
+  - /foo/bar
+  - /foo/bar/.keep
+steps:
+- mkdir -p /foo/bar
+- touch /foo/bar/.keep
+- chown 100:100 /foo/bar
+- chown 101:101 /foo/bar/.keep

tests/fixtures/perms/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "perms"
+version: "0.1"

tests/fixtures/perms/pkgB/0.1/build.yaml

@@ -0,0 +1,7 @@
+image: "alpine"
+
+steps:
+- mkdir -p /foo/baz
+- touch /foo/baz/.keep
+- chown 100:100 /foo/baz
+- chown 101:101 /foo/baz/.keep

tests/fixtures/perms/pkgB/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "perms2"
+version: "0.1"

tests/fixtures/plugin/test-foo

@@ -1,3 +1,5 @@
 #!/bin/bash
 echo "$1" >> $EVENT_FILE
-echo "$2" >> $PAYLOAD_FILE
+echo "$2" >> $PAYLOAD_FILE
+
+echo "{}"

tests/integration/12_config_protect_samefile.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testrootfs/testbuild -p
+    luet build --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+      --destination $tmpdir/testrootfs/testbuild --compression gzip test/a
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testrootfs/testbuild/a-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+    --output $tmpdir/testrootfs/testbuild \
+    --packages $tmpdir/testrootfs/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testrootfs/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+
+    mkdir $tmpdir/testrootfs/etc/luet/config.protect.d -p
+
+    cat <<EOF > $tmpdir/testrootfs/etc/luet/config.protect.d/conf1.yml
+name: "protect1"
+dirs:
+- /etc/
+EOF
+
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_protect_confdir:
+    - /etc/luet/config.protect.d
+config_from_host: false
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+
+
+testInstall() {
+
+    # Simulate previous installation
+    mkdir $tmpdir/testrootfs/etc/a -p
+    echo config > $tmpdir/testrootfs/etc/a/conf
+
+    luet install -y --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+
+    # Simulate config protect
+    assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect not created, file is the same' "[ ! -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
+    assertEquals 'config protect content' "$(cat $tmpdir/testrootfs/etc/a/conf)" "config"
+}
+
+
+testUnInstall() {
+    luet uninstall -y --full --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect maintains the protected files' "[ -e '$tmpdir/testrootfs/etc/a/conf' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/a-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/16_perms.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testbuild
+    luet build -d --tree "$ROOT_DIR/tests/fixtures/perms" --same-owner=true --destination $tmpdir/testbuild --compression gzip --full
+    buildst=$?
+    assertTrue 'create package perms 0.1' "[ -e '$tmpdir/testbuild/perms-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/perms" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    $ROOT_DIR/tests/integration/bin/luet install -y --config $tmpdir/luet.yaml test/perms@0.1 test/perms2@0.1
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+   
+    assertTrue 'package installed perms baz' "[ -d '$tmpdir/testrootfs/foo/baz' ]"
+    assertTrue 'package installed perms bar' "[ -d '$tmpdir/testrootfs/foo/bar' ]"
+
+    assertContains 'perms1' "$(stat -c %u:%g $tmpdir/testrootfs/foo/baz)" "100:100"
+    assertContains 'perms2' "$(stat -c %u:%g $tmpdir/testrootfs/foo/bar)" "100:100"
+    assertContains 'perms11' "$(stat -c %u:%g $tmpdir/testrootfs/foo/baz/.keep)" "101:101"
+    assertContains 'perms22' "$(stat -c %u:%g $tmpdir/testrootfs/foo/bar/.keep)" "101:101"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

vendor/github.com/codegangsta/inject/.gitignore

@@ -1,2 +0,0 @@
-inject
-inject.test

vendor/github.com/codegangsta/inject/LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2013 Jeremy Saenz
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/codegangsta/inject/README.md

@@ -1,92 +0,0 @@
-# inject
---
-    import "github.com/codegangsta/inject"
-
-Package inject provides utilities for mapping and injecting dependencies in
-various ways.
-
-Language Translations:
-* [简体中文](translations/README_zh_cn.md)
-
-## Usage
-
-#### func  InterfaceOf
-
-```go
-func InterfaceOf(value interface{}) reflect.Type
-```
-InterfaceOf dereferences a pointer to an Interface type. It panics if value is
-not an pointer to an interface.
-
-#### type Applicator
-
-```go
-type Applicator interface {
-	// Maps dependencies in the Type map to each field in the struct
-	// that is tagged with 'inject'. Returns an error if the injection
-	// fails.
-	Apply(interface{}) error
-}
-```
-
-Applicator represents an interface for mapping dependencies to a struct.
-
-#### type Injector
-
-```go
-type Injector interface {
-	Applicator
-	Invoker
-	TypeMapper
-	// SetParent sets the parent of the injector. If the injector cannot find a
-	// dependency in its Type map it will check its parent before returning an
-	// error.
-	SetParent(Injector)
-}
-```
-
-Injector represents an interface for mapping and injecting dependencies into
-structs and function arguments.
-
-#### func  New
-
-```go
-func New() Injector
-```
-New returns a new Injector.
-
-#### type Invoker
-
-```go
-type Invoker interface {
-	// Invoke attempts to call the interface{} provided as a function,
-	// providing dependencies for function arguments based on Type. Returns
-	// a slice of reflect.Value representing the returned values of the function.
-	// Returns an error if the injection fails.
-	Invoke(interface{}) ([]reflect.Value, error)
-}
-```
-
-Invoker represents an interface for calling functions via reflection.
-
-#### type TypeMapper
-
-```go
-type TypeMapper interface {
-	// Maps the interface{} value based on its immediate type from reflect.TypeOf.
-	Map(interface{}) TypeMapper
-	// Maps the interface{} value based on the pointer of an Interface provided.
-	// This is really only useful for mapping a value as an interface, as interfaces
-	// cannot at this time be referenced directly without a pointer.
-	MapTo(interface{}, interface{}) TypeMapper
-	// Provides a possibility to directly insert a mapping based on type and value.
-	// This makes it possible to directly map type arguments not possible to instantiate
-	// with reflect like unidirectional channels.
-	Set(reflect.Type, reflect.Value) TypeMapper
-	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
-	// the Type has not been mapped.
-	Get(reflect.Type) reflect.Value
-}
-```
-
-TypeMapper represents an interface for mapping interface{} values based on type.

vendor/github.com/codegangsta/inject/inject.go

@@ -1,187 +0,0 @@
-// Package inject provides utilities for mapping and injecting dependencies in various ways.
-package inject
-
-import (
-	"fmt"
-	"reflect"
-)
-
-// Injector represents an interface for mapping and injecting dependencies into structs
-// and function arguments.
-type Injector interface {
-	Applicator
-	Invoker
-	TypeMapper
-	// SetParent sets the parent of the injector. If the injector cannot find a
-	// dependency in its Type map it will check its parent before returning an
-	// error.
-	SetParent(Injector)
-}
-
-// Applicator represents an interface for mapping dependencies to a struct.
-type Applicator interface {
-	// Maps dependencies in the Type map to each field in the struct
-	// that is tagged with 'inject'. Returns an error if the injection
-	// fails.
-	Apply(interface{}) error
-}
-
-// Invoker represents an interface for calling functions via reflection.
-type Invoker interface {
-	// Invoke attempts to call the interface{} provided as a function,
-	// providing dependencies for function arguments based on Type. Returns
-	// a slice of reflect.Value representing the returned values of the function.
-	// Returns an error if the injection fails.
-	Invoke(interface{}) ([]reflect.Value, error)
-}
-
-// TypeMapper represents an interface for mapping interface{} values based on type.
-type TypeMapper interface {
-	// Maps the interface{} value based on its immediate type from reflect.TypeOf.
-	Map(interface{}) TypeMapper
-	// Maps the interface{} value based on the pointer of an Interface provided.
-	// This is really only useful for mapping a value as an interface, as interfaces
-	// cannot at this time be referenced directly without a pointer.
-	MapTo(interface{}, interface{}) TypeMapper
-	// Provides a possibility to directly insert a mapping based on type and value.
-	// This makes it possible to directly map type arguments not possible to instantiate
-	// with reflect like unidirectional channels.
-	Set(reflect.Type, reflect.Value) TypeMapper
-	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
-	// the Type has not been mapped.
-	Get(reflect.Type) reflect.Value
-}
-
-type injector struct {
-	values map[reflect.Type]reflect.Value
-	parent Injector
-}
-
-// InterfaceOf dereferences a pointer to an Interface type.
-// It panics if value is not an pointer to an interface.
-func InterfaceOf(value interface{}) reflect.Type {
-	t := reflect.TypeOf(value)
-
-	for t.Kind() == reflect.Ptr {
-		t = t.Elem()
-	}
-
-	if t.Kind() != reflect.Interface {
-		panic("Called inject.InterfaceOf with a value that is not a pointer to an interface. (*MyInterface)(nil)")
-	}
-
-	return t
-}
-
-// New returns a new Injector.
-func New() Injector {
-	return &injector{
-		values: make(map[reflect.Type]reflect.Value),
-	}
-}
-
-// Invoke attempts to call the interface{} provided as a function,
-// providing dependencies for function arguments based on Type.
-// Returns a slice of reflect.Value representing the returned values of the function.
-// Returns an error if the injection fails.
-// It panics if f is not a function
-func (inj *injector) Invoke(f interface{}) ([]reflect.Value, error) {
-	t := reflect.TypeOf(f)
-
-	var in = make([]reflect.Value, t.NumIn()) //Panic if t is not kind of Func
-	for i := 0; i < t.NumIn(); i++ {
-		argType := t.In(i)
-		val := inj.Get(argType)
-		if !val.IsValid() {
-			return nil, fmt.Errorf("Value not found for type %v", argType)
-		}
-
-		in[i] = val
-	}
-
-	return reflect.ValueOf(f).Call(in), nil
-}
-
-// Maps dependencies in the Type map to each field in the struct
-// that is tagged with 'inject'.
-// Returns an error if the injection fails.
-func (inj *injector) Apply(val interface{}) error {
-	v := reflect.ValueOf(val)
-
-	for v.Kind() == reflect.Ptr {
-		v = v.Elem()
-	}
-
-	if v.Kind() != reflect.Struct {
-		return nil // Should not panic here ?
-	}
-
-	t := v.Type()
-
-	for i := 0; i < v.NumField(); i++ {
-		f := v.Field(i)
-		structField := t.Field(i)
-		if f.CanSet() && (structField.Tag == "inject" || structField.Tag.Get("inject") != "") {
-			ft := f.Type()
-			v := inj.Get(ft)
-			if !v.IsValid() {
-				return fmt.Errorf("Value not found for type %v", ft)
-			}
-
-			f.Set(v)
-		}
-
-	}
-
-	return nil
-}
-
-// Maps the concrete value of val to its dynamic type using reflect.TypeOf,
-// It returns the TypeMapper registered in.
-func (i *injector) Map(val interface{}) TypeMapper {
-	i.values[reflect.TypeOf(val)] = reflect.ValueOf(val)
-	return i
-}
-
-func (i *injector) MapTo(val interface{}, ifacePtr interface{}) TypeMapper {
-	i.values[InterfaceOf(ifacePtr)] = reflect.ValueOf(val)
-	return i
-}
-
-// Maps the given reflect.Type to the given reflect.Value and returns
-// the Typemapper the mapping has been registered in.
-func (i *injector) Set(typ reflect.Type, val reflect.Value) TypeMapper {
-	i.values[typ] = val
-	return i
-}
-
-func (i *injector) Get(t reflect.Type) reflect.Value {
-	val := i.values[t]
-
-	if val.IsValid() {
-		return val
-	}
-
-	// no concrete types found, try to find implementors
-	// if t is an interface
-	if t.Kind() == reflect.Interface {
-		for k, v := range i.values {
-			if k.Implements(t) {
-				val = v
-				break
-			}
-		}
-	}
-
-	// Still no type found, try to look it up on the parent
-	if !val.IsValid() && i.parent != nil {
-		val = i.parent.Get(t)
-	}
-
-	return val
-
-}
-
-func (i *injector) SetParent(parent Injector) {
-	i.parent = parent
-}

vendor/github.com/codegangsta/inject/update_readme.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-go get github.com/robertkrimen/godocdown/godocdown
-godocdown > README.md

vendor/github.com/mudler/go-pluggable/README.md

@@ -1,6 +1,7 @@
-# go-pluggable
+# :bento: go-pluggable
+[![PkgGoDev](https://pkg.go.dev/badge/github.com/mudler/go-pluggable)](https://pkg.go.dev/github.com/mudler/go-pluggable) [![Go Report Card](https://goreportcard.com/badge/github.com/mudler/go-pluggable)](https://goreportcard.com/report/github.com/mudler/go-pluggable) [![Test](https://github.com/mudler/go-pluggable/workflows/Test/badge.svg)](https://github.com/mudler/go-pluggable/actions?query=workflow%3ATest)
 
-light Bus-event driven plugin library for Golang.
+:bento: *go-pluggable* is a light Bus-event driven plugin library for Golang.
 
 `go-pluggable` implements the event/sub pattern to extend your Golang project with external binary plugins that can be written in any language.
 
@@ -24,8 +25,24 @@ func main() {
     m.Autoload("test", temp)
     m.Register()
 
-    // ...
-    m.Publish(myEv,  map[string]string{"foo": "bar"}) // test-foo, will receive our data as json payload
+    // Optionally process plugin results response
+    // The plugins has to return as output a json in stdout in the format { 'state': "somestate", data: "some data", error: "some error" }
+    // e.g. with jq:  
+    // jq --arg key0   'state' \
+    // --arg value0 '' \
+    // --arg key1   'data' \
+    // --arg value1 "" \
+    // --arg key2   'error' \
+    // --arg value2 '' \
+    // '. | .[$key0]=$value0 | .[$key1]=$value1 | .[$key2]=$value2' \
+    // <<<'{}'
+    m.Response(myEv, func(p *pluggable.Plugin, r *pluggable.EventResponse) { ... }) 
+
+    // Emit events, they are encoded and passed as JSON payloads to the plugins.
+    // In our case, test-foo will receive the map as JSON
+    m.Publish(myEv,  map[string]string{"foo": "bar"})
+
+
 }
 
-```
+```

vendor/github.com/mudler/go-pluggable/bus.go

@@ -1,73 +0,0 @@
-// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package pluggable
-
-import (
-	"sync"
-
-	"github.com/chuckpreslar/emission"
-	"github.com/codegangsta/inject"
-)
-
-// Bus represent the bus event system
-type Bus struct {
-	inject.Injector
-	emission.Emitter
-	sync.Mutex
-}
-
-// NewBus returns a new Bus instance
-func NewBus() *Bus {
-	return &Bus{
-		Injector: inject.New(),
-		Emitter:  *emission.NewEmitter(),
-	}
-}
-
-// Listen Binds a callback to an event, mapping the arguments on a global level
-func (a *Bus) Listen(event EventType, listener interface{}) *Bus {
-	a.Lock()
-	defer a.Unlock()
-	a.On(string(event), func() { a.Invoke(listener) })
-	return a
-}
-
-// Publish publishes an event, it does accept only the event as argument, since
-// the callback will have access to the service mapped by the injector
-func (a *Bus) Publish(e *Event) *Bus {
-	a.Lock()
-	defer a.Unlock()
-	a.Map(e)
-	a.Emit(string(e.Name))
-	return a
-}
-
-// OnlyOnce Binds a callback to an event, mapping the arguments on a global level
-// It is fired only once.
-func (a *Bus) OnlyOnce(event EventType, listener interface{}) *Bus {
-	a.Lock()
-	defer a.Unlock()
-	a.Once(string(event), func() { a.Invoke(listener) })
-	return a
-}
-
-func (a *Bus) propagateEvent(p Plugin) func(e *Event) {
-	return func(e *Event) {
-		resp, _ := p.Run(*e)
-		a.Map(&resp)
-		a.Emit(p.Name)
-	}
-}

vendor/github.com/mudler/go-pluggable/events.go

@@ -15,7 +15,10 @@
 
 package pluggable
 
-import "encoding/json"
+import (
+	"encoding/json"
+	"fmt"
+)
 
 // EventType describes an event type
 type EventType string
@@ -26,6 +29,7 @@ type EventType string
 type Event struct {
 	Name EventType `json:"name"`
 	Data string    `json:"data"`
+	File string    `json:"file"` // If Data >> 10K write content to file instead
 }
 
 // EventResponse describes the event response structure
@@ -42,6 +46,16 @@ func (e Event) JSON() (string, error) {
 	return string(dat), err
 }
 
+// Copy returns a copy of Event
+func (e Event) Copy() *Event {
+	copy := &e
+	return copy
+}
+
+func (e Event) ResponseEventName(s string) EventType {
+	return EventType(fmt.Sprintf("%s-%s", e.Name, s))
+}
+
 // Unmarshal decodes the json payload in the given parameteer
 func (r EventResponse) Unmarshal(i interface{}) error {
 	return json.Unmarshal([]byte(r.Data), i)
@@ -52,7 +66,7 @@ func (r EventResponse) Errored() bool {
 	return len(r.Error) != 0
 }
 
-// NewEvent retuns a new event which can be used for publishing
+// NewEvent returns a new event which can be used for publishing
 // the obj gets automatically serialized in json.
 func NewEvent(name EventType, obj interface{}) (*Event, error) {
 	dat, err := json.Marshal(obj)

vendor/github.com/mudler/go-pluggable/go.mod

@@ -4,7 +4,6 @@ go 1.14
 
 require (
 	github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9
-	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3
 	github.com/pkg/errors v0.9.1

vendor/github.com/mudler/go-pluggable/go.sum

@@ -1,7 +1,5 @@
 github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9 h1:xz6Nv3zcwO2Lila35hcb0QloCQsc38Al13RNEzWRpX4=
 github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9/go.mod h1:2wSM9zJkl1UQEFZgSd68NfCgRz1VL1jzy/RjCg+ULrs=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=

vendor/github.com/mudler/go-pluggable/manager.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/chuckpreslar/emission"
 	"github.com/pkg/errors"
 )
 
@@ -29,14 +30,14 @@ import (
 type Manager struct {
 	Plugins []Plugin
 	Events  []EventType
-	Bus     *Bus
+	Bus     *emission.Emitter
 }
 
 // NewManager returns a manager instance with a new bus and
 func NewManager(events []EventType) *Manager {
 	return &Manager{
 		Events: events,
-		Bus:    NewBus(),
+		Bus:    emission.NewEmitter(),
 	}
 }
 
@@ -47,19 +48,41 @@ func (m *Manager) Register() *Manager {
 }
 
 // Publish is a wrapper around NewEvent and the Manager internal Bus publishing system
+// It accepts optionally a list of functions that are called with the plugin result (only once)
 func (m *Manager) Publish(event EventType, obj interface{}) (*Manager, error) {
 	ev, err := NewEvent(event, obj)
 	if err == nil && ev != nil {
-		m.Bus.Publish(ev)
+		m.Bus.Emit(string(ev.Name), ev)
 	}
 	return m, err
 }
 
+// Response binds a set of listeners to an event type. The listeners are called for each result from
+// every plugin when Publish is called.
+func (m *Manager) Response(event EventType, listener ...func(p *Plugin, r *EventResponse)) *Manager {
+	ev, _ := NewEvent(event, nil)
+	for _, l := range listener {
+		m.Bus.On(string(ev.ResponseEventName("results")), l)
+	}
+	return m
+}
+
+func (m *Manager) propagateEvent(p Plugin) func(e *Event) {
+	return func(e *Event) {
+		resp, err := p.Run(*e)
+		r := &resp
+		if err != nil && !resp.Errored() {
+			resp.Error = err.Error()
+		}
+		m.Bus.Emit(string(e.ResponseEventName("results")), &p, r)
+	}
+}
+
 // Subscribe subscribes the plugin to the events in the given bus
-func (m *Manager) Subscribe(b *Bus) *Manager {
+func (m *Manager) Subscribe(b *emission.Emitter) *Manager {
 	for _, p := range m.Plugins {
 		for _, e := range m.Events {
-			b.Listen(e, b.propagateEvent(p))
+			b.On(string(e), m.propagateEvent(p))
 		}
 	}
 	return m
@@ -74,13 +97,6 @@ func relativeToCwd(p string) (string, error) {
 	return filepath.Join(cwd, p), nil
 }
 
-// ListenAll Binds a callback to all plugins event
-func (m *Manager) ListenAll(event EventType, listener interface{}) {
-	for _, p := range m.Plugins {
-		m.Bus.Listen(EventType(p.Name), listener)
-	}
-}
-
 // Autoload automatically loads plugins binaries prefixed by 'prefix' in the current path
 // optionally takes a list of paths to look also into
 func (m *Manager) Autoload(prefix string, extensionpath ...string) *Manager {

vendor/github.com/mudler/go-pluggable/plugin.go

@@ -16,7 +16,9 @@
 package pluggable
 
 import (
+	"bytes"
 	"encoding/json"
+	"io/ioutil"
 	"os"
 	"os/exec"
 
@@ -29,19 +31,42 @@ type Plugin struct {
 	Executable string
 }
 
+// A safe threshold to avoid unpleasant exec buffer fill for argv too big. Seems 128K is the limit on Linux.
+const maxMessageSize = 1 << 13
+
 // Run runs the Event on the plugin, and returns an EventResponse
 func (p Plugin) Run(e Event) (EventResponse, error) {
 	r := EventResponse{}
-	k, err := e.JSON()
+
+	eventToprocess := &e
+
+	if len(e.Data) > maxMessageSize {
+		copy := e.Copy()
+		copy.Data = ""
+		f, err := ioutil.TempFile(os.TempDir(), "pluggable")
+		if err != nil {
+			return r, errors.Wrap(err, "while creating temporary file")
+		}
+		if err := ioutil.WriteFile(f.Name(), []byte(e.Data), os.ModePerm); err != nil {
+			return r, errors.Wrap(err, "while writing to temporary file")
+		}
+		copy.File = f.Name()
+		eventToprocess = copy
+		defer os.RemoveAll(f.Name())
+	}
+
+	k, err := eventToprocess.JSON()
 	if err != nil {
 		return r, errors.Wrap(err, "while marshalling event")
 	}
 	cmd := exec.Command(p.Executable, string(e.Name), k)
 	cmd.Env = os.Environ()
+	var b bytes.Buffer
+	cmd.Stderr = &b
 	out, err := cmd.Output()
 	if err != nil {
-		r.Error = err.Error()
-		return r, errors.Wrap(err, "while executing plugin")
+		r.Error = "error while executing plugin: " + err.Error() + string(b.String())
+		return r, errors.Wrap(err, "while executing plugin: "+string(b.String()))
 	}
 
 	if err := json.Unmarshal(out, &r); err != nil {

vendor/modules.txt

@@ -62,8 +62,6 @@ github.com/cavaliercoder/grab/bps
 github.com/cespare/xxhash/v2
 # github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9
 github.com/chuckpreslar/emission
-# github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0
-github.com/codegangsta/inject
 # github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d
 github.com/containerd/cgroups/stats/v1
 # github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc => github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55
@@ -209,7 +207,6 @@ github.com/fatih/color
 # github.com/fsnotify/fsnotify v1.4.9
 github.com/fsnotify/fsnotify
 # github.com/fsouza/go-dockerclient v1.6.4
-## explicit
 github.com/fsouza/go-dockerclient
 # github.com/genuinetools/img v0.5.11
 ## explicit
@@ -478,7 +475,7 @@ github.com/mudler/cobra-extensions
 # github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
 ## explicit
 github.com/mudler/docker-companion/api
-# github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+# github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 ## explicit
 github.com/mudler/go-pluggable
 # github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290