Tag 0.16.3

A new keyword `join` is introduced to generate the parent image. It
takes precedence over a `requires` or a `image` already defined in a
spec.

It will generate all the artifacts from the packages listed and join
them in a single image which will be used as parent for the package
build process.

This is a change which invalidates priorly generated hashes.

Fixes #173

cmd/build.go

@@ -247,11 +247,12 @@ Build packages specifying multiple definition trees:
 			}
 
 			for _, sp := range toCalculate {
-				packs, err := luetCompiler.ComputeDepTree(sp)
+				ht := compiler.NewHashTree(generalRecipe.GetDatabase())
+				hashTree, err := ht.Query(luetCompiler, sp)
 				if err != nil {
 					errs = append(errs, err)
 				}
-				for _, p := range packs {
+				for _, p := range hashTree.Dependencies {
 					results.Packages = append(results.Packages,
 						PackageResult{
 							Name:       p.Package.GetName(),

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.14.2"
+	LuetCLIVersion = "0.16.3"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -97,7 +97,7 @@ To build a package, from a tree definition:
 
 		plugin := viper.GetStringSlice("plugin")
 
-		bus.Manager.Load(plugin...).Register()
+		bus.Manager.Initialize(plugin...)
 		if len(bus.Manager.Plugins) != 0 {
 			Info(":lollipop:Enabled plugins:")
 			for _, p := range bus.Manager.Plugins {

cmd/tree/images.go

@@ -96,9 +96,14 @@ func NewTreeImageCommand() *cobra.Command {
 			if err != nil {
 				Fatal("Error: " + err.Error())
 			}
-			asserts, err := luetCompiler.ComputeDepTree(spec)
 
-			for _, assertion := range asserts { //highly dependent on the order
+			ht := compiler.NewHashTree(reciper.GetDatabase())
+			hashtree, err := ht.Query(luetCompiler, spec)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
+			for _, assertion := range hashtree.Solution { //highly dependent on the order
 
 				//buildImageHash := imageRepository + ":" + assertion.Hash.BuildHash
 				currentPackageImageHash := imageRepository + ":" + assertion.Hash.PackageHash

go.mod

@@ -16,7 +16,6 @@ require (
 	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
-	github.com/fsouza/go-dockerclient v1.6.4
 	github.com/genuinetools/img v0.5.11
 	github.com/ghodss/yaml v1.0.0
 	github.com/google/go-containerregistry v0.2.1
@@ -33,11 +32,12 @@ require (
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
+	github.com/mitchellh/hashstructure/v2 v2.0.1
 	github.com/moby/buildkit v0.7.2
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+	github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3

go.sum

@@ -188,8 +188,6 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d h1:UKAt78F1OvM4ceTn1VvXuYuatXohsFU1eSI2IBtTw9g=
@@ -730,6 +728,8 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
 github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y=
 github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
+github.com/mitchellh/hashstructure/v2 v2.0.1 h1:L60q1+q7cXE4JeEJJKMnh2brFIe3rZxCihYAB61ypAY=
+github.com/mitchellh/hashstructure/v2 v2.0.1/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -760,8 +760,8 @@ github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d h1:fKh+rvw
 github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d/go.mod h1:puRUWSwyecW2V355tKncwPVPRAjQBduPsFjG0mrV/Nw=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87 h1:mGz7T8KvmHH0gLWPI5tQne8xl2cO3T8wrrb6Aa16Jxo=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87/go.mod h1:1w4zI1LYXDeiUXqedPcrT5eQJnmKR6dbg5iJMgSIP/Y=
-github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82 h1:Hkefw2tzoKATVUTFsCtDlUnY180+OE851qGbq45ATxk=
-github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82/go.mod h1:4P/ULate+2QxoAQtojaRjyO5VGMhV0KLnSdAS8nuBbo=
+github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af h1:jixIxEgLSqu24eMiyzfCI+roa5IaOUhF546ePSFyHeY=
+github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290 h1:426hFyXMpXeqIeGJn2cGAW9ogvM2Jf+Jv23gtVPvBLM=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290/go.mod h1:uP5BBgFxq2wNWo7n1vnY5SSbgL0WDshVJrOO12tZ/lA=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=

pkg/bus/events.go

@@ -1,6 +1,8 @@
 package bus
 
 import (
+	. "github.com/mudler/luet/pkg/logger"
+
 	"github.com/mudler/go-pluggable"
 )
 
@@ -47,21 +49,47 @@ var (
 )
 
 // Manager is the bus instance manager, which subscribes plugins to events emitted by Luet
-var Manager *pluggable.Manager = pluggable.NewManager(
-	[]pluggable.EventType{
-		EventPackageInstall,
-		EventPackageUnInstall,
-		EventPackagePreBuild,
-		EventPackagePreBuildArtifact,
-		EventPackagePostBuildArtifact,
-		EventPackagePostBuild,
-		EventRepositoryPreBuild,
-		EventRepositoryPostBuild,
-		EventImagePreBuild,
-		EventImagePrePull,
-		EventImagePrePush,
-		EventImagePostBuild,
-		EventImagePostPull,
-		EventImagePostPush,
-	},
-)
+var Manager *Bus = &Bus{
+	Manager: pluggable.NewManager(
+		[]pluggable.EventType{
+			EventPackageInstall,
+			EventPackageUnInstall,
+			EventPackagePreBuild,
+			EventPackagePreBuildArtifact,
+			EventPackagePostBuildArtifact,
+			EventPackagePostBuild,
+			EventRepositoryPreBuild,
+			EventRepositoryPostBuild,
+			EventImagePreBuild,
+			EventImagePrePull,
+			EventImagePrePush,
+			EventImagePostBuild,
+			EventImagePostPull,
+			EventImagePostPush,
+		},
+	),
+}
+
+type Bus struct {
+	*pluggable.Manager
+}
+
+func (b *Bus) Initialize(plugin ...string) {
+	b.Manager.Load(plugin...).Register()
+
+	for _, e := range b.Manager.Events {
+		b.Manager.Response(e, func(p *pluggable.Plugin, r *pluggable.EventResponse) {
+			if r.Errored() {
+				Fatal("Plugin", p.Name, "at", p.Executable, "Error", r.Error)
+			}
+			Debug(
+				"plugin_event",
+				"received from",
+				p.Name,
+				"at",
+				p.Executable,
+				r,
+			)
+		})
+	}
+}

pkg/compiler/compiler.go

@@ -16,7 +16,9 @@
 package compiler
 
 import (
+	"crypto/md5"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -89,7 +91,7 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *c
 	defer wg.Done()
 
 	for s := range cspecs {
-		ar, err := cs.compile(concurrency, keepPermissions, s)
+		ar, err := cs.compile(concurrency, keepPermissions, nil, s)
 		if err != nil {
 			errors <- err
 		}
@@ -146,11 +148,6 @@ func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps *compilerspec.L
 	}
 
 	for _, p := range ps.All() {
-		asserts, err := cs.ComputeDepTree(p)
-		if err != nil {
-			panic(err)
-		}
-		p.SetSourceAssertion(asserts)
 		all <- p
 	}
 
@@ -295,17 +292,6 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co
 	return artifact, nil
 }
 
-func (cs *LuetCompiler) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
-	// Use packageImage as salt into the fp being used
-	// so the hash is unique also in cases where
-	// some package deps does have completely different
-	// depgraphs
-	// TODO: We should use the image tag, or pass by the package assertion hash which is unique
-	// and identifies the deptree of the package.
-	return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(helpers.StripRegistryFromImage(packageImage)))
-
-}
-
 func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
 	concurrency int, keepPermissions bool,
 	p *compilerspec.LuetCompilationSpec) (backend.Options, backend.Options, error) {
@@ -522,7 +508,7 @@ func oneOfImagesAvailable(images []string, b CompilerBackend) (bool, string) {
 	return false, ""
 }
 
-func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
+func (cs *LuetCompiler) findImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
 	var resolvedImage string
 	Debug("Resolving image hash for", p.Package.HumanReadableString(), "hash", imageHash, "Pull repositories", p.BuildOptions.PullImageRepository)
 	toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)},
@@ -535,6 +521,11 @@ func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilersp
 			resolvedImage = which
 		}
 	}
+	return resolvedImage
+}
+
+func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
+	resolvedImage := cs.findImageHash(imageHash, p)
 
 	if resolvedImage == "" {
 		resolvedImage = fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)
@@ -619,12 +610,24 @@ func (cs *LuetCompiler) compileWithImage(image, builderHash string, packageTagHa
 		Debug("Checking if an image is already available")
 		// FIXUP here. If packageimage hash exists and pull is true, generate package
 		resolved := cs.resolveExistingImageHash(packageTagHash, p)
+		Debug("Resolved: " + resolved)
+		Debug("Expected remote: " + resolved)
+		Debug("Package image: " + packageImage)
+		Debug("Resolved builder image: " + builderResolved)
 
-		//
-		if resolved != packageImage && remoteBuildertaggedImage != builderResolved { // an image is there already
-			Debug("Images available for", p.Package.HumanReadableString(), "generating artifact from remote images:", resolved)
+		// a remote image is there already
+		remoteImageAvailable := resolved != packageImage && remoteBuildertaggedImage != builderResolved
+		// or a local one is available
+		localImageAvailable := cs.Backend.ImageExists(remoteBuildertaggedImage) && cs.Backend.ImageExists(packageImage)
+
+		switch {
+		case remoteImageAvailable:
+			Debug("Images available remotely for", p.Package.HumanReadableString(), "generating artifact from remote images:", resolved)
 			return cs.genArtifact(p, backend.Options{ImageName: builderResolved}, backend.Options{ImageName: resolved}, concurrency, keepPermissions)
-		} else {
+		case localImageAvailable:
+			Debug("Images locally available for", p.Package.HumanReadableString(), "generating artifact from image:", resolved)
+			return cs.genArtifact(p, backend.Options{ImageName: remoteBuildertaggedImage}, backend.Options{ImageName: packageImage}, concurrency, keepPermissions)
+		default:
 			Debug("Images not available for", p.Package.HumanReadableString())
 		}
 	}
@@ -680,33 +683,7 @@ func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst s
 	}
 }
 
-// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
-func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
-	// Generate a set with all the deps of the provided specs
-	// we will use that set to remove the deps from the list of provided compilation specs
-	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
-	result := []*compilerspec.LuetCompilationSpec{}
-	for _, spec := range p {
-		ass, err := cs.ComputeDepTree(spec)
-		if err != nil {
-			return result, errors.Wrap(err, "computin specs deptree")
-		}
-
-		allDependencies = append(allDependencies, ass.Drop(spec.GetPackage())...)
-	}
-
-	for _, spec := range p {
-		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
-			result = append(result, spec)
-		}
-	}
-	return result, nil
-}
-
-// ComputeDepTree computes the dependency tree of a compilation spec and returns solver assertions
-// in order to be able to compile the spec.
 func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
-
 	s := solver.NewResolver(cs.Options.SolverOptions.Options, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
 
 	solution, err := s.Install(pkg.Packages{p.GetPackage()})
@@ -718,32 +695,35 @@ func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (sol
 	if err != nil {
 		return nil, errors.Wrap(err, "While order a solution for "+p.GetPackage().HumanReadableString())
 	}
+	return dependencies, nil
+}
 
-	assertions := solver.PackagesAssertions{}
-	for _, assertion := range dependencies { //highly dependent on the order
-		if assertion.Value {
-			nthsolution := dependencies.Cut(assertion.Package)
-			assertion.Hash = solver.PackageHash{
-				BuildHash:   nthsolution.HashFrom(assertion.Package),
-				PackageHash: nthsolution.AssertionHash(),
-			}
-			assertion.Package.SetTreeDir(p.Package.GetTreeDir())
-			assertions = append(assertions, assertion)
+// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
+func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
+	// Generate a set with all the deps of the provided specs
+	// we will use that set to remove the deps from the list of provided compilation specs
+	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
+	result := []*compilerspec.LuetCompilationSpec{}
+	for _, spec := range p {
+		sol, err := cs.ComputeDepTree(spec)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed querying hashtree")
+		}
+		allDependencies = append(allDependencies, sol.Drop(spec.GetPackage())...)
+	}
+
+	for _, spec := range p {
+		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
+			result = append(result, spec)
 		}
 	}
-	p.SetSourceAssertion(assertions)
-	return assertions, nil
+	return result, nil
 }
 
 // Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
 // an artifact
 func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	asserts, err := cs.ComputeDepTree(p)
-	if err != nil {
-		return nil, err
-	}
-	p.SetSourceAssertion(asserts)
-	return cs.compile(cs.Options.Concurrency, keepPermissions, p)
+	return cs.compile(cs.Options.Concurrency, keepPermissions, nil, p)
 }
 
 func genImageList(refs []string, hash string) []string {
@@ -772,14 +752,183 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS
 	Debug(p.GetPackage().HumanReadableString(), "Build options after inherit", p.BuildOptions)
 }
 
-func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	// TODO: Racy, remove it
-	// Inherit build options from compilation specs metadata
-	// orig := cs.Options.PullImageRepository
-	// defer func() { cs.Options.PullImageRepository = orig }()
+func (cs *LuetCompiler) getSpecHash(pkgs pkg.DefaultPackages, salt string) (string, error) {
+	ht := NewHashTree(cs.Database)
+	overallFp := ""
 
+	for _, p := range pkgs {
+		compileSpec, err := cs.FromPackage(p)
+		if err != nil {
+			return "", errors.Wrap(err, "Error while generating compilespec for "+p.GetName())
+		}
+		packageHashTree, err := ht.Query(cs, compileSpec)
+		if err != nil {
+			return "nil", errors.Wrap(err, "failed querying hashtree")
+		}
+		overallFp = overallFp + packageHashTree.Target.Hash.PackageHash + p.GetFingerPrint()
+	}
+
+	h := md5.New()
+	io.WriteString(h, fmt.Sprintf("%s-%s", overallFp, salt))
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
+func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+
+	joinTag := ">:loop: join<"
+	if len(p.Join) != 0 {
+		Info(joinTag, "Generating a joint parent image from final packages")
+	} else {
+		return nil
+	}
+
+	// First compute a hash and check if image is available. if it is, then directly consume that
+	overallFp, err := cs.getSpecHash(p.Join, "join")
+	if err != nil {
+		return errors.Wrap(err, "could not generate image hash")
+	}
+
+	Info(joinTag, "Searching existing image with hash ", overallFp)
+
+	image := cs.findImageHash(overallFp, p)
+	if image != "" {
+		Info("Image already found", image)
+		p.SetImage(image)
+		return nil
+	}
+	Info(joinTag, "Image not found. Generating image join with hash ", overallFp)
+
+	// Make sure there is an output path
+	if err := os.MkdirAll(p.GetOutputPath(), os.ModePerm); err != nil {
+		return errors.Wrap(err, "while creating output path")
+	}
+
+	// otherwise, generate it and push it aside
+	joinDir, err := ioutil.TempDir(p.GetOutputPath(), "join")
+	if err != nil {
+		return errors.Wrap(err, "could not create tempdir for joining images")
+	}
+	defer os.RemoveAll(joinDir) // clean up
+
+	for _, p := range p.Join { //highly dependent on the order
+		Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
+	}
+
+	current := 0
+	for _, c := range p.Join {
+		current++
+		if c != nil && c.Name != "" && c.Version != "" {
+			joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(p.Join), c.HumanReadableString())
+
+			Info(joinTag2, "compilation starts")
+			spec, err := cs.FromPackage(c)
+			if err != nil {
+				return errors.Wrap(err, "while generating images to join from")
+			}
+			wantsArtifact := true
+			spec.SetOutputPath(p.GetOutputPath())
+
+			artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, spec)
+			if err != nil {
+				return errors.Wrap(err, "failed building join image")
+			}
+
+			err = artifact.Unpack(joinDir, keepPermissions)
+			if err != nil {
+				return errors.Wrap(err, "failed building join image")
+			}
+			Info(joinTag2, ":white_check_mark: Done")
+		}
+	}
+
+	artifactDir, err := ioutil.TempDir(p.GetOutputPath(), "artifact")
+	if err != nil {
+		return errors.Wrap(err, "could not create tempdir for final artifact")
+	}
+	defer os.RemoveAll(joinDir) // clean up
+
+	Info(joinTag, ":droplet: generating artifact for source image of", p.GetPackage().HumanReadableString())
+
+	// After unpack, create a new artifact and a new final image from it.
+	// no need to compress, as we are going to toss it away.
+	a := artifact.NewPackageArtifact(filepath.Join(artifactDir, p.GetPackage().GetFingerPrint()+".join.tar"))
+	if err := a.Compress(joinDir, concurrency); err != nil {
+		return errors.Wrap(err, "error met while creating package archive")
+	}
+
+	joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp)
+	Info(joinTag, ":droplet: generating image from artifact", joinImageName)
+	opts, err := a.GenerateFinalImage(joinImageName, cs.Backend, keepPermissions)
+	if err != nil {
+		return errors.Wrap(err, "could not create final image")
+	}
+	if cs.Options.Push {
+		Info(joinTag, ":droplet: pushing image from artifact", joinImageName)
+		if err = cs.Backend.Push(opts); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
+		}
+	}
+	Info(joinTag, ":droplet: Consuming image", joinImageName)
+	p.SetImage(joinImageName)
+	return nil
+}
+
+func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+	resolvedCopyFields := []compilerspec.CopyField{}
+	copyTag := ">:droplet: copy<"
+
+	if len(p.Copy) != 0 {
+		Info(copyTag, "Package has multi-stage copy, generating required images")
+	}
+
+	current := 0
+	// TODO: we should run this only if we are going to build the image
+	for _, c := range p.Copy {
+		current++
+		if c.Package != nil && c.Package.Name != "" && c.Package.Version != "" {
+			copyTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", copyTag, current, len(p.Copy), c.Package.HumanReadableString())
+
+			Info(copyTag2, "generating multi-stage images for", c.Package.HumanReadableString())
+			spec, err := cs.FromPackage(c.Package)
+			if err != nil {
+				return errors.Wrap(err, "while generating images to copy from")
+			}
+
+			// If we specify --only-target package, we don't want any artifact, otherwise we do
+			genArtifact := !cs.Options.PackageTargetOnly
+			spec.SetOutputPath(p.GetOutputPath())
+			artifact, err := cs.compile(concurrency, keepPermissions, &genArtifact, spec)
+			if err != nil {
+				return errors.Wrap(err, "failed building multi-stage image")
+			}
+
+			resolvedCopyFields = append(resolvedCopyFields, compilerspec.CopyField{
+				Image:       cs.resolveExistingImageHash(artifact.PackageCacheImage, spec),
+				Source:      c.Source,
+				Destination: c.Destination,
+			})
+			Info(copyTag2, ":white_check_mark: Done")
+		} else {
+			resolvedCopyFields = append(resolvedCopyFields, c)
+		}
+	}
+	p.Copy = resolvedCopyFields
+	return nil
+}
+
+func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
 	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
+	//Before multistage : join - same as multistage, but keep artifacts, join them, create a new one and generate a final image.
+	// When the image is there, use it as a source here, in place of GetImage().
+	if err := cs.resolveJoinImages(concurrency, keepPermissions, p); err != nil {
+		return nil, errors.Wrap(err, "while resolving join images")
+	}
+
+	if err := cs.resolveMultiStageImages(concurrency, keepPermissions, p); err != nil {
+		return nil, errors.Wrap(err, "while resolving multi-stage images")
+	}
+
 	Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
 	if !p.HasImageSource() && !p.EmptyPackage() {
 		return nil,
@@ -789,40 +938,62 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 			)
 	}
 
-	targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
+	ht := NewHashTree(cs.Database)
+
+	packageHashTree, err := ht.Query(cs, p)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed querying hashtree")
+	}
+
+	// This is in order to have the metadata in the yaml
+	p.SetSourceAssertion(packageHashTree.Solution)
+	targetAssertion := packageHashTree.Target
 
 	bus.Manager.Publish(bus.EventPackagePreBuild, struct {
-		CompileSpec *compilerspec.LuetCompilationSpec
-		Assert      solver.PackageAssert
+		CompileSpec     *compilerspec.LuetCompilationSpec
+		Assert          solver.PackageAssert
+		PackageHashTree *PackageImageHashTree
 	}{
-		CompileSpec: p,
-		Assert:      *targetAssertion,
+		CompileSpec:     p,
+		Assert:          *targetAssertion,
+		PackageHashTree: packageHashTree,
 	})
 
 	// Update compilespec build options - it will be then serialized into the compilation metadata file
-	//p.SetBuildOptions(cs.Options)
 	p.BuildOptions.PushImageRepository = cs.Options.PushImageRepository
-	//p.BuildOptions.BuildValues = cs.Options.BuildValues
-	//p.BuildOptions.BuildValuesFile = cs.Options.BuildValuesFile
 
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		return cs.compileWithImage(p.GetImage(), cs.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
+		localGenerateArtifact := true
+		if generateArtifact != nil {
+			localGenerateArtifact = *generateArtifact
+		}
+
+		a, err := cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
+		if err != nil {
+			return nil, errors.Wrap(err, "building direct image")
+		}
+		a.SourceAssertion = p.GetSourceAssertion()
+
+		a.PackageCacheImage = targetAssertion.Hash.PackageHash
+		return a, nil
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
 	// on how we compute the resolvable tree.
 	// This means to recursively build all the build-images needed to reach that tree part.
 	// - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image.
-
-	dependencies := p.GetSourceAssertion().Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
-	departifacts := []*artifact.PackageArtifact{}               // TODO: Return this somehow
-	var lastHash string
+	dependencies := packageHashTree.Dependencies  // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
+	departifacts := []*artifact.PackageArtifact{} // TODO: Return this somehow
 	depsN := 0
 	currentN := 0
 
 	packageDeps := !cs.Options.PackageTargetOnly
+	if generateArtifact != nil {
+		packageDeps = *generateArtifact
+	}
+
 	buildDeps := !cs.Options.NoDeps
 	buildTarget := !cs.Options.OnlyDeps
 
@@ -845,8 +1016,6 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 			Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
 
 			compileSpec.SetOutputPath(p.GetOutputPath())
-			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash)
-			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
 
 			bus.Manager.Publish(bus.EventPackagePreBuild, struct {
 				CompileSpec *compilerspec.LuetCompilationSpec
@@ -856,29 +1025,53 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 				Assert:      assertion,
 			})
 
-			lastHash = assertion.Hash.PackageHash
-			// for the source instead, pick an image and a buildertaggedImage from hashes if they exists.
-			// otherways fallback to the pushed repo
-			// Resolve images from the hashtree
-			resolvedBuildImage := cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec)
-			if compileSpec.GetImage() != "" {
-				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
-
-				a, err := cs.compileWithImage(compileSpec.GetImage(), assertion.Hash.BuildHash, assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps)
-				if err != nil {
-					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
-				}
-				departifacts = append(departifacts, a)
-				Info(pkgTag, ":white_check_mark: Done")
-				continue
+			if err := cs.resolveJoinImages(concurrency, keepPermissions, compileSpec); err != nil {
+				return nil, errors.Wrap(err, "while resolving join images")
 			}
 
-			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
-			a, err := cs.compileWithImage(resolvedBuildImage, cs.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash), assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps)
+			if err := cs.resolveMultiStageImages(concurrency, keepPermissions, compileSpec); err != nil {
+				return nil, errors.Wrap(err, "while resolving multi-stage images")
+			}
+
+			buildHash, err := packageHashTree.DependencyBuildImage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed looking for dependency in hashtree")
+			}
+
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash)
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
+
+			var sourceImage string
+
+			if compileSpec.GetImage() != "" {
+				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
+				sourceImage = compileSpec.GetImage()
+			} else {
+				// for the source instead, pick an image and a buildertaggedImage from hashes if they exists.
+				// otherways fallback to the pushed repo
+				// Resolve images from the hashtree
+				sourceImage = cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec)
+				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
+			}
+
+			a, err := cs.compileWithImage(
+				sourceImage,
+				buildHash,
+				assertion.Hash.PackageHash,
+				concurrency,
+				keepPermissions,
+				cs.Options.KeepImg,
+				compileSpec,
+				packageDeps,
+			)
 			if err != nil {
 				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 			}
 
+			a.PackageCacheImage = assertion.Hash.PackageHash
+
+			Info(pkgTag, ":white_check_mark: Done")
+
 			bus.Manager.Publish(bus.EventPackagePostBuild, struct {
 				CompileSpec *compilerspec.LuetCompilationSpec
 				Artifact    *artifact.PackageArtifact
@@ -888,24 +1081,24 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 			})
 
 			departifacts = append(departifacts, a)
-			Info(pkgTag, ":white_check_mark: Done")
 		}
-
-	} else if len(dependencies) > 0 {
-		lastHash = dependencies[len(dependencies)-1].Hash.PackageHash
 	}
 
 	if buildTarget {
-		resolvedBuildImage := cs.resolveExistingImageHash(lastHash, p)
+		localGenerateArtifact := true
+		if generateArtifact != nil {
+			localGenerateArtifact = *generateArtifact
+		}
+		resolvedSourceImage := cs.resolveExistingImageHash(packageHashTree.SourceHash, p)
 		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
-		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedBuildImage)
-		a, err := cs.compileWithImage(resolvedBuildImage, cs.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
+		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedSourceImage)
+		a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
 		if err != nil {
 			return a, err
 		}
 		a.Dependencies = departifacts
 		a.SourceAssertion = p.GetSourceAssertion()
-
+		a.PackageCacheImage = targetAssertion.Hash.PackageHash
 		bus.Manager.Publish(bus.EventPackagePostBuild, struct {
 			CompileSpec *compilerspec.LuetCompilationSpec
 			Artifact    *artifact.PackageArtifact

pkg/compiler/compiler_test.go

@@ -75,6 +75,68 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Copy and Join", func() {
+		It("Compiles it correctly with Copy", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/copy")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("result"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("bina/busybox"))).To(BeTrue())
+		})
+
+		It("Compiles it correctly with Join", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/join")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("newc"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
+		})
+	})
+
 	Context("Simple package build definition", func() {
 		It("Compiles it in parallel", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))

pkg/compiler/imagehashtree.go

@@ -0,0 +1,161 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"fmt"
+
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	"github.com/mudler/luet/pkg/config"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	"github.com/pkg/errors"
+)
+
+// ImageHashTree is holding the Database
+// and the options to resolve PackageImageHashTrees
+// for a given specfile
+// It is responsible of returning a concrete result
+// which identifies a Package in a HashTree
+type ImageHashTree struct {
+	Database      pkg.PackageDatabase
+	SolverOptions config.LuetSolverOptions
+}
+
+// PackageImageHashTree represent the Package into a given image hash tree
+// The hash tree is constructed by a set of images representing
+// the package during its build stage. A Hash is assigned to each image
+// from the package fingerprint, plus the SAT solver assertion result (which is hashed as well)
+// and the specfile signatures. This guarantees that each image of the build stage
+// is unique and can be identified later on.
+type PackageImageHashTree struct {
+	Target                       *solver.PackageAssert
+	Dependencies                 solver.PackagesAssertions
+	Solution                     solver.PackagesAssertions
+	dependencyBuilderImageHashes map[string]string
+	SourceHash                   string
+	BuilderImageHash             string
+}
+
+func NewHashTree(db pkg.PackageDatabase) *ImageHashTree {
+	return &ImageHashTree{
+		Database: db,
+	}
+}
+
+func (ht *PackageImageHashTree) DependencyBuildImage(p pkg.Package) (string, error) {
+	found, ok := ht.dependencyBuilderImageHashes[p.GetFingerPrint()]
+	if !ok {
+		return "", errors.New("package hash not found")
+	}
+	return found, nil
+}
+
+func (ht *PackageImageHashTree) String() string {
+	return fmt.Sprintf(
+		"Target buildhash: %s\nTarget packagehash: %s\nBuilder Imagehash: %s\nSource Imagehash: %s\n",
+		ht.Target.Hash.BuildHash,
+		ht.Target.Hash.PackageHash,
+		ht.BuilderImageHash,
+		ht.SourceHash,
+	)
+}
+
+// Query takes a compiler and a compilation spec and returns a PackageImageHashTree tied to it.
+// PackageImageHashTree contains all the informations to resolve the spec build images in order to
+// reproducibly re-build images from packages
+func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) {
+	assertions, err := ht.resolve(cs, p)
+	if err != nil {
+		return nil, err
+	}
+	targetAssertion := assertions.Search(p.GetPackage().GetFingerPrint())
+
+	dependencies := assertions.Drop(p.GetPackage())
+	var sourceHash string
+	imageHashes := map[string]string{}
+	for _, assertion := range dependencies {
+		var depbuildImageTag string
+		compileSpec, err := cs.FromPackage(assertion.Package)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+		}
+		if compileSpec.GetImage() != "" {
+			depbuildImageTag = assertion.Hash.BuildHash
+		} else {
+			depbuildImageTag = ht.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash)
+		}
+		imageHashes[assertion.Package.GetFingerPrint()] = depbuildImageTag
+		sourceHash = assertion.Hash.PackageHash
+	}
+
+	return &PackageImageHashTree{
+		Dependencies:                 dependencies,
+		Target:                       targetAssertion,
+		SourceHash:                   sourceHash,
+		BuilderImageHash:             ht.genBuilderImageTag(p, targetAssertion.Hash.PackageHash),
+		dependencyBuilderImageHashes: imageHashes,
+		Solution:                     assertions,
+	}, nil
+}
+
+func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(packageImage))
+}
+
+// resolve computes the dependency tree of a compilation spec and returns solver assertions
+// in order to be able to compile the spec.
+func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
+	dependencies, err := cs.ComputeDepTree(p)
+	if err != nil {
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
+	}
+
+	// Get hash from buildpsecs
+	salts := map[string]string{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			spec, err := cs.FromPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "while computing hash buildspecs")
+			}
+			hash, err := spec.Hash()
+			if err != nil {
+				return nil, errors.Wrap(err, "failed computing hash")
+			}
+			salts[assertion.Package.GetFingerPrint()] = hash
+		}
+	}
+
+	assertions := solver.PackagesAssertions{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			nthsolution := dependencies.Cut(assertion.Package)
+			assertion.Hash = solver.PackageHash{
+				BuildHash:   nthsolution.SaltedHashFrom(assertion.Package, salts),
+				PackageHash: nthsolution.SaltedAssertionHash(salts),
+			}
+			assertion.Package.SetTreeDir(p.Package.GetTreeDir())
+			assertions = append(assertions, assertion)
+		}
+	}
+
+	return assertions, nil
+}

pkg/compiler/imagehashtree_test.go

@@ -0,0 +1,145 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler_test
+
+import (
+	. "github.com/mudler/luet/pkg/compiler"
+	sd "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ImageHashTree", func() {
+	generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+	compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+	hashtree := NewHashTree(generalRecipe.GetDatabase())
+	Context("Simple package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			err := generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+
+		It("Calculates the hash correctly", func() {
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(packageHash.Target.Hash.BuildHash).To(Equal("4db24406e8db30a3310a1cf8c4d4e19597745e6d41b189dc51a73ac4a50cc9e6"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("4c867c9bab6c71d9420df75806e7a2f171dbc08487852ab4e2487bab04066cf2"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-e6f9c5552a67c463215b0a9e4f7c7fc8"))
+		})
+	})
+
+	expectedPackageHash := "15811d83d0f8360318c54d91dcae3714f8efb39bf872572294834880f00ee7a8"
+
+	Context("complex package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(expectedPackageHash))
+			Expect(packageHash.SourceHash).To(Equal(expectedPackageHash))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-3d739cab442aec15a6da238481df73c5"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("99c4ebb4bc4754985fcc28677badf90f525aa231b1db0fe75659f11b86dc20e8"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+			Expect(assertionA.Hash.PackageHash).To(Equal(expectedPackageHash))
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+			Expect(assertionB.Hash.PackageHash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hashB).To(Equal("2668e418eab6861404834ad617713e39b8e58f68016a1fbfcc9384efdd037376"))
+		})
+	})
+
+	Context("complex package definition, with small change in build.yaml", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			//Definition of A here is slightly changed in the steps build.yaml file (1 character only)
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision_content_changed")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).ToNot(Equal(expectedPackageHash))
+			sourceHash := "1d91b13d0246fa000085a1071c63397d21546300b17f69493f22315a64b717d4"
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(sourceHash))
+			Expect(packageHash.SourceHash).To(Equal(sourceHash))
+			Expect(packageHash.SourceHash).ToNot(Equal(expectedPackageHash))
+
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-03ee108a7c56b17ee568ace0800dd16d"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("7677da23b2cc866c2d07aa4a58fbf703340f2f78c0efbb1ba9faf8979f250c87"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+
+			Expect(assertionA.Hash.PackageHash).To(Equal("1d91b13d0246fa000085a1071c63397d21546300b17f69493f22315a64b717d4"))
+			Expect(assertionA.Hash.PackageHash).ToNot(Equal(expectedPackageHash))
+
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+
+			Expect(assertionB.Hash.PackageHash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hashB).To(Equal("2668e418eab6861404834ad617713e39b8e58f68016a1fbfcc9384efdd037376"))
+		})
+	})
+
+})

pkg/compiler/types/artifact/artifact.go

@@ -19,6 +19,8 @@ import (
 	"archive/tar"
 	"bufio"
 	"bytes"
+	"crypto/sha1"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -53,12 +55,13 @@ import (
 type PackageArtifact struct {
 	Path string `json:"path"`
 
-	Dependencies    []*PackageArtifact                `json:"dependencies"`
-	CompileSpec     *compilerspec.LuetCompilationSpec `json:"compilationspec"`
-	Checksums       Checksums                         `json:"checksums"`
-	SourceAssertion solver.PackagesAssertions         `json:"-"`
-	CompressionType compression.Implementation        `json:"compressiontype"`
-	Files           []string                          `json:"files"`
+	Dependencies      []*PackageArtifact                `json:"dependencies"`
+	CompileSpec       *compilerspec.LuetCompilationSpec `json:"compilationspec"`
+	Checksums         Checksums                         `json:"checksums"`
+	SourceAssertion   solver.PackagesAssertions         `json:"-"`
+	CompressionType   compression.Implementation        `json:"compressiontype"`
+	Files             []string                          `json:"files"`
+	PackageCacheImage string                            `json:"package_cacheimage"`
 }
 
 func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
@@ -316,7 +319,6 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	default:
 		return helpers.Tar(src, a.getCompressedName())
 	}
-	return errors.New("Compression type must be supplied")
 }
 
 func (a *PackageArtifact) getCompressedName() string {
@@ -339,6 +341,28 @@ func (a *PackageArtifact) GetUncompressedName() string {
 	return a.Path
 }
 
+func hashContent(bv []byte) string {
+	hasher := sha1.New()
+	hasher.Write(bv)
+	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	return sha
+}
+
+func hashFileContent(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	h := sha1.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+
+	return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil
+}
+
 func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
 	// If the destination path already exists I rename target file name with postfix.
 	var destPath string
@@ -350,6 +374,7 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return nil, nil, err
 		}
 	}
+	tarHash := hashContent(buffer.Bytes())
 
 	// If file is not present on archive but is defined on mods
 	// I receive the callback. Prevent nil exception.
@@ -362,8 +387,21 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return header, buffer.Bytes(), nil
 		}
 
+		existingHash := ""
+		f, err := os.Lstat(destPath)
+		if err == nil {
+			Debug("File exists already, computing hash for", destPath)
+			hash, herr := hashFileContent(destPath)
+			if herr == nil {
+				existingHash = hash
+			}
+		}
+
+		Debug("Existing file hash: ", existingHash, "Tar file hashsum: ", tarHash)
+		// We want to protect file only if the hash of the files are differing OR the file size are
+		differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
 		// Check if exists
-		if helpers.Exists(destPath) {
+		if helpers.Exists(destPath) && differs {
 			for i := 1; i < 1000; i++ {
 				name := filepath.Join(filepath.Join(filepath.Dir(path),
 					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))

pkg/compiler/types/artifact/artifact_suite_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestArtifact(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Artifact Suite")
+}

pkg/compiler/types/artifact/artifact_test.go

@@ -41,7 +41,7 @@ var _ = Describe("Artifact", func() {
 
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
+			err := generalRecipe.Load("../../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))

pkg/compiler/types/artifact/checksum_test.go

@@ -40,13 +40,13 @@ var _ = Describe("Checksum", func() {
 			Expect(len(definitionsum)).To(Equal(0))
 			Expect(len(definitionsum2)).To(Equal(0))
 
-			err = buildsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/build.yaml"))
+			err = buildsum.Generate(NewPackageArtifact("../../../../tests/fixtures/layers/alpine/build.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = definitionsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			err = definitionsum.Generate(NewPackageArtifact("../../../../tests/fixtures/layers/alpine/definition.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = definitionsum2.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			err = definitionsum2.Generate(NewPackageArtifact("../../../../tests/fixtures/layers/alpine/definition.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(buildsum)).To(Equal(1))

pkg/compiler/types/spec/spec.go

@@ -16,9 +16,11 @@
 package compilerspec
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
 
+	"github.com/mitchellh/hashstructure/v2"
 	options "github.com/mudler/luet/pkg/compiler/types/options"
 
 	pkg "github.com/mudler/luet/pkg/package"
@@ -85,6 +87,13 @@ func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
 	return &newSpecs
 }
 
+type CopyField struct {
+	Package     *pkg.DefaultPackage `json:"package"`
+	Image       string              `json:"image"`
+	Source      string              `json:"source"`
+	Destination string              `json:"destination"`
+}
+
 type LuetCompilationSpec struct {
 	Steps           []string                  `json:"steps"` // Are run inside a container and the result layer diff is saved
 	Env             []string                  `json:"env"`
@@ -103,6 +112,43 @@ type LuetCompilationSpec struct {
 	Excludes   []string `json:"excludes"`
 
 	BuildOptions *options.Compiler `json:"build_options"`
+
+	Copy []CopyField `json:"copy"`
+
+	Join pkg.DefaultPackages `json:"join"`
+}
+
+// Signature is a portion of the spec that yields a signature for the hash
+type Signature struct {
+	Image      string
+	Steps      []string
+	PackageDir string
+	Prelude    []string
+	Seed       string
+	Env        []string
+	Retrieve   []string
+	Unpack     bool
+	Includes   []string
+	Excludes   []string
+	Copy       []CopyField
+	Join       pkg.DefaultPackages
+}
+
+func (cs *LuetCompilationSpec) signature() Signature {
+	return Signature{
+		Image:      cs.Image,
+		Steps:      cs.Steps,
+		PackageDir: cs.PackageDir,
+		Prelude:    cs.Prelude,
+		Seed:       cs.Seed,
+		Env:        cs.Env,
+		Retrieve:   cs.Retrieve,
+		Unpack:     cs.Unpack,
+		Includes:   cs.Includes,
+		Excludes:   cs.Excludes,
+		Copy:       cs.Copy,
+		Join:       cs.Join,
+	}
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
@@ -213,7 +259,14 @@ func (cs *LuetCompilationSpec) UnpackedPackage() bool {
 // a compilation spec has an image source when it depends on other packages or have a source image
 // explictly supplied
 func (cs *LuetCompilationSpec) HasImageSource() bool {
-	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != ""
+	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != "" || len(cs.Join) != 0
+}
+
+func (cs *LuetCompilationSpec) Hash() (string, error) {
+	// build a signature, we want to be part of the hash only the fields that are relevant for build purposes
+	signature := cs.signature()
+	h, err := hashstructure.Hash(signature, hashstructure.FormatV2, nil)
+	return fmt.Sprint(h), err
 }
 
 func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
@@ -256,6 +309,13 @@ ADD ` + s + ` /luetbuild/`
 		}
 	}
 
+	for _, c := range cs.Copy {
+		if c.Image != "" {
+			copyLine := fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
+			spec = spec + copyLine
+		}
+	}
+
 	for _, s := range cs.Env {
 		spec = spec + `
 ENV ` + s

pkg/compiler/types/spec/spec_suite_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compilerspec_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSpec(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Spec Suite")
+}

pkg/compiler/types/spec/spec_test.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
+	options "github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 
 	. "github.com/mudler/luet/pkg/compiler"
@@ -74,11 +75,67 @@ var _ = Describe("Spec", func() {
 		})
 	})
 
+	Context("Image hashing", func() {
+		It("is stable", func() {
+			spec1 := &compilerspec.LuetCompilationSpec{
+				Image:        "foo",
+				BuildOptions: &options.Compiler{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
+
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			spec2 := &compilerspec.LuetCompilationSpec{
+				Image:        "foo",
+				BuildOptions: &options.Compiler{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			spec3 := &compilerspec.LuetCompilationSpec{
+				Image: "foo",
+				Steps: []string{"foo"},
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			hash, err := spec1.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			hash2, err := spec2.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			hash3, err := spec3.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal(hash2))
+			hashagain, err := spec2.Hash()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hash).ToNot(Equal(hash3))
+			Expect(hash).To(Equal(hashagain))
+		})
+	})
+
 	Context("Simple package build definition", func() {
 		It("Loads it correctly", func() {
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
+			err := generalRecipe.Load("../../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
@@ -130,7 +187,7 @@ RUN echo bar > /test2`))
 	It("Renders retrieve and env fields", func() {
 		generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-		err := generalRecipe.Load("../../tests/fixtures/retrieve")
+		err := generalRecipe.Load("../../../../tests/fixtures/retrieve")
 		Expect(err).ToNot(HaveOccurred())
 
 		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))

pkg/helpers/archive.go

@@ -90,8 +90,7 @@ func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modi
 	}
 
 	if sameOwner {
-		// PRE: i have root privileged.
-
+		// we do have root permissions, so we can extract keeping the same permissions.
 		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
 
 		opts := &archive.TarOptions{

pkg/package/package.go

@@ -118,6 +118,8 @@ type Package interface {
 	SetTreeDir(s string)
 	GetTreeDir() string
 
+	Mark() Package
+
 	JSON() ([]byte, error)
 }
 
@@ -131,6 +133,19 @@ type Tree interface {
 
 type Packages []Package
 
+type DefaultPackages []*DefaultPackage
+
+func (d DefaultPackages) Hash(salt string) string {
+
+	overallFp := ""
+	for _, c := range d {
+		overallFp = overallFp + c.HashFingerprint("join")
+	}
+	h := md5.New()
+	io.WriteString(h, fmt.Sprintf("%s-%s", overallFp, salt))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
 // >> Unmarshallers
 // DefaultPackageFromYaml decodes a package from yaml bytes
 func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) {
@@ -492,6 +507,12 @@ func (p *DefaultPackage) Matches(m Package) bool {
 	return false
 }
 
+func (p *DefaultPackage) Mark() Package {
+	marked := p.Clone()
+	marked.SetName("@@" + marked.GetName())
+	return marked
+}
+
 func (p *DefaultPackage) Expand(definitiondb PackageDatabase) (Packages, error) {
 	var versionsInWorld Packages
 

pkg/solver/decoder.go

@@ -260,24 +260,42 @@ func (a PackagesAssertions) TrueLen() int {
 // and checks it's not the only one. if it's unique it marks it specially - so the hash
 // which is generated is unique for the selected package
 func (assertions PackagesAssertions) HashFrom(p pkg.Package) string {
+	return assertions.SaltedHashFrom(p, map[string]string{})
+}
 
+func (assertions PackagesAssertions) AssertionHash() string {
+	return assertions.SaltedAssertionHash(map[string]string{})
+}
+
+func (assertions PackagesAssertions) SaltedHashFrom(p pkg.Package, salts map[string]string) string {
 	var assertionhash string
 
 	// When we don't have any solution to hash for, we need to generate an UUID by ourselves
 	latestsolution := assertions.Drop(p)
 	if latestsolution.TrueLen() == 0 {
-		assertionhash = assertions.Mark(p).AssertionHash()
+		// Preserve the hash if supplied of marked packages
+		marked := p.Mark()
+		if markedHash, exists := salts[p.GetFingerPrint()]; exists {
+			salts[marked.GetFingerPrint()] = markedHash
+		}
+		assertionhash = assertions.Mark(p).SaltedAssertionHash(salts)
 	} else {
-		assertionhash = latestsolution.AssertionHash()
+		assertionhash = latestsolution.SaltedAssertionHash(salts)
 	}
 	return assertionhash
 }
 
-func (assertions PackagesAssertions) AssertionHash() string {
+func (assertions PackagesAssertions) SaltedAssertionHash(salts map[string]string) string {
 	var fingerprint string
 	for _, assertion := range assertions { // Note: Always order them first!
 		if assertion.Value { // Tke into account only dependencies installed (get fingerprint of subgraph)
-			fingerprint += assertion.ToString() + "\n"
+			salt, exists := salts[assertion.Package.GetFingerPrint()]
+			if exists {
+				fingerprint += assertion.ToString() + salt + "\n"
+
+			} else {
+				fingerprint += assertion.ToString() + "\n"
+			}
 		}
 	}
 	hash := sha256.Sum256([]byte(fingerprint))
@@ -316,8 +334,7 @@ func (assertions PackagesAssertions) Mark(p pkg.Package) PackagesAssertions {
 
 	for _, a := range assertions {
 		if a.Package.Matches(p) {
-			marked := a.Package.Clone()
-			marked.SetName("@@" + marked.GetName())
+			marked := a.Package.Mark()
 			a = PackageAssert{Package: marked.(*pkg.DefaultPackage), Value: a.Value, Hash: a.Hash}
 		}
 		ass = append(ass, a)

pkg/solver/decoder_test.go

@@ -382,6 +382,9 @@ var _ = Describe("Decoder", func() {
 
 			Expect(solution.HashFrom(X)).ToNot(Equal(solution2.HashFrom(F)))
 			Expect(solution3.HashFrom(D)).To(Equal(solution.HashFrom(X)))
+			Expect(solution3.SaltedHashFrom(D, map[string]string{D.GetFingerPrint(): "foo"})).ToNot(Equal(solution3.HashFrom(D)))
+
+			Expect(solution4.SaltedHashFrom(Y, map[string]string{X.GetFingerPrint(): "foo"})).ToNot(Equal(solution4.HashFrom(Y)))
 
 			Expect(empty.AssertionHash()).ToNot(Equal(solution3.HashFrom(D)))
 			Expect(empty.AssertionHash()).ToNot(Equal(solution2.HashFrom(F)))

tests/fixtures/copy/c/a/build.yaml

@@ -0,0 +1,17 @@
+image: "alpine"
+
+copy:
+  - package:
+      name: "a"
+      category: "test"
+      version: ">=0"
+    source: /test3
+    destination: /test3
+  - image: "busybox"
+    source: /bin/busybox
+    destination: /busybox
+
+steps:
+- mkdir /bina
+- cp /test3 /result
+- cp -rf /busybox /bina/busybox

tests/fixtures/copy/c/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.2"

tests/fixtures/copy/cat/a/a/build.yaml

@@ -0,0 +1,7 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/copy/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/copy/cat/b-1.1/build.yaml

@@ -0,0 +1,13 @@
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0"
+
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/copy/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/copy/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/fixtures/join/c/a/build.yaml

@@ -0,0 +1,8 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"
+unpack: true

tests/fixtures/join/c/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.2"

tests/fixtures/join/cat/a/a/build.yaml

@@ -0,0 +1,7 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/join/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/join/cat/b-1.1/build.yaml

@@ -0,0 +1,13 @@
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0"
+
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/join/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/join/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/fixtures/join_complex/c/c1/build.yaml

@@ -0,0 +1,7 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"

tests/fixtures/join_complex/c/c1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.2"

tests/fixtures/join_complex/c/f/build.yaml

@@ -0,0 +1,8 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"
+unpack: true

tests/fixtures/join_complex/c/f/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "f"
+version: "1.2"

tests/fixtures/join_complex/cat/a/a/build.yaml

@@ -0,0 +1,2 @@
+image: "alpine"
+unpack: true

tests/fixtures/join_complex/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/join_complex/cat/b-1.1/build.yaml

@@ -0,0 +1,13 @@
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0"
+
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/join_complex/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/join_complex/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/fixtures/join_complex/x/build.yaml

@@ -0,0 +1,10 @@
+join:
+- category: "test"
+  name: "f"
+  version: ">=0"
+# this is actually a virtual, no files shipped in c
+- category: "test"
+  name: "c"
+  version: ">=0"
+steps:
+- mv /newnewc /x

tests/fixtures/join_complex/x/definition.yaml

@@ -0,0 +1,3 @@
+name: "x"
+category: "test"
+version: "0.1"

tests/fixtures/join_complex/z/build.yaml

@@ -0,0 +1,6 @@
+requires:
+- category: "test"
+  name: "c"
+  version: ">=0"
+steps:
+- mv /newnewc /z

tests/fixtures/join_complex/z/definition.yaml

@@ -0,0 +1,3 @@
+name: "z"
+category: "test"
+version: "0.1"

tests/fixtures/plugin/test-foo

@@ -1,3 +1,5 @@
 #!/bin/bash
 echo "$1" >> $EVENT_FILE
-echo "$2" >> $PAYLOAD_FILE
+echo "$2" >> $PAYLOAD_FILE
+
+echo "{}"

tests/fixtures/upgrade_old_repo_revision_content_changed/c/build.yaml

@@ -0,0 +1,10 @@
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo c > /c
+  - echo c > /cd
+requires:
+- category: "test"
+  name: "a"
+  version: ">=1.0"

tests/fixtures/upgrade_old_repo_revision_content_changed/c/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.0"

tests/fixtures/upgrade_old_repo_revision_content_changed/cat/a/a/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+prelude:
+  - echo fozo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/upgrade_old_repo_revision_content_changed/cat/a/a/definition.yaml

@@ -0,0 +1,8 @@
+category: "test"
+name: "a"
+version: "1.1"
+requires:
+- category: "test2"
+  name: "b"
+  version: "1.0"
+

tests/fixtures/upgrade_old_repo_revision_content_changed/cat/b-1.1/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/upgrade_old_repo_revision_content_changed/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/fixtures/upgrade_old_repo_revision_content_changed/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/integration/12_config_protect_samefile.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testrootfs/testbuild -p
+    luet build --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+      --destination $tmpdir/testrootfs/testbuild --compression gzip test/a
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testrootfs/testbuild/a-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+    --output $tmpdir/testrootfs/testbuild \
+    --packages $tmpdir/testrootfs/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testrootfs/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+
+    mkdir $tmpdir/testrootfs/etc/luet/config.protect.d -p
+
+    cat <<EOF > $tmpdir/testrootfs/etc/luet/config.protect.d/conf1.yml
+name: "protect1"
+dirs:
+- /etc/
+EOF
+
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_protect_confdir:
+    - /etc/luet/config.protect.d
+config_from_host: false
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+
+
+testInstall() {
+
+    # Simulate previous installation
+    mkdir $tmpdir/testrootfs/etc/a -p
+    echo config > $tmpdir/testrootfs/etc/a/conf
+
+    luet install -y --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+
+    # Simulate config protect
+    assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect not created, file is the same' "[ ! -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
+    assertEquals 'config protect content' "$(cat $tmpdir/testrootfs/etc/a/conf)" "config"
+}
+
+
+testUnInstall() {
+    luet uninstall -y --full --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect maintains the protected files' "[ -e '$tmpdir/testrootfs/etc/a/conf' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/a-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/28_nobuildtreedocker.sh

@@ -51,9 +51,9 @@ testBuild() {
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:d620e573c81eab36a9dc5cc314e80fd7b6e04aeff26127de4225bf24fe1f8e71"
-    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:b0f34b0d2d271f0f2619324476b2857b3b39ca895bddc2474a741f3c8c1acbbc"
-    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:c1f11f48113cd71d8795a06c7b49e1558bd7211d2aa88f5d79a3334f0393c64d"
+    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available remotely for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:ac34beb3fc2752bab54549db095dd6994d7531b88e1ff7f902d01ae80fdd030d"
+    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available remotely for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:d8b8881d50ae83646728bb28ad678fc14e4e003e4b0d0a66f8e6167c6116e024"
+    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2aaccd929ebe5683f95c70c6ec8d68f240ceea8633f3350b94904ad73da5fd47"
 }
 
 testRepo() {

tests/integration/29_nobuildtreeinheritvalues.sh

@@ -45,7 +45,7 @@ EOF
     mkdir $tmpdir/testbuild
     mkdir $tmpdir/empty
 
-    # With --rebuild, the package gets ignored
+    # Without --rebuild, the package gets ignored
     build_output=$(luet build --pull --tree "$tmpdir/empty" \
     --config $tmpdir/luet.yaml --values $tmpdir/default.yaml --concurrency 1 \
     --from-repositories --destination $tmpdir/testbuild --compression zstd test/c@1.0 test/z test/interpolated)
@@ -56,7 +56,8 @@ EOF
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Images available for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:c1f11f48113cd71d8795a06c7b49e1558bd7211d2aa88f5d79a3334f0393c64d"
+    assertNotContains 'Does NOT use the upstream cache without specifying it' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2aaccd929ebe5683f95c70c6ec8d68f240ceea8633f3350b94904ad73da5fd47"
+    assertContains 'Does generate a new hash as values changed build.yaml for test/interpolated-1.0+2 package image' "$build_output" "Building image luet/cache:c34b533cf76886c332fe9b2d3208f04265360a465a90c996cb4fcdaf959dee36 done"
 }
 
 testRepo() {
@@ -113,7 +114,7 @@ testInstall() {
     assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
     assertTrue 'package Z installed' "[ -e '$tmpdir/testrootfs/z' ]"
     ls -liah $tmpdir/testrootfs/
-    assertTrue 'package interpolated installed' "[ -e '$tmpdir/testrootfs/interpolated-baz-bar' ]"
+    assertTrue 'package interpolated installed' "[ -e '$tmpdir/testrootfs/interpolated-baz-an' ]"
 }
 
 testReInstall() {

tests/integration/29_nobuildtreeinheritvalues_noignored.sh

@@ -56,7 +56,7 @@ EOF
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Downloading image quay.io/mocaccinoos/integration-test-cache:6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"
+    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Downloading image quay.io/mocaccinoos/integration-test-cache:4db24406e8db30a3310a1cf8c4d4e19597745e6d41b189dc51a73ac4a50cc9e6"
 }
 
 testRepo() {

tests/integration/30_copy.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    cat <<EOF > $tmpdir/default.yaml
+extra: "bar"
+foo: "baz"
+EOF
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/copy" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --compression gzip --values $tmpdir/default.yaml \
+               test/c
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package c' "[ -e '$tmpdir/testbuild/c-test-1.2.package.tar.gz' ]"
+    mkdir $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/c-test-1.2.package.tar.gz -C $tmpdir/extract
+    assertTrue 'create result in package c' "[ -e '$tmpdir/extract/result' ]"
+    assertTrue 'create busybox in package c' "[ -f '$tmpdir/extract/bina/busybox' ]"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/31_join.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    cat <<EOF > $tmpdir/default.yaml
+extra: "bar"
+foo: "baz"
+EOF
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/join" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --compression gzip --values $tmpdir/default.yaml \
+               test/c
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package c' "[ -e '$tmpdir/testbuild/c-test-1.2.package.tar.gz' ]"
+    mkdir $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/c-test-1.2.package.tar.gz -C $tmpdir/extract
+    assertTrue 'create result from join' "[ -e '$tmpdir/extract/test3' ]"
+    assertTrue 'create result from join' "[ -f '$tmpdir/extract/newc' ]"
+    assertTrue 'create result from join' "[ -e '$tmpdir/extract/test4' ]"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/31_join_complex.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/join_complex" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --compression gzip \
+               test/z test/x
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package z' "[ -e '$tmpdir/testbuild/z-test-0.1.package.tar.gz' ]"
+    assertTrue 'create package z' "[ -e '$tmpdir/testbuild/x-test-0.1.package.tar.gz' ]"
+
+    mkdir $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/x-test-0.1.package.tar.gz -C $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/z-test-0.1.package.tar.gz -C $tmpdir/extract
+    assertTrue 'create result from a package that requires a join' "[ -e '$tmpdir/extract/z' ]"
+    assertTrue 'create result from join of a join' "[ -e '$tmpdir/extract/x' ]"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-
+export LUET_NO_SPINNER=true
 export LUET_YES=true
 export ROOT_DIR="$(git rev-parse --show-toplevel)"
 

vendor/github.com/codegangsta/inject/.gitignore

@@ -1,2 +0,0 @@
-inject
-inject.test

vendor/github.com/codegangsta/inject/LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2013 Jeremy Saenz
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/codegangsta/inject/README.md

@@ -1,92 +0,0 @@
-# inject
---
-    import "github.com/codegangsta/inject"
-
-Package inject provides utilities for mapping and injecting dependencies in
-various ways.
-
-Language Translations:
-* [简体中文](translations/README_zh_cn.md)
-
-## Usage
-
-#### func  InterfaceOf
-
-```go
-func InterfaceOf(value interface{}) reflect.Type
-```
-InterfaceOf dereferences a pointer to an Interface type. It panics if value is
-not an pointer to an interface.
-
-#### type Applicator
-
-```go
-type Applicator interface {
-	// Maps dependencies in the Type map to each field in the struct
-	// that is tagged with 'inject'. Returns an error if the injection
-	// fails.
-	Apply(interface{}) error
-}
-```
-
-Applicator represents an interface for mapping dependencies to a struct.
-
-#### type Injector
-
-```go
-type Injector interface {
-	Applicator
-	Invoker
-	TypeMapper
-	// SetParent sets the parent of the injector. If the injector cannot find a
-	// dependency in its Type map it will check its parent before returning an
-	// error.
-	SetParent(Injector)
-}
-```
-
-Injector represents an interface for mapping and injecting dependencies into
-structs and function arguments.
-
-#### func  New
-
-```go
-func New() Injector
-```
-New returns a new Injector.
-
-#### type Invoker
-
-```go
-type Invoker interface {
-	// Invoke attempts to call the interface{} provided as a function,
-	// providing dependencies for function arguments based on Type. Returns
-	// a slice of reflect.Value representing the returned values of the function.
-	// Returns an error if the injection fails.
-	Invoke(interface{}) ([]reflect.Value, error)
-}
-```
-
-Invoker represents an interface for calling functions via reflection.
-
-#### type TypeMapper
-
-```go
-type TypeMapper interface {
-	// Maps the interface{} value based on its immediate type from reflect.TypeOf.
-	Map(interface{}) TypeMapper
-	// Maps the interface{} value based on the pointer of an Interface provided.
-	// This is really only useful for mapping a value as an interface, as interfaces
-	// cannot at this time be referenced directly without a pointer.
-	MapTo(interface{}, interface{}) TypeMapper
-	// Provides a possibility to directly insert a mapping based on type and value.
-	// This makes it possible to directly map type arguments not possible to instantiate
-	// with reflect like unidirectional channels.
-	Set(reflect.Type, reflect.Value) TypeMapper
-	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
-	// the Type has not been mapped.
-	Get(reflect.Type) reflect.Value
-}
-```
-
-TypeMapper represents an interface for mapping interface{} values based on type.

vendor/github.com/codegangsta/inject/inject.go

@@ -1,187 +0,0 @@
-// Package inject provides utilities for mapping and injecting dependencies in various ways.
-package inject
-
-import (
-	"fmt"
-	"reflect"
-)
-
-// Injector represents an interface for mapping and injecting dependencies into structs
-// and function arguments.
-type Injector interface {
-	Applicator
-	Invoker
-	TypeMapper
-	// SetParent sets the parent of the injector. If the injector cannot find a
-	// dependency in its Type map it will check its parent before returning an
-	// error.
-	SetParent(Injector)
-}
-
-// Applicator represents an interface for mapping dependencies to a struct.
-type Applicator interface {
-	// Maps dependencies in the Type map to each field in the struct
-	// that is tagged with 'inject'. Returns an error if the injection
-	// fails.
-	Apply(interface{}) error
-}
-
-// Invoker represents an interface for calling functions via reflection.
-type Invoker interface {
-	// Invoke attempts to call the interface{} provided as a function,
-	// providing dependencies for function arguments based on Type. Returns
-	// a slice of reflect.Value representing the returned values of the function.
-	// Returns an error if the injection fails.
-	Invoke(interface{}) ([]reflect.Value, error)
-}
-
-// TypeMapper represents an interface for mapping interface{} values based on type.
-type TypeMapper interface {
-	// Maps the interface{} value based on its immediate type from reflect.TypeOf.
-	Map(interface{}) TypeMapper
-	// Maps the interface{} value based on the pointer of an Interface provided.
-	// This is really only useful for mapping a value as an interface, as interfaces
-	// cannot at this time be referenced directly without a pointer.
-	MapTo(interface{}, interface{}) TypeMapper
-	// Provides a possibility to directly insert a mapping based on type and value.
-	// This makes it possible to directly map type arguments not possible to instantiate
-	// with reflect like unidirectional channels.
-	Set(reflect.Type, reflect.Value) TypeMapper
-	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
-	// the Type has not been mapped.
-	Get(reflect.Type) reflect.Value
-}
-
-type injector struct {
-	values map[reflect.Type]reflect.Value
-	parent Injector
-}
-
-// InterfaceOf dereferences a pointer to an Interface type.
-// It panics if value is not an pointer to an interface.
-func InterfaceOf(value interface{}) reflect.Type {
-	t := reflect.TypeOf(value)
-
-	for t.Kind() == reflect.Ptr {
-		t = t.Elem()
-	}
-
-	if t.Kind() != reflect.Interface {
-		panic("Called inject.InterfaceOf with a value that is not a pointer to an interface. (*MyInterface)(nil)")
-	}
-
-	return t
-}
-
-// New returns a new Injector.
-func New() Injector {
-	return &injector{
-		values: make(map[reflect.Type]reflect.Value),
-	}
-}
-
-// Invoke attempts to call the interface{} provided as a function,
-// providing dependencies for function arguments based on Type.
-// Returns a slice of reflect.Value representing the returned values of the function.
-// Returns an error if the injection fails.
-// It panics if f is not a function
-func (inj *injector) Invoke(f interface{}) ([]reflect.Value, error) {
-	t := reflect.TypeOf(f)
-
-	var in = make([]reflect.Value, t.NumIn()) //Panic if t is not kind of Func
-	for i := 0; i < t.NumIn(); i++ {
-		argType := t.In(i)
-		val := inj.Get(argType)
-		if !val.IsValid() {
-			return nil, fmt.Errorf("Value not found for type %v", argType)
-		}
-
-		in[i] = val
-	}
-
-	return reflect.ValueOf(f).Call(in), nil
-}
-
-// Maps dependencies in the Type map to each field in the struct
-// that is tagged with 'inject'.
-// Returns an error if the injection fails.
-func (inj *injector) Apply(val interface{}) error {
-	v := reflect.ValueOf(val)
-
-	for v.Kind() == reflect.Ptr {
-		v = v.Elem()
-	}
-
-	if v.Kind() != reflect.Struct {
-		return nil // Should not panic here ?
-	}
-
-	t := v.Type()
-
-	for i := 0; i < v.NumField(); i++ {
-		f := v.Field(i)
-		structField := t.Field(i)
-		if f.CanSet() && (structField.Tag == "inject" || structField.Tag.Get("inject") != "") {
-			ft := f.Type()
-			v := inj.Get(ft)
-			if !v.IsValid() {
-				return fmt.Errorf("Value not found for type %v", ft)
-			}
-
-			f.Set(v)
-		}
-
-	}
-
-	return nil
-}
-
-// Maps the concrete value of val to its dynamic type using reflect.TypeOf,
-// It returns the TypeMapper registered in.
-func (i *injector) Map(val interface{}) TypeMapper {
-	i.values[reflect.TypeOf(val)] = reflect.ValueOf(val)
-	return i
-}
-
-func (i *injector) MapTo(val interface{}, ifacePtr interface{}) TypeMapper {
-	i.values[InterfaceOf(ifacePtr)] = reflect.ValueOf(val)
-	return i
-}
-
-// Maps the given reflect.Type to the given reflect.Value and returns
-// the Typemapper the mapping has been registered in.
-func (i *injector) Set(typ reflect.Type, val reflect.Value) TypeMapper {
-	i.values[typ] = val
-	return i
-}
-
-func (i *injector) Get(t reflect.Type) reflect.Value {
-	val := i.values[t]
-
-	if val.IsValid() {
-		return val
-	}
-
-	// no concrete types found, try to find implementors
-	// if t is an interface
-	if t.Kind() == reflect.Interface {
-		for k, v := range i.values {
-			if k.Implements(t) {
-				val = v
-				break
-			}
-		}
-	}
-
-	// Still no type found, try to look it up on the parent
-	if !val.IsValid() && i.parent != nil {
-		val = i.parent.Get(t)
-	}
-
-	return val
-
-}
-
-func (i *injector) SetParent(parent Injector) {
-	i.parent = parent
-}

vendor/github.com/codegangsta/inject/update_readme.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-go get github.com/robertkrimen/godocdown/godocdown
-godocdown > README.md

vendor/github.com/mitchellh/hashstructure/v2/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Mitchell Hashimoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/mitchellh/hashstructure/v2/README.md

@@ -0,0 +1,76 @@
+# hashstructure [![GoDoc](https://godoc.org/github.com/mitchellh/hashstructure?status.svg)](https://godoc.org/github.com/mitchellh/hashstructure)
+
+hashstructure is a Go library for creating a unique hash value
+for arbitrary values in Go.
+
+This can be used to key values in a hash (for use in a map, set, etc.)
+that are complex. The most common use case is comparing two values without
+sending data across the network, caching values locally (de-dup), and so on.
+
+## Features
+
+  * Hash any arbitrary Go value, including complex types.
+
+  * Tag a struct field to ignore it and not affect the hash value.
+
+  * Tag a slice type struct field to treat it as a set where ordering
+    doesn't affect the hash code but the field itself is still taken into
+    account to create the hash value.
+
+  * Optionally, specify a custom hash function to optimize for speed, collision
+    avoidance for your data set, etc.
+
+  * Optionally, hash the output of `.String()` on structs that implement fmt.Stringer,
+    allowing effective hashing of time.Time
+
+  * Optionally, override the hashing process by implementing `Hashable`.
+
+## Installation
+
+Standard `go get`:
+
+```
+$ go get github.com/mitchellh/hashstructure/v2
+```
+
+**Note on v2:** It is highly recommended you use the "v2" release since this
+fixes some significant hash collisions issues from v1. In practice, we used
+v1 for many years in real projects at HashiCorp and never had issues, but it
+is highly dependent on the shape of the data you're hashing and how you use
+those hashes.
+
+When using v2+, you can still generate weaker v1 hashes by using the
+`FormatV1` format when calling `Hash`.
+
+## Usage & Example
+
+For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/hashstructure).
+
+A quick code example is shown below:
+
+```go
+type ComplexStruct struct {
+    Name     string
+    Age      uint
+    Metadata map[string]interface{}
+}
+
+v := ComplexStruct{
+    Name: "mitchellh",
+    Age:  64,
+    Metadata: map[string]interface{}{
+        "car":      true,
+        "location": "California",
+        "siblings": []string{"Bob", "John"},
+    },
+}
+
+hash, err := hashstructure.Hash(v, nil)
+if err != nil {
+    panic(err)
+}
+
+fmt.Printf("%d", hash)
+// Output:
+// 2307517237273902113
+```

vendor/github.com/mitchellh/hashstructure/v2/errors.go

@@ -0,0 +1,22 @@
+package hashstructure
+
+import (
+	"fmt"
+)
+
+// ErrNotStringer is returned when there's an error with hash:"string"
+type ErrNotStringer struct {
+	Field string
+}
+
+// Error implements error for ErrNotStringer
+func (ens *ErrNotStringer) Error() string {
+	return fmt.Sprintf("hashstructure: %s has hash:\"string\" set, but does not implement fmt.Stringer", ens.Field)
+}
+
+// ErrFormat is returned when an invalid format is given to the Hash function.
+type ErrFormat struct{}
+
+func (*ErrFormat) Error() string {
+	return "format must be one of the defined Format values in the hashstructure library"
+}

vendor/github.com/mitchellh/hashstructure/v2/go.mod

@@ -0,0 +1,3 @@
+module github.com/mitchellh/hashstructure/v2
+
+go 1.14

vendor/github.com/mitchellh/hashstructure/v2/hashstructure.go

@@ -0,0 +1,483 @@
+package hashstructure
+
+import (
+	"encoding/binary"
+	"fmt"
+	"hash"
+	"hash/fnv"
+	"reflect"
+	"time"
+)
+
+// HashOptions are options that are available for hashing.
+type HashOptions struct {
+	// Hasher is the hash function to use. If this isn't set, it will
+	// default to FNV.
+	Hasher hash.Hash64
+
+	// TagName is the struct tag to look at when hashing the structure.
+	// By default this is "hash".
+	TagName string
+
+	// ZeroNil is flag determining if nil pointer should be treated equal
+	// to a zero value of pointed type. By default this is false.
+	ZeroNil bool
+
+	// IgnoreZeroValue is determining if zero value fields should be
+	// ignored for hash calculation.
+	IgnoreZeroValue bool
+
+	// SlicesAsSets assumes that a `set` tag is always present for slices.
+	// Default is false (in which case the tag is used instead)
+	SlicesAsSets bool
+
+	// UseStringer will attempt to use fmt.Stringer aways. If the struct
+	// doesn't implement fmt.Stringer, it'll fall back to trying usual tricks.
+	// If this is true, and the "string" tag is also set, the tag takes
+	// precedense (meaning that if the type doesn't implement fmt.Stringer, we
+	// panic)
+	UseStringer bool
+}
+
+// Format specifies the hashing process used. Different formats typically
+// generate different hashes for the same value and have different properties.
+type Format uint
+
+const (
+	// To disallow the zero value
+	formatInvalid Format = iota
+
+	// FormatV1 is the format used in v1.x of this library. This has the
+	// downsides noted in issue #18 but allows simultaneous v1/v2 usage.
+	FormatV1
+
+	// FormatV2 is the current recommended format and fixes the issues
+	// noted in FormatV1.
+	FormatV2
+
+	formatMax // so we can easily find the end
+)
+
+// Hash returns the hash value of an arbitrary value.
+//
+// If opts is nil, then default options will be used. See HashOptions
+// for the default values. The same *HashOptions value cannot be used
+// concurrently. None of the values within a *HashOptions struct are
+// safe to read/write while hashing is being done.
+//
+// The "format" is required and must be one of the format values defined
+// by this library. You should probably just use "FormatV2". This allows
+// generated hashes uses alternate logic to maintain compatibility with
+// older versions.
+//
+// Notes on the value:
+//
+//   * Unexported fields on structs are ignored and do not affect the
+//     hash value.
+//
+//   * Adding an exported field to a struct with the zero value will change
+//     the hash value.
+//
+// For structs, the hashing can be controlled using tags. For example:
+//
+//    struct {
+//        Name string
+//        UUID string `hash:"ignore"`
+//    }
+//
+// The available tag values are:
+//
+//   * "ignore" or "-" - The field will be ignored and not affect the hash code.
+//
+//   * "set" - The field will be treated as a set, where ordering doesn't
+//             affect the hash code. This only works for slices.
+//
+//   * "string" - The field will be hashed as a string, only works when the
+//                field implements fmt.Stringer
+//
+func Hash(v interface{}, format Format, opts *HashOptions) (uint64, error) {
+	// Validate our format
+	if format <= formatInvalid || format >= formatMax {
+		return 0, &ErrFormat{}
+	}
+
+	// Create default options
+	if opts == nil {
+		opts = &HashOptions{}
+	}
+	if opts.Hasher == nil {
+		opts.Hasher = fnv.New64()
+	}
+	if opts.TagName == "" {
+		opts.TagName = "hash"
+	}
+
+	// Reset the hash
+	opts.Hasher.Reset()
+
+	// Create our walker and walk the structure
+	w := &walker{
+		format:          format,
+		h:               opts.Hasher,
+		tag:             opts.TagName,
+		zeronil:         opts.ZeroNil,
+		ignorezerovalue: opts.IgnoreZeroValue,
+		sets:            opts.SlicesAsSets,
+		stringer:        opts.UseStringer,
+	}
+	return w.visit(reflect.ValueOf(v), nil)
+}
+
+type walker struct {
+	format          Format
+	h               hash.Hash64
+	tag             string
+	zeronil         bool
+	ignorezerovalue bool
+	sets            bool
+	stringer        bool
+}
+
+type visitOpts struct {
+	// Flags are a bitmask of flags to affect behavior of this visit
+	Flags visitFlag
+
+	// Information about the struct containing this field
+	Struct      interface{}
+	StructField string
+}
+
+var timeType = reflect.TypeOf(time.Time{})
+
+func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
+	t := reflect.TypeOf(0)
+
+	// Loop since these can be wrapped in multiple layers of pointers
+	// and interfaces.
+	for {
+		// If we have an interface, dereference it. We have to do this up
+		// here because it might be a nil in there and the check below must
+		// catch that.
+		if v.Kind() == reflect.Interface {
+			v = v.Elem()
+			continue
+		}
+
+		if v.Kind() == reflect.Ptr {
+			if w.zeronil {
+				t = v.Type().Elem()
+			}
+			v = reflect.Indirect(v)
+			continue
+		}
+
+		break
+	}
+
+	// If it is nil, treat it like a zero.
+	if !v.IsValid() {
+		v = reflect.Zero(t)
+	}
+
+	// Binary writing can use raw ints, we have to convert to
+	// a sized-int, we'll choose the largest...
+	switch v.Kind() {
+	case reflect.Int:
+		v = reflect.ValueOf(int64(v.Int()))
+	case reflect.Uint:
+		v = reflect.ValueOf(uint64(v.Uint()))
+	case reflect.Bool:
+		var tmp int8
+		if v.Bool() {
+			tmp = 1
+		}
+		v = reflect.ValueOf(tmp)
+	}
+
+	k := v.Kind()
+
+	// We can shortcut numeric values by directly binary writing them
+	if k >= reflect.Int && k <= reflect.Complex64 {
+		// A direct hash calculation
+		w.h.Reset()
+		err := binary.Write(w.h, binary.LittleEndian, v.Interface())
+		return w.h.Sum64(), err
+	}
+
+	switch v.Type() {
+	case timeType:
+		w.h.Reset()
+		b, err := v.Interface().(time.Time).MarshalBinary()
+		if err != nil {
+			return 0, err
+		}
+
+		err = binary.Write(w.h, binary.LittleEndian, b)
+		return w.h.Sum64(), err
+	}
+
+	switch k {
+	case reflect.Array:
+		var h uint64
+		l := v.Len()
+		for i := 0; i < l; i++ {
+			current, err := w.visit(v.Index(i), nil)
+			if err != nil {
+				return 0, err
+			}
+
+			h = hashUpdateOrdered(w.h, h, current)
+		}
+
+		return h, nil
+
+	case reflect.Map:
+		var includeMap IncludableMap
+		if opts != nil && opts.Struct != nil {
+			if v, ok := opts.Struct.(IncludableMap); ok {
+				includeMap = v
+			}
+		}
+
+		// Build the hash for the map. We do this by XOR-ing all the key
+		// and value hashes. This makes it deterministic despite ordering.
+		var h uint64
+		for _, k := range v.MapKeys() {
+			v := v.MapIndex(k)
+			if includeMap != nil {
+				incl, err := includeMap.HashIncludeMap(
+					opts.StructField, k.Interface(), v.Interface())
+				if err != nil {
+					return 0, err
+				}
+				if !incl {
+					continue
+				}
+			}
+
+			kh, err := w.visit(k, nil)
+			if err != nil {
+				return 0, err
+			}
+			vh, err := w.visit(v, nil)
+			if err != nil {
+				return 0, err
+			}
+
+			fieldHash := hashUpdateOrdered(w.h, kh, vh)
+			h = hashUpdateUnordered(h, fieldHash)
+		}
+
+		if w.format != FormatV1 {
+			// Important: read the docs for hashFinishUnordered
+			h = hashFinishUnordered(w.h, h)
+		}
+
+		return h, nil
+
+	case reflect.Struct:
+		parent := v.Interface()
+		var include Includable
+		if impl, ok := parent.(Includable); ok {
+			include = impl
+		}
+
+		if impl, ok := parent.(Hashable); ok {
+			return impl.Hash()
+		}
+
+		// If we can address this value, check if the pointer value
+		// implements our interfaces and use that if so.
+		if v.CanAddr() {
+			vptr := v.Addr()
+			parentptr := vptr.Interface()
+			if impl, ok := parentptr.(Includable); ok {
+				include = impl
+			}
+
+			if impl, ok := parentptr.(Hashable); ok {
+				return impl.Hash()
+			}
+		}
+
+		t := v.Type()
+		h, err := w.visit(reflect.ValueOf(t.Name()), nil)
+		if err != nil {
+			return 0, err
+		}
+
+		l := v.NumField()
+		for i := 0; i < l; i++ {
+			if innerV := v.Field(i); v.CanSet() || t.Field(i).Name != "_" {
+				var f visitFlag
+				fieldType := t.Field(i)
+				if fieldType.PkgPath != "" {
+					// Unexported
+					continue
+				}
+
+				tag := fieldType.Tag.Get(w.tag)
+				if tag == "ignore" || tag == "-" {
+					// Ignore this field
+					continue
+				}
+
+				if w.ignorezerovalue {
+					zeroVal := reflect.Zero(reflect.TypeOf(innerV.Interface())).Interface()
+					if innerV.Interface() == zeroVal {
+						continue
+					}
+				}
+
+				// if string is set, use the string value
+				if tag == "string" || w.stringer {
+					if impl, ok := innerV.Interface().(fmt.Stringer); ok {
+						innerV = reflect.ValueOf(impl.String())
+					} else if tag == "string" {
+						// We only show this error if the tag explicitly
+						// requests a stringer.
+						return 0, &ErrNotStringer{
+							Field: v.Type().Field(i).Name,
+						}
+					}
+				}
+
+				// Check if we implement includable and check it
+				if include != nil {
+					incl, err := include.HashInclude(fieldType.Name, innerV)
+					if err != nil {
+						return 0, err
+					}
+					if !incl {
+						continue
+					}
+				}
+
+				switch tag {
+				case "set":
+					f |= visitFlagSet
+				}
+
+				kh, err := w.visit(reflect.ValueOf(fieldType.Name), nil)
+				if err != nil {
+					return 0, err
+				}
+
+				vh, err := w.visit(innerV, &visitOpts{
+					Flags:       f,
+					Struct:      parent,
+					StructField: fieldType.Name,
+				})
+				if err != nil {
+					return 0, err
+				}
+
+				fieldHash := hashUpdateOrdered(w.h, kh, vh)
+				h = hashUpdateUnordered(h, fieldHash)
+			}
+
+			if w.format != FormatV1 {
+				// Important: read the docs for hashFinishUnordered
+				h = hashFinishUnordered(w.h, h)
+			}
+		}
+
+		return h, nil
+
+	case reflect.Slice:
+		// We have two behaviors here. If it isn't a set, then we just
+		// visit all the elements. If it is a set, then we do a deterministic
+		// hash code.
+		var h uint64
+		var set bool
+		if opts != nil {
+			set = (opts.Flags & visitFlagSet) != 0
+		}
+		l := v.Len()
+		for i := 0; i < l; i++ {
+			current, err := w.visit(v.Index(i), nil)
+			if err != nil {
+				return 0, err
+			}
+
+			if set || w.sets {
+				h = hashUpdateUnordered(h, current)
+			} else {
+				h = hashUpdateOrdered(w.h, h, current)
+			}
+		}
+
+		if set && w.format != FormatV1 {
+			// Important: read the docs for hashFinishUnordered
+			h = hashFinishUnordered(w.h, h)
+		}
+
+		return h, nil
+
+	case reflect.String:
+		// Directly hash
+		w.h.Reset()
+		_, err := w.h.Write([]byte(v.String()))
+		return w.h.Sum64(), err
+
+	default:
+		return 0, fmt.Errorf("unknown kind to hash: %s", k)
+	}
+
+}
+
+func hashUpdateOrdered(h hash.Hash64, a, b uint64) uint64 {
+	// For ordered updates, use a real hash function
+	h.Reset()
+
+	// We just panic if the binary writes fail because we are writing
+	// an int64 which should never be fail-able.
+	e1 := binary.Write(h, binary.LittleEndian, a)
+	e2 := binary.Write(h, binary.LittleEndian, b)
+	if e1 != nil {
+		panic(e1)
+	}
+	if e2 != nil {
+		panic(e2)
+	}
+
+	return h.Sum64()
+}
+
+func hashUpdateUnordered(a, b uint64) uint64 {
+	return a ^ b
+}
+
+// After mixing a group of unique hashes with hashUpdateUnordered, it's always
+// necessary to call hashFinishUnordered. Why? Because hashUpdateUnordered
+// is a simple XOR, and calling hashUpdateUnordered on hashes produced by
+// hashUpdateUnordered can effectively cancel out a previous change to the hash
+// result if the same hash value appears later on. For example, consider:
+//
+//   hashUpdateUnordered(hashUpdateUnordered("A", "B"), hashUpdateUnordered("A", "C")) =
+//   H("A") ^ H("B")) ^ (H("A") ^ H("C")) =
+//   (H("A") ^ H("A")) ^ (H("B") ^ H(C)) =
+//   H(B) ^ H(C) =
+//   hashUpdateUnordered(hashUpdateUnordered("Z", "B"), hashUpdateUnordered("Z", "C"))
+//
+// hashFinishUnordered "hardens" the result, so that encountering partially
+// overlapping input data later on in a different context won't cancel out.
+func hashFinishUnordered(h hash.Hash64, a uint64) uint64 {
+	h.Reset()
+
+	// We just panic if the writes fail
+	e1 := binary.Write(h, binary.LittleEndian, a)
+	if e1 != nil {
+		panic(e1)
+	}
+
+	return h.Sum64()
+}
+
+// visitFlag is used as a bitmask for affecting visit behavior
+type visitFlag uint
+
+const (
+	visitFlagInvalid visitFlag = iota
+	visitFlagSet               = iota << 1
+)

vendor/github.com/mitchellh/hashstructure/v2/include.go

@@ -0,0 +1,22 @@
+package hashstructure
+
+// Includable is an interface that can optionally be implemented by
+// a struct. It will be called for each field in the struct to check whether
+// it should be included in the hash.
+type Includable interface {
+	HashInclude(field string, v interface{}) (bool, error)
+}
+
+// IncludableMap is an interface that can optionally be implemented by
+// a struct. It will be called when a map-type field is found to ask the
+// struct if the map item should be included in the hash.
+type IncludableMap interface {
+	HashIncludeMap(field string, k, v interface{}) (bool, error)
+}
+
+// Hashable is an interface that can optionally be implemented by a struct
+// to override the hash value. This value will override the hash value for
+// the entire struct. Entries in the struct will not be hashed.
+type Hashable interface {
+	Hash() (uint64, error)
+}

vendor/github.com/mudler/go-pluggable/README.md

@@ -1,6 +1,7 @@
-# go-pluggable
+# :bento: go-pluggable
+[![PkgGoDev](https://pkg.go.dev/badge/github.com/mudler/go-pluggable)](https://pkg.go.dev/github.com/mudler/go-pluggable) [![Go Report Card](https://goreportcard.com/badge/github.com/mudler/go-pluggable)](https://goreportcard.com/report/github.com/mudler/go-pluggable) [![Test](https://github.com/mudler/go-pluggable/workflows/Test/badge.svg)](https://github.com/mudler/go-pluggable/actions?query=workflow%3ATest)
 
-light Bus-event driven plugin library for Golang.
+:bento: *go-pluggable* is a light Bus-event driven plugin library for Golang.
 
 `go-pluggable` implements the event/sub pattern to extend your Golang project with external binary plugins that can be written in any language.
 
@@ -24,8 +25,24 @@ func main() {
     m.Autoload("test", temp)
     m.Register()
 
-    // ...
-    m.Publish(myEv,  map[string]string{"foo": "bar"}) // test-foo, will receive our data as json payload
+    // Optionally process plugin results response
+    // The plugins has to return as output a json in stdout in the format { 'state': "somestate", data: "some data", error: "some error" }
+    // e.g. with jq:  
+    // jq --arg key0   'state' \
+    // --arg value0 '' \
+    // --arg key1   'data' \
+    // --arg value1 "" \
+    // --arg key2   'error' \
+    // --arg value2 '' \
+    // '. | .[$key0]=$value0 | .[$key1]=$value1 | .[$key2]=$value2' \
+    // <<<'{}'
+    m.Response(myEv, func(p *pluggable.Plugin, r *pluggable.EventResponse) { ... }) 
+
+    // Emit events, they are encoded and passed as JSON payloads to the plugins.
+    // In our case, test-foo will receive the map as JSON
+    m.Publish(myEv,  map[string]string{"foo": "bar"})
+
+
 }
 
-```
+```

vendor/github.com/mudler/go-pluggable/bus.go

@@ -1,73 +0,0 @@
-// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package pluggable
-
-import (
-	"sync"
-
-	"github.com/chuckpreslar/emission"
-	"github.com/codegangsta/inject"
-)
-
-// Bus represent the bus event system
-type Bus struct {
-	inject.Injector
-	emission.Emitter
-	sync.Mutex
-}
-
-// NewBus returns a new Bus instance
-func NewBus() *Bus {
-	return &Bus{
-		Injector: inject.New(),
-		Emitter:  *emission.NewEmitter(),
-	}
-}
-
-// Listen Binds a callback to an event, mapping the arguments on a global level
-func (a *Bus) Listen(event EventType, listener interface{}) *Bus {
-	a.Lock()
-	defer a.Unlock()
-	a.On(string(event), func() { a.Invoke(listener) })
-	return a
-}
-
-// Publish publishes an event, it does accept only the event as argument, since
-// the callback will have access to the service mapped by the injector
-func (a *Bus) Publish(e *Event) *Bus {
-	a.Lock()
-	defer a.Unlock()
-	a.Map(e)
-	a.Emit(string(e.Name))
-	return a
-}
-
-// OnlyOnce Binds a callback to an event, mapping the arguments on a global level
-// It is fired only once.
-func (a *Bus) OnlyOnce(event EventType, listener interface{}) *Bus {
-	a.Lock()
-	defer a.Unlock()
-	a.Once(string(event), func() { a.Invoke(listener) })
-	return a
-}
-
-func (a *Bus) propagateEvent(p Plugin) func(e *Event) {
-	return func(e *Event) {
-		resp, _ := p.Run(*e)
-		a.Map(&resp)
-		a.Emit(p.Name)
-	}
-}

vendor/github.com/mudler/go-pluggable/events.go

@@ -15,7 +15,10 @@
 
 package pluggable
 
-import "encoding/json"
+import (
+	"encoding/json"
+	"fmt"
+)
 
 // EventType describes an event type
 type EventType string
@@ -26,6 +29,7 @@ type EventType string
 type Event struct {
 	Name EventType `json:"name"`
 	Data string    `json:"data"`
+	File string    `json:"file"` // If Data >> 10K write content to file instead
 }
 
 // EventResponse describes the event response structure
@@ -42,6 +46,16 @@ func (e Event) JSON() (string, error) {
 	return string(dat), err
 }
 
+// Copy returns a copy of Event
+func (e Event) Copy() *Event {
+	copy := &e
+	return copy
+}
+
+func (e Event) ResponseEventName(s string) EventType {
+	return EventType(fmt.Sprintf("%s-%s", e.Name, s))
+}
+
 // Unmarshal decodes the json payload in the given parameteer
 func (r EventResponse) Unmarshal(i interface{}) error {
 	return json.Unmarshal([]byte(r.Data), i)
@@ -52,7 +66,7 @@ func (r EventResponse) Errored() bool {
 	return len(r.Error) != 0
 }
 
-// NewEvent retuns a new event which can be used for publishing
+// NewEvent returns a new event which can be used for publishing
 // the obj gets automatically serialized in json.
 func NewEvent(name EventType, obj interface{}) (*Event, error) {
 	dat, err := json.Marshal(obj)

vendor/github.com/mudler/go-pluggable/go.mod

@@ -4,7 +4,6 @@ go 1.14
 
 require (
 	github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9
-	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3
 	github.com/pkg/errors v0.9.1

vendor/github.com/mudler/go-pluggable/go.sum

@@ -1,7 +1,5 @@
 github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9 h1:xz6Nv3zcwO2Lila35hcb0QloCQsc38Al13RNEzWRpX4=
 github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9/go.mod h1:2wSM9zJkl1UQEFZgSd68NfCgRz1VL1jzy/RjCg+ULrs=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=

vendor/github.com/mudler/go-pluggable/manager.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/chuckpreslar/emission"
 	"github.com/pkg/errors"
 )
 
@@ -29,14 +30,14 @@ import (
 type Manager struct {
 	Plugins []Plugin
 	Events  []EventType
-	Bus     *Bus
+	Bus     *emission.Emitter
 }
 
 // NewManager returns a manager instance with a new bus and
 func NewManager(events []EventType) *Manager {
 	return &Manager{
 		Events: events,
-		Bus:    NewBus(),
+		Bus:    emission.NewEmitter(),
 	}
 }
 
@@ -47,19 +48,41 @@ func (m *Manager) Register() *Manager {
 }
 
 // Publish is a wrapper around NewEvent and the Manager internal Bus publishing system
+// It accepts optionally a list of functions that are called with the plugin result (only once)
 func (m *Manager) Publish(event EventType, obj interface{}) (*Manager, error) {
 	ev, err := NewEvent(event, obj)
 	if err == nil && ev != nil {
-		m.Bus.Publish(ev)
+		m.Bus.Emit(string(ev.Name), ev)
 	}
 	return m, err
 }
 
+// Response binds a set of listeners to an event type. The listeners are called for each result from
+// every plugin when Publish is called.
+func (m *Manager) Response(event EventType, listener ...func(p *Plugin, r *EventResponse)) *Manager {
+	ev, _ := NewEvent(event, nil)
+	for _, l := range listener {
+		m.Bus.On(string(ev.ResponseEventName("results")), l)
+	}
+	return m
+}
+
+func (m *Manager) propagateEvent(p Plugin) func(e *Event) {
+	return func(e *Event) {
+		resp, err := p.Run(*e)
+		r := &resp
+		if err != nil && !resp.Errored() {
+			resp.Error = err.Error()
+		}
+		m.Bus.Emit(string(e.ResponseEventName("results")), &p, r)
+	}
+}
+
 // Subscribe subscribes the plugin to the events in the given bus
-func (m *Manager) Subscribe(b *Bus) *Manager {
+func (m *Manager) Subscribe(b *emission.Emitter) *Manager {
 	for _, p := range m.Plugins {
 		for _, e := range m.Events {
-			b.Listen(e, b.propagateEvent(p))
+			b.On(string(e), m.propagateEvent(p))
 		}
 	}
 	return m
@@ -74,13 +97,6 @@ func relativeToCwd(p string) (string, error) {
 	return filepath.Join(cwd, p), nil
 }
 
-// ListenAll Binds a callback to all plugins event
-func (m *Manager) ListenAll(event EventType, listener interface{}) {
-	for _, p := range m.Plugins {
-		m.Bus.Listen(EventType(p.Name), listener)
-	}
-}
-
 // Autoload automatically loads plugins binaries prefixed by 'prefix' in the current path
 // optionally takes a list of paths to look also into
 func (m *Manager) Autoload(prefix string, extensionpath ...string) *Manager {

vendor/github.com/mudler/go-pluggable/plugin.go

@@ -16,7 +16,9 @@
 package pluggable
 
 import (
+	"bytes"
 	"encoding/json"
+	"io/ioutil"
 	"os"
 	"os/exec"
 
@@ -29,19 +31,42 @@ type Plugin struct {
 	Executable string
 }
 
+// A safe threshold to avoid unpleasant exec buffer fill for argv too big. Seems 128K is the limit on Linux.
+const maxMessageSize = 1 << 13
+
 // Run runs the Event on the plugin, and returns an EventResponse
 func (p Plugin) Run(e Event) (EventResponse, error) {
 	r := EventResponse{}
-	k, err := e.JSON()
+
+	eventToprocess := &e
+
+	if len(e.Data) > maxMessageSize {
+		copy := e.Copy()
+		copy.Data = ""
+		f, err := ioutil.TempFile(os.TempDir(), "pluggable")
+		if err != nil {
+			return r, errors.Wrap(err, "while creating temporary file")
+		}
+		if err := ioutil.WriteFile(f.Name(), []byte(e.Data), os.ModePerm); err != nil {
+			return r, errors.Wrap(err, "while writing to temporary file")
+		}
+		copy.File = f.Name()
+		eventToprocess = copy
+		defer os.RemoveAll(f.Name())
+	}
+
+	k, err := eventToprocess.JSON()
 	if err != nil {
 		return r, errors.Wrap(err, "while marshalling event")
 	}
 	cmd := exec.Command(p.Executable, string(e.Name), k)
 	cmd.Env = os.Environ()
+	var b bytes.Buffer
+	cmd.Stderr = &b
 	out, err := cmd.Output()
 	if err != nil {
-		r.Error = err.Error()
-		return r, errors.Wrap(err, "while executing plugin")
+		r.Error = "error while executing plugin: " + err.Error() + string(b.String())
+		return r, errors.Wrap(err, "while executing plugin: "+string(b.String()))
 	}
 
 	if err := json.Unmarshal(out, &r); err != nil {

vendor/modules.txt

@@ -62,8 +62,6 @@ github.com/cavaliercoder/grab/bps
 github.com/cespare/xxhash/v2
 # github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9
 github.com/chuckpreslar/emission
-# github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0
-github.com/codegangsta/inject
 # github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d
 github.com/containerd/cgroups/stats/v1
 # github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc => github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55
@@ -209,7 +207,6 @@ github.com/fatih/color
 # github.com/fsnotify/fsnotify v1.4.9
 github.com/fsnotify/fsnotify
 # github.com/fsouza/go-dockerclient v1.6.4
-## explicit
 github.com/fsouza/go-dockerclient
 # github.com/genuinetools/img v0.5.11
 ## explicit
@@ -379,6 +376,9 @@ github.com/mitchellh/colorstring
 github.com/mitchellh/copystructure
 # github.com/mitchellh/hashstructure v1.0.0
 github.com/mitchellh/hashstructure
+# github.com/mitchellh/hashstructure/v2 v2.0.1
+## explicit
+github.com/mitchellh/hashstructure/v2
 # github.com/mitchellh/mapstructure v1.1.2
 github.com/mitchellh/mapstructure
 # github.com/mitchellh/reflectwalk v1.0.0
@@ -478,7 +478,7 @@ github.com/mudler/cobra-extensions
 # github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
 ## explicit
 github.com/mudler/docker-companion/api
-# github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+# github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 ## explicit
 github.com/mudler/go-pluggable
 # github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290