Tag 0.16.3

A new keyword `join` is introduced to generate the parent image. It
takes precedence over a `requires` or a `image` already defined in a
spec.

It will generate all the artifacts from the packages listed and join
them in a single image which will be used as parent for the package
build process.

This is a change which invalidates priorly generated hashes.

Fixes #173

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.14.7"
+	LuetCLIVersion = "0.16.3"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/tree/images.go

@@ -96,9 +96,14 @@ func NewTreeImageCommand() *cobra.Command {
 			if err != nil {
 				Fatal("Error: " + err.Error())
 			}
-			asserts, err := luetCompiler.ComputeDepTree(spec)
 
-			for _, assertion := range asserts { //highly dependent on the order
+			ht := compiler.NewHashTree(reciper.GetDatabase())
+			hashtree, err := ht.Query(luetCompiler, spec)
+			if err != nil {
+				Fatal("Error: " + err.Error())
+			}
+
+			for _, assertion := range hashtree.Solution { //highly dependent on the order
 
 				//buildImageHash := imageRepository + ":" + assertion.Hash.BuildHash
 				currentPackageImageHash := imageRepository + ":" + assertion.Hash.PackageHash

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/kyokomi/emoji v2.1.0+incompatible
 	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
+	github.com/mitchellh/hashstructure/v2 v2.0.1
 	github.com/moby/buildkit v0.7.2
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d

go.sum

@@ -728,6 +728,8 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
 github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y=
 github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
+github.com/mitchellh/hashstructure/v2 v2.0.1 h1:L60q1+q7cXE4JeEJJKMnh2brFIe3rZxCihYAB61ypAY=
+github.com/mitchellh/hashstructure/v2 v2.0.1/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=

pkg/compiler/compiler.go

@@ -16,7 +16,9 @@
 package compiler
 
 import (
+	"crypto/md5"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -89,7 +91,7 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *c
 	defer wg.Done()
 
 	for s := range cspecs {
-		ar, err := cs.compile(concurrency, keepPermissions, s)
+		ar, err := cs.compile(concurrency, keepPermissions, nil, s)
 		if err != nil {
 			errors <- err
 		}
@@ -506,7 +508,7 @@ func oneOfImagesAvailable(images []string, b CompilerBackend) (bool, string) {
 	return false, ""
 }
 
-func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
+func (cs *LuetCompiler) findImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
 	var resolvedImage string
 	Debug("Resolving image hash for", p.Package.HumanReadableString(), "hash", imageHash, "Pull repositories", p.BuildOptions.PullImageRepository)
 	toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)},
@@ -519,6 +521,11 @@ func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilersp
 			resolvedImage = which
 		}
 	}
+	return resolvedImage
+}
+
+func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
+	resolvedImage := cs.findImageHash(imageHash, p)
 
 	if resolvedImage == "" {
 		resolvedImage = fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)
@@ -603,12 +610,24 @@ func (cs *LuetCompiler) compileWithImage(image, builderHash string, packageTagHa
 		Debug("Checking if an image is already available")
 		// FIXUP here. If packageimage hash exists and pull is true, generate package
 		resolved := cs.resolveExistingImageHash(packageTagHash, p)
+		Debug("Resolved: " + resolved)
+		Debug("Expected remote: " + resolved)
+		Debug("Package image: " + packageImage)
+		Debug("Resolved builder image: " + builderResolved)
 
-		//
-		if resolved != packageImage && remoteBuildertaggedImage != builderResolved { // an image is there already
-			Debug("Images available for", p.Package.HumanReadableString(), "generating artifact from remote images:", resolved)
+		// a remote image is there already
+		remoteImageAvailable := resolved != packageImage && remoteBuildertaggedImage != builderResolved
+		// or a local one is available
+		localImageAvailable := cs.Backend.ImageExists(remoteBuildertaggedImage) && cs.Backend.ImageExists(packageImage)
+
+		switch {
+		case remoteImageAvailable:
+			Debug("Images available remotely for", p.Package.HumanReadableString(), "generating artifact from remote images:", resolved)
 			return cs.genArtifact(p, backend.Options{ImageName: builderResolved}, backend.Options{ImageName: resolved}, concurrency, keepPermissions)
-		} else {
+		case localImageAvailable:
+			Debug("Images locally available for", p.Package.HumanReadableString(), "generating artifact from image:", resolved)
+			return cs.genArtifact(p, backend.Options{ImageName: remoteBuildertaggedImage}, backend.Options{ImageName: packageImage}, concurrency, keepPermissions)
+		default:
 			Debug("Images not available for", p.Package.HumanReadableString())
 		}
 	}
@@ -704,7 +723,7 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompi
 // Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
 // an artifact
 func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	return cs.compile(cs.Options.Concurrency, keepPermissions, p)
+	return cs.compile(cs.Options.Concurrency, keepPermissions, nil, p)
 }
 
 func genImageList(refs []string, hash string) []string {
@@ -733,9 +752,183 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS
 	Debug(p.GetPackage().HumanReadableString(), "Build options after inherit", p.BuildOptions)
 }
 
-func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
+func (cs *LuetCompiler) getSpecHash(pkgs pkg.DefaultPackages, salt string) (string, error) {
+	ht := NewHashTree(cs.Database)
+	overallFp := ""
+
+	for _, p := range pkgs {
+		compileSpec, err := cs.FromPackage(p)
+		if err != nil {
+			return "", errors.Wrap(err, "Error while generating compilespec for "+p.GetName())
+		}
+		packageHashTree, err := ht.Query(cs, compileSpec)
+		if err != nil {
+			return "nil", errors.Wrap(err, "failed querying hashtree")
+		}
+		overallFp = overallFp + packageHashTree.Target.Hash.PackageHash + p.GetFingerPrint()
+	}
+
+	h := md5.New()
+	io.WriteString(h, fmt.Sprintf("%s-%s", overallFp, salt))
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
+func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+
+	joinTag := ">:loop: join<"
+	if len(p.Join) != 0 {
+		Info(joinTag, "Generating a joint parent image from final packages")
+	} else {
+		return nil
+	}
+
+	// First compute a hash and check if image is available. if it is, then directly consume that
+	overallFp, err := cs.getSpecHash(p.Join, "join")
+	if err != nil {
+		return errors.Wrap(err, "could not generate image hash")
+	}
+
+	Info(joinTag, "Searching existing image with hash ", overallFp)
+
+	image := cs.findImageHash(overallFp, p)
+	if image != "" {
+		Info("Image already found", image)
+		p.SetImage(image)
+		return nil
+	}
+	Info(joinTag, "Image not found. Generating image join with hash ", overallFp)
+
+	// Make sure there is an output path
+	if err := os.MkdirAll(p.GetOutputPath(), os.ModePerm); err != nil {
+		return errors.Wrap(err, "while creating output path")
+	}
+
+	// otherwise, generate it and push it aside
+	joinDir, err := ioutil.TempDir(p.GetOutputPath(), "join")
+	if err != nil {
+		return errors.Wrap(err, "could not create tempdir for joining images")
+	}
+	defer os.RemoveAll(joinDir) // clean up
+
+	for _, p := range p.Join { //highly dependent on the order
+		Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
+	}
+
+	current := 0
+	for _, c := range p.Join {
+		current++
+		if c != nil && c.Name != "" && c.Version != "" {
+			joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(p.Join), c.HumanReadableString())
+
+			Info(joinTag2, "compilation starts")
+			spec, err := cs.FromPackage(c)
+			if err != nil {
+				return errors.Wrap(err, "while generating images to join from")
+			}
+			wantsArtifact := true
+			spec.SetOutputPath(p.GetOutputPath())
+
+			artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, spec)
+			if err != nil {
+				return errors.Wrap(err, "failed building join image")
+			}
+
+			err = artifact.Unpack(joinDir, keepPermissions)
+			if err != nil {
+				return errors.Wrap(err, "failed building join image")
+			}
+			Info(joinTag2, ":white_check_mark: Done")
+		}
+	}
+
+	artifactDir, err := ioutil.TempDir(p.GetOutputPath(), "artifact")
+	if err != nil {
+		return errors.Wrap(err, "could not create tempdir for final artifact")
+	}
+	defer os.RemoveAll(joinDir) // clean up
+
+	Info(joinTag, ":droplet: generating artifact for source image of", p.GetPackage().HumanReadableString())
+
+	// After unpack, create a new artifact and a new final image from it.
+	// no need to compress, as we are going to toss it away.
+	a := artifact.NewPackageArtifact(filepath.Join(artifactDir, p.GetPackage().GetFingerPrint()+".join.tar"))
+	if err := a.Compress(joinDir, concurrency); err != nil {
+		return errors.Wrap(err, "error met while creating package archive")
+	}
+
+	joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp)
+	Info(joinTag, ":droplet: generating image from artifact", joinImageName)
+	opts, err := a.GenerateFinalImage(joinImageName, cs.Backend, keepPermissions)
+	if err != nil {
+		return errors.Wrap(err, "could not create final image")
+	}
+	if cs.Options.Push {
+		Info(joinTag, ":droplet: pushing image from artifact", joinImageName)
+		if err = cs.Backend.Push(opts); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
+		}
+	}
+	Info(joinTag, ":droplet: Consuming image", joinImageName)
+	p.SetImage(joinImageName)
+	return nil
+}
+
+func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+	resolvedCopyFields := []compilerspec.CopyField{}
+	copyTag := ">:droplet: copy<"
+
+	if len(p.Copy) != 0 {
+		Info(copyTag, "Package has multi-stage copy, generating required images")
+	}
+
+	current := 0
+	// TODO: we should run this only if we are going to build the image
+	for _, c := range p.Copy {
+		current++
+		if c.Package != nil && c.Package.Name != "" && c.Package.Version != "" {
+			copyTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", copyTag, current, len(p.Copy), c.Package.HumanReadableString())
+
+			Info(copyTag2, "generating multi-stage images for", c.Package.HumanReadableString())
+			spec, err := cs.FromPackage(c.Package)
+			if err != nil {
+				return errors.Wrap(err, "while generating images to copy from")
+			}
+
+			// If we specify --only-target package, we don't want any artifact, otherwise we do
+			genArtifact := !cs.Options.PackageTargetOnly
+			spec.SetOutputPath(p.GetOutputPath())
+			artifact, err := cs.compile(concurrency, keepPermissions, &genArtifact, spec)
+			if err != nil {
+				return errors.Wrap(err, "failed building multi-stage image")
+			}
+
+			resolvedCopyFields = append(resolvedCopyFields, compilerspec.CopyField{
+				Image:       cs.resolveExistingImageHash(artifact.PackageCacheImage, spec),
+				Source:      c.Source,
+				Destination: c.Destination,
+			})
+			Info(copyTag2, ":white_check_mark: Done")
+		} else {
+			resolvedCopyFields = append(resolvedCopyFields, c)
+		}
+	}
+	p.Copy = resolvedCopyFields
+	return nil
+}
+
+func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
 	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
+	//Before multistage : join - same as multistage, but keep artifacts, join them, create a new one and generate a final image.
+	// When the image is there, use it as a source here, in place of GetImage().
+	if err := cs.resolveJoinImages(concurrency, keepPermissions, p); err != nil {
+		return nil, errors.Wrap(err, "while resolving join images")
+	}
+
+	if err := cs.resolveMultiStageImages(concurrency, keepPermissions, p); err != nil {
+		return nil, errors.Wrap(err, "while resolving multi-stage images")
+	}
+
 	Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
 	if !p.HasImageSource() && !p.EmptyPackage() {
 		return nil,
@@ -772,7 +965,19 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		return cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
+		localGenerateArtifact := true
+		if generateArtifact != nil {
+			localGenerateArtifact = *generateArtifact
+		}
+
+		a, err := cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
+		if err != nil {
+			return nil, errors.Wrap(err, "building direct image")
+		}
+		a.SourceAssertion = p.GetSourceAssertion()
+
+		a.PackageCacheImage = targetAssertion.Hash.PackageHash
+		return a, nil
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@@ -785,6 +990,10 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 	currentN := 0
 
 	packageDeps := !cs.Options.PackageTargetOnly
+	if generateArtifact != nil {
+		packageDeps = *generateArtifact
+	}
+
 	buildDeps := !cs.Options.NoDeps
 	buildTarget := !cs.Options.OnlyDeps
 
@@ -816,6 +1025,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 				Assert:      assertion,
 			})
 
+			if err := cs.resolveJoinImages(concurrency, keepPermissions, compileSpec); err != nil {
+				return nil, errors.Wrap(err, "while resolving join images")
+			}
+
+			if err := cs.resolveMultiStageImages(concurrency, keepPermissions, compileSpec); err != nil {
+				return nil, errors.Wrap(err, "while resolving multi-stage images")
+			}
+
 			buildHash, err := packageHashTree.DependencyBuildImage(assertion.Package)
 			if err != nil {
 				return nil, errors.Wrap(err, "failed looking for dependency in hashtree")
@@ -851,6 +1068,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 			}
 
+			a.PackageCacheImage = assertion.Hash.PackageHash
+
 			Info(pkgTag, ":white_check_mark: Done")
 
 			bus.Manager.Publish(bus.EventPackagePostBuild, struct {
@@ -866,16 +1085,20 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 	}
 
 	if buildTarget {
+		localGenerateArtifact := true
+		if generateArtifact != nil {
+			localGenerateArtifact = *generateArtifact
+		}
 		resolvedSourceImage := cs.resolveExistingImageHash(packageHashTree.SourceHash, p)
 		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
 		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedSourceImage)
-		a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
+		a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
 		if err != nil {
 			return a, err
 		}
 		a.Dependencies = departifacts
 		a.SourceAssertion = p.GetSourceAssertion()
-
+		a.PackageCacheImage = targetAssertion.Hash.PackageHash
 		bus.Manager.Publish(bus.EventPackagePostBuild, struct {
 			CompileSpec *compilerspec.LuetCompilationSpec
 			Artifact    *artifact.PackageArtifact

pkg/compiler/compiler_test.go

@@ -75,6 +75,68 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Copy and Join", func() {
+		It("Compiles it correctly with Copy", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/copy")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("result"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("bina/busybox"))).To(BeTrue())
+		})
+
+		It("Compiles it correctly with Join", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/join")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("newc"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
+		})
+	})
+
 	Context("Simple package build definition", func() {
 		It("Compiles it in parallel", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))

pkg/compiler/imagehashtree.go

@@ -25,11 +25,22 @@ import (
 	"github.com/pkg/errors"
 )
 
+// ImageHashTree is holding the Database
+// and the options to resolve PackageImageHashTrees
+// for a given specfile
+// It is responsible of returning a concrete result
+// which identifies a Package in a HashTree
 type ImageHashTree struct {
 	Database      pkg.PackageDatabase
 	SolverOptions config.LuetSolverOptions
 }
 
+// PackageImageHashTree represent the Package into a given image hash tree
+// The hash tree is constructed by a set of images representing
+// the package during its build stage. A Hash is assigned to each image
+// from the package fingerprint, plus the SAT solver assertion result (which is hashed as well)
+// and the specfile signatures. This guarantees that each image of the build stage
+// is unique and can be identified later on.
 type PackageImageHashTree struct {
 	Target                       *solver.PackageAssert
 	Dependencies                 solver.PackagesAssertions
@@ -53,11 +64,19 @@ func (ht *PackageImageHashTree) DependencyBuildImage(p pkg.Package) (string, err
 	return found, nil
 }
 
-// TODO: ___ When computing the hash per package (and evaluating the sat solver solution tree part)
-// we should use the hash of each package  + its fingerprint  instead as a salt.
-// That's because the hash will be salted with its `build.yaml`.
-// In this way, we trigger recompilations if some dep of a target changes
-// a build.yaml, without touching the version
+func (ht *PackageImageHashTree) String() string {
+	return fmt.Sprintf(
+		"Target buildhash: %s\nTarget packagehash: %s\nBuilder Imagehash: %s\nSource Imagehash: %s\n",
+		ht.Target.Hash.BuildHash,
+		ht.Target.Hash.PackageHash,
+		ht.BuilderImageHash,
+		ht.SourceHash,
+	)
+}
+
+// Query takes a compiler and a compilation spec and returns a PackageImageHashTree tied to it.
+// PackageImageHashTree contains all the informations to resolve the spec build images in order to
+// reproducibly re-build images from packages
 func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) {
 	assertions, err := ht.resolve(cs, p)
 	if err != nil {
@@ -109,13 +128,29 @@ func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilati
 		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
 	}
 
+	// Get hash from buildpsecs
+	salts := map[string]string{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			spec, err := cs.FromPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "while computing hash buildspecs")
+			}
+			hash, err := spec.Hash()
+			if err != nil {
+				return nil, errors.Wrap(err, "failed computing hash")
+			}
+			salts[assertion.Package.GetFingerPrint()] = hash
+		}
+	}
+
 	assertions := solver.PackagesAssertions{}
 	for _, assertion := range dependencies { //highly dependent on the order
 		if assertion.Value {
 			nthsolution := dependencies.Cut(assertion.Package)
 			assertion.Hash = solver.PackageHash{
-				BuildHash:   nthsolution.HashFrom(assertion.Package),
-				PackageHash: nthsolution.AssertionHash(),
+				BuildHash:   nthsolution.SaltedHashFrom(assertion.Package, salts),
+				PackageHash: nthsolution.SaltedAssertionHash(salts),
 			}
 			assertion.Package.SetTreeDir(p.Package.GetTreeDir())
 			assertions = append(assertions, assertion)

pkg/compiler/imagehashtree_test.go

@@ -46,12 +46,14 @@ var _ = Describe("ImageHashTree", func() {
 
 			packageHash, err := hashtree.Query(compiler, spec)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(packageHash.Target.Hash.BuildHash).To(Equal("6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
-			Expect(packageHash.BuilderImageHash).To(Equal("builder-79462b60bf899ad79db63f194a3c9c2a"))
+			Expect(packageHash.Target.Hash.BuildHash).To(Equal("4db24406e8db30a3310a1cf8c4d4e19597745e6d41b189dc51a73ac4a50cc9e6"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("4c867c9bab6c71d9420df75806e7a2f171dbc08487852ab4e2487bab04066cf2"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-e6f9c5552a67c463215b0a9e4f7c7fc8"))
 		})
 	})
 
+	expectedPackageHash := "15811d83d0f8360318c54d91dcae3714f8efb39bf872572294834880f00ee7a8"
+
 	Context("complex package definition", func() {
 		BeforeEach(func() {
 			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -69,25 +71,74 @@ var _ = Describe("ImageHashTree", func() {
 			packageHash, err := hashtree.Query(compiler, spec)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
-			Expect(packageHash.SourceHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
-			Expect(packageHash.BuilderImageHash).To(Equal("builder-37f4d05ba8a39525742ca364f69b4090"))
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(expectedPackageHash))
+			Expect(packageHash.SourceHash).To(Equal(expectedPackageHash))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-3d739cab442aec15a6da238481df73c5"))
 
 			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("bb1d9a99c0c309a297c75b436504e664a42121fadbb4e035bda403cd418117aa"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("99c4ebb4bc4754985fcc28677badf90f525aa231b1db0fe75659f11b86dc20e8"))
 			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
 			hash, err := packageHash.DependencyBuildImage(a)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(hash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(hash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
 
 			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
-			Expect(assertionA.Hash.PackageHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
+			Expect(assertionA.Hash.PackageHash).To(Equal(expectedPackageHash))
 			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
 			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
-			Expect(assertionB.Hash.PackageHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(assertionB.Hash.PackageHash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
 			hashB, err := packageHash.DependencyBuildImage(b)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(hashB).To(Equal("6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"))
+			Expect(hashB).To(Equal("2668e418eab6861404834ad617713e39b8e58f68016a1fbfcc9384efdd037376"))
+		})
+	})
+
+	Context("complex package definition, with small change in build.yaml", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			//Definition of A here is slightly changed in the steps build.yaml file (1 character only)
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision_content_changed")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).ToNot(Equal(expectedPackageHash))
+			sourceHash := "1d91b13d0246fa000085a1071c63397d21546300b17f69493f22315a64b717d4"
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(sourceHash))
+			Expect(packageHash.SourceHash).To(Equal(sourceHash))
+			Expect(packageHash.SourceHash).ToNot(Equal(expectedPackageHash))
+
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-03ee108a7c56b17ee568ace0800dd16d"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("7677da23b2cc866c2d07aa4a58fbf703340f2f78c0efbb1ba9faf8979f250c87"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+
+			Expect(assertionA.Hash.PackageHash).To(Equal("1d91b13d0246fa000085a1071c63397d21546300b17f69493f22315a64b717d4"))
+			Expect(assertionA.Hash.PackageHash).ToNot(Equal(expectedPackageHash))
+
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+
+			Expect(assertionB.Hash.PackageHash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hashB).To(Equal("2668e418eab6861404834ad617713e39b8e58f68016a1fbfcc9384efdd037376"))
 		})
 	})
 

pkg/compiler/types/artifact/artifact.go

@@ -55,12 +55,13 @@ import (
 type PackageArtifact struct {
 	Path string `json:"path"`
 
-	Dependencies    []*PackageArtifact                `json:"dependencies"`
-	CompileSpec     *compilerspec.LuetCompilationSpec `json:"compilationspec"`
-	Checksums       Checksums                         `json:"checksums"`
-	SourceAssertion solver.PackagesAssertions         `json:"-"`
-	CompressionType compression.Implementation        `json:"compressiontype"`
-	Files           []string                          `json:"files"`
+	Dependencies      []*PackageArtifact                `json:"dependencies"`
+	CompileSpec       *compilerspec.LuetCompilationSpec `json:"compilationspec"`
+	Checksums         Checksums                         `json:"checksums"`
+	SourceAssertion   solver.PackagesAssertions         `json:"-"`
+	CompressionType   compression.Implementation        `json:"compressiontype"`
+	Files             []string                          `json:"files"`
+	PackageCacheImage string                            `json:"package_cacheimage"`
 }
 
 func (p *PackageArtifact) ShallowCopy() *PackageArtifact {

pkg/compiler/types/artifact/artifact_suite_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestArtifact(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Artifact Suite")
+}

pkg/compiler/types/artifact/artifact_test.go

@@ -41,7 +41,7 @@ var _ = Describe("Artifact", func() {
 
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
+			err := generalRecipe.Load("../../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))

pkg/compiler/types/artifact/checksum_test.go

@@ -40,13 +40,13 @@ var _ = Describe("Checksum", func() {
 			Expect(len(definitionsum)).To(Equal(0))
 			Expect(len(definitionsum2)).To(Equal(0))
 
-			err = buildsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/build.yaml"))
+			err = buildsum.Generate(NewPackageArtifact("../../../../tests/fixtures/layers/alpine/build.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = definitionsum.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			err = definitionsum.Generate(NewPackageArtifact("../../../../tests/fixtures/layers/alpine/definition.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			err = definitionsum2.Generate(NewPackageArtifact("../../tests/fixtures/layers/alpine/definition.yaml"))
+			err = definitionsum2.Generate(NewPackageArtifact("../../../../tests/fixtures/layers/alpine/definition.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(buildsum)).To(Equal(1))

pkg/compiler/types/spec/spec.go

@@ -20,6 +20,7 @@ import (
 	"io/ioutil"
 	"path/filepath"
 
+	"github.com/mitchellh/hashstructure/v2"
 	options "github.com/mudler/luet/pkg/compiler/types/options"
 
 	pkg "github.com/mudler/luet/pkg/package"
@@ -89,8 +90,8 @@ func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
 type CopyField struct {
 	Package     *pkg.DefaultPackage `json:"package"`
 	Image       string              `json:"image"`
-	Source      string              `json:"src"`
-	Destination string              `json:"dst"`
+	Source      string              `json:"source"`
+	Destination string              `json:"destination"`
 }
 
 type LuetCompilationSpec struct {
@@ -113,6 +114,41 @@ type LuetCompilationSpec struct {
 	BuildOptions *options.Compiler `json:"build_options"`
 
 	Copy []CopyField `json:"copy"`
+
+	Join pkg.DefaultPackages `json:"join"`
+}
+
+// Signature is a portion of the spec that yields a signature for the hash
+type Signature struct {
+	Image      string
+	Steps      []string
+	PackageDir string
+	Prelude    []string
+	Seed       string
+	Env        []string
+	Retrieve   []string
+	Unpack     bool
+	Includes   []string
+	Excludes   []string
+	Copy       []CopyField
+	Join       pkg.DefaultPackages
+}
+
+func (cs *LuetCompilationSpec) signature() Signature {
+	return Signature{
+		Image:      cs.Image,
+		Steps:      cs.Steps,
+		PackageDir: cs.PackageDir,
+		Prelude:    cs.Prelude,
+		Seed:       cs.Seed,
+		Env:        cs.Env,
+		Retrieve:   cs.Retrieve,
+		Unpack:     cs.Unpack,
+		Includes:   cs.Includes,
+		Excludes:   cs.Excludes,
+		Copy:       cs.Copy,
+		Join:       cs.Join,
+	}
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
@@ -223,7 +259,14 @@ func (cs *LuetCompilationSpec) UnpackedPackage() bool {
 // a compilation spec has an image source when it depends on other packages or have a source image
 // explictly supplied
 func (cs *LuetCompilationSpec) HasImageSource() bool {
-	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != ""
+	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != "" || len(cs.Join) != 0
+}
+
+func (cs *LuetCompilationSpec) Hash() (string, error) {
+	// build a signature, we want to be part of the hash only the fields that are relevant for build purposes
+	signature := cs.signature()
+	h, err := hashstructure.Hash(signature, hashstructure.FormatV2, nil)
+	return fmt.Sprint(h), err
 }
 
 func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
@@ -268,7 +311,8 @@ ADD ` + s + ` /luetbuild/`
 
 	for _, c := range cs.Copy {
 		if c.Image != "" {
-			spec = spec + fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
+			copyLine := fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
+			spec = spec + copyLine
 		}
 	}
 

pkg/compiler/types/spec/spec_suite_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compilerspec_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSpec(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Spec Suite")
+}

pkg/compiler/types/spec/spec_test.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
+	options "github.com/mudler/luet/pkg/compiler/types/options"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 
 	. "github.com/mudler/luet/pkg/compiler"
@@ -74,11 +75,67 @@ var _ = Describe("Spec", func() {
 		})
 	})
 
+	Context("Image hashing", func() {
+		It("is stable", func() {
+			spec1 := &compilerspec.LuetCompilationSpec{
+				Image:        "foo",
+				BuildOptions: &options.Compiler{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
+
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			spec2 := &compilerspec.LuetCompilationSpec{
+				Image:        "foo",
+				BuildOptions: &options.Compiler{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			spec3 := &compilerspec.LuetCompilationSpec{
+				Image: "foo",
+				Steps: []string{"foo"},
+				Package: &pkg.DefaultPackage{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			hash, err := spec1.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			hash2, err := spec2.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			hash3, err := spec3.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal(hash2))
+			hashagain, err := spec2.Hash()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hash).ToNot(Equal(hash3))
+			Expect(hash).To(Equal(hashagain))
+		})
+	})
+
 	Context("Simple package build definition", func() {
 		It("Loads it correctly", func() {
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
+			err := generalRecipe.Load("../../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
@@ -130,7 +187,7 @@ RUN echo bar > /test2`))
 	It("Renders retrieve and env fields", func() {
 		generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
-		err := generalRecipe.Load("../../tests/fixtures/retrieve")
+		err := generalRecipe.Load("../../../../tests/fixtures/retrieve")
 		Expect(err).ToNot(HaveOccurred())
 
 		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))

pkg/package/package.go

@@ -118,6 +118,8 @@ type Package interface {
 	SetTreeDir(s string)
 	GetTreeDir() string
 
+	Mark() Package
+
 	JSON() ([]byte, error)
 }
 
@@ -131,6 +133,19 @@ type Tree interface {
 
 type Packages []Package
 
+type DefaultPackages []*DefaultPackage
+
+func (d DefaultPackages) Hash(salt string) string {
+
+	overallFp := ""
+	for _, c := range d {
+		overallFp = overallFp + c.HashFingerprint("join")
+	}
+	h := md5.New()
+	io.WriteString(h, fmt.Sprintf("%s-%s", overallFp, salt))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
 // >> Unmarshallers
 // DefaultPackageFromYaml decodes a package from yaml bytes
 func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) {
@@ -492,6 +507,12 @@ func (p *DefaultPackage) Matches(m Package) bool {
 	return false
 }
 
+func (p *DefaultPackage) Mark() Package {
+	marked := p.Clone()
+	marked.SetName("@@" + marked.GetName())
+	return marked
+}
+
 func (p *DefaultPackage) Expand(definitiondb PackageDatabase) (Packages, error) {
 	var versionsInWorld Packages
 

pkg/solver/decoder.go

@@ -260,24 +260,42 @@ func (a PackagesAssertions) TrueLen() int {
 // and checks it's not the only one. if it's unique it marks it specially - so the hash
 // which is generated is unique for the selected package
 func (assertions PackagesAssertions) HashFrom(p pkg.Package) string {
+	return assertions.SaltedHashFrom(p, map[string]string{})
+}
 
+func (assertions PackagesAssertions) AssertionHash() string {
+	return assertions.SaltedAssertionHash(map[string]string{})
+}
+
+func (assertions PackagesAssertions) SaltedHashFrom(p pkg.Package, salts map[string]string) string {
 	var assertionhash string
 
 	// When we don't have any solution to hash for, we need to generate an UUID by ourselves
 	latestsolution := assertions.Drop(p)
 	if latestsolution.TrueLen() == 0 {
-		assertionhash = assertions.Mark(p).AssertionHash()
+		// Preserve the hash if supplied of marked packages
+		marked := p.Mark()
+		if markedHash, exists := salts[p.GetFingerPrint()]; exists {
+			salts[marked.GetFingerPrint()] = markedHash
+		}
+		assertionhash = assertions.Mark(p).SaltedAssertionHash(salts)
 	} else {
-		assertionhash = latestsolution.AssertionHash()
+		assertionhash = latestsolution.SaltedAssertionHash(salts)
 	}
 	return assertionhash
 }
 
-func (assertions PackagesAssertions) AssertionHash() string {
+func (assertions PackagesAssertions) SaltedAssertionHash(salts map[string]string) string {
 	var fingerprint string
 	for _, assertion := range assertions { // Note: Always order them first!
 		if assertion.Value { // Tke into account only dependencies installed (get fingerprint of subgraph)
-			fingerprint += assertion.ToString() + "\n"
+			salt, exists := salts[assertion.Package.GetFingerPrint()]
+			if exists {
+				fingerprint += assertion.ToString() + salt + "\n"
+
+			} else {
+				fingerprint += assertion.ToString() + "\n"
+			}
 		}
 	}
 	hash := sha256.Sum256([]byte(fingerprint))
@@ -316,8 +334,7 @@ func (assertions PackagesAssertions) Mark(p pkg.Package) PackagesAssertions {
 
 	for _, a := range assertions {
 		if a.Package.Matches(p) {
-			marked := a.Package.Clone()
-			marked.SetName("@@" + marked.GetName())
+			marked := a.Package.Mark()
 			a = PackageAssert{Package: marked.(*pkg.DefaultPackage), Value: a.Value, Hash: a.Hash}
 		}
 		ass = append(ass, a)

pkg/solver/decoder_test.go

@@ -382,6 +382,9 @@ var _ = Describe("Decoder", func() {
 
 			Expect(solution.HashFrom(X)).ToNot(Equal(solution2.HashFrom(F)))
 			Expect(solution3.HashFrom(D)).To(Equal(solution.HashFrom(X)))
+			Expect(solution3.SaltedHashFrom(D, map[string]string{D.GetFingerPrint(): "foo"})).ToNot(Equal(solution3.HashFrom(D)))
+
+			Expect(solution4.SaltedHashFrom(Y, map[string]string{X.GetFingerPrint(): "foo"})).ToNot(Equal(solution4.HashFrom(Y)))
 
 			Expect(empty.AssertionHash()).ToNot(Equal(solution3.HashFrom(D)))
 			Expect(empty.AssertionHash()).ToNot(Equal(solution2.HashFrom(F)))

tests/fixtures/copy/c/a/build.yaml

@@ -0,0 +1,17 @@
+image: "alpine"
+
+copy:
+  - package:
+      name: "a"
+      category: "test"
+      version: ">=0"
+    source: /test3
+    destination: /test3
+  - image: "busybox"
+    source: /bin/busybox
+    destination: /busybox
+
+steps:
+- mkdir /bina
+- cp /test3 /result
+- cp -rf /busybox /bina/busybox

tests/fixtures/copy/c/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.2"

tests/fixtures/copy/cat/a/a/build.yaml

@@ -0,0 +1,7 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/copy/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/copy/cat/b-1.1/build.yaml

@@ -0,0 +1,13 @@
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0"
+
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/copy/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/copy/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/fixtures/join/c/a/build.yaml

@@ -0,0 +1,8 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"
+unpack: true

tests/fixtures/join/c/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.2"

tests/fixtures/join/cat/a/a/build.yaml

@@ -0,0 +1,7 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/join/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/join/cat/b-1.1/build.yaml

@@ -0,0 +1,13 @@
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0"
+
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/join/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/join/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/fixtures/join_complex/c/c1/build.yaml

@@ -0,0 +1,7 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"

tests/fixtures/join_complex/c/c1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.2"

tests/fixtures/join_complex/c/f/build.yaml

@@ -0,0 +1,8 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"
+unpack: true

tests/fixtures/join_complex/c/f/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "f"
+version: "1.2"

tests/fixtures/join_complex/cat/a/a/build.yaml

@@ -0,0 +1,2 @@
+image: "alpine"
+unpack: true

tests/fixtures/join_complex/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/join_complex/cat/b-1.1/build.yaml

@@ -0,0 +1,13 @@
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0"
+
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/join_complex/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/join_complex/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/fixtures/join_complex/x/build.yaml

@@ -0,0 +1,10 @@
+join:
+- category: "test"
+  name: "f"
+  version: ">=0"
+# this is actually a virtual, no files shipped in c
+- category: "test"
+  name: "c"
+  version: ">=0"
+steps:
+- mv /newnewc /x

tests/fixtures/join_complex/x/definition.yaml

@@ -0,0 +1,3 @@
+name: "x"
+category: "test"
+version: "0.1"

tests/fixtures/join_complex/z/build.yaml

@@ -0,0 +1,6 @@
+requires:
+- category: "test"
+  name: "c"
+  version: ">=0"
+steps:
+- mv /newnewc /z

tests/fixtures/join_complex/z/definition.yaml

@@ -0,0 +1,3 @@
+name: "z"
+category: "test"
+version: "0.1"

tests/fixtures/upgrade_old_repo_revision_content_changed/c/build.yaml

@@ -0,0 +1,10 @@
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo c > /c
+  - echo c > /cd
+requires:
+- category: "test"
+  name: "a"
+  version: ">=1.0"

tests/fixtures/upgrade_old_repo_revision_content_changed/c/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.0"

tests/fixtures/upgrade_old_repo_revision_content_changed/cat/a/a/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+prelude:
+  - echo fozo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/upgrade_old_repo_revision_content_changed/cat/a/a/definition.yaml

@@ -0,0 +1,8 @@
+category: "test"
+name: "a"
+version: "1.1"
+requires:
+- category: "test2"
+  name: "b"
+  version: "1.0"
+

tests/fixtures/upgrade_old_repo_revision_content_changed/cat/b-1.1/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/upgrade_old_repo_revision_content_changed/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/fixtures/upgrade_old_repo_revision_content_changed/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/integration/28_nobuildtreedocker.sh

@@ -51,9 +51,9 @@ testBuild() {
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:d620e573c81eab36a9dc5cc314e80fd7b6e04aeff26127de4225bf24fe1f8e71"
-    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:b0f34b0d2d271f0f2619324476b2857b3b39ca895bddc2474a741f3c8c1acbbc"
-    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:c1f11f48113cd71d8795a06c7b49e1558bd7211d2aa88f5d79a3334f0393c64d"
+    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available remotely for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:ac34beb3fc2752bab54549db095dd6994d7531b88e1ff7f902d01ae80fdd030d"
+    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available remotely for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:d8b8881d50ae83646728bb28ad678fc14e4e003e4b0d0a66f8e6167c6116e024"
+    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2aaccd929ebe5683f95c70c6ec8d68f240ceea8633f3350b94904ad73da5fd47"
 }
 
 testRepo() {

tests/integration/29_nobuildtreeinheritvalues.sh

@@ -45,7 +45,7 @@ EOF
     mkdir $tmpdir/testbuild
     mkdir $tmpdir/empty
 
-    # With --rebuild, the package gets ignored
+    # Without --rebuild, the package gets ignored
     build_output=$(luet build --pull --tree "$tmpdir/empty" \
     --config $tmpdir/luet.yaml --values $tmpdir/default.yaml --concurrency 1 \
     --from-repositories --destination $tmpdir/testbuild --compression zstd test/c@1.0 test/z test/interpolated)
@@ -56,7 +56,8 @@ EOF
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Images available for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:c1f11f48113cd71d8795a06c7b49e1558bd7211d2aa88f5d79a3334f0393c64d"
+    assertNotContains 'Does NOT use the upstream cache without specifying it' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2aaccd929ebe5683f95c70c6ec8d68f240ceea8633f3350b94904ad73da5fd47"
+    assertContains 'Does generate a new hash as values changed build.yaml for test/interpolated-1.0+2 package image' "$build_output" "Building image luet/cache:c34b533cf76886c332fe9b2d3208f04265360a465a90c996cb4fcdaf959dee36 done"
 }
 
 testRepo() {
@@ -113,7 +114,7 @@ testInstall() {
     assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
     assertTrue 'package Z installed' "[ -e '$tmpdir/testrootfs/z' ]"
     ls -liah $tmpdir/testrootfs/
-    assertTrue 'package interpolated installed' "[ -e '$tmpdir/testrootfs/interpolated-baz-bar' ]"
+    assertTrue 'package interpolated installed' "[ -e '$tmpdir/testrootfs/interpolated-baz-an' ]"
 }
 
 testReInstall() {

tests/integration/29_nobuildtreeinheritvalues_noignored.sh

@@ -56,7 +56,7 @@ EOF
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Downloading image quay.io/mocaccinoos/integration-test-cache:6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"
+    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Downloading image quay.io/mocaccinoos/integration-test-cache:4db24406e8db30a3310a1cf8c4d4e19597745e6d41b189dc51a73ac4a50cc9e6"
 }
 
 testRepo() {

tests/integration/30_copy.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    cat <<EOF > $tmpdir/default.yaml
+extra: "bar"
+foo: "baz"
+EOF
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/copy" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --compression gzip --values $tmpdir/default.yaml \
+               test/c
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package c' "[ -e '$tmpdir/testbuild/c-test-1.2.package.tar.gz' ]"
+    mkdir $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/c-test-1.2.package.tar.gz -C $tmpdir/extract
+    assertTrue 'create result in package c' "[ -e '$tmpdir/extract/result' ]"
+    assertTrue 'create busybox in package c' "[ -f '$tmpdir/extract/bina/busybox' ]"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/31_join.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    cat <<EOF > $tmpdir/default.yaml
+extra: "bar"
+foo: "baz"
+EOF
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/join" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --compression gzip --values $tmpdir/default.yaml \
+               test/c
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package c' "[ -e '$tmpdir/testbuild/c-test-1.2.package.tar.gz' ]"
+    mkdir $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/c-test-1.2.package.tar.gz -C $tmpdir/extract
+    assertTrue 'create result from join' "[ -e '$tmpdir/extract/test3' ]"
+    assertTrue 'create result from join' "[ -f '$tmpdir/extract/newc' ]"
+    assertTrue 'create result from join' "[ -e '$tmpdir/extract/test4' ]"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/31_join_complex.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/join_complex" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --compression gzip \
+               test/z test/x
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package z' "[ -e '$tmpdir/testbuild/z-test-0.1.package.tar.gz' ]"
+    assertTrue 'create package z' "[ -e '$tmpdir/testbuild/x-test-0.1.package.tar.gz' ]"
+
+    mkdir $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/x-test-0.1.package.tar.gz -C $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/z-test-0.1.package.tar.gz -C $tmpdir/extract
+    assertTrue 'create result from a package that requires a join' "[ -e '$tmpdir/extract/z' ]"
+    assertTrue 'create result from join of a join' "[ -e '$tmpdir/extract/x' ]"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-
+export LUET_NO_SPINNER=true
 export LUET_YES=true
 export ROOT_DIR="$(git rev-parse --show-toplevel)"
 

vendor/github.com/mitchellh/hashstructure/v2/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Mitchell Hashimoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/mitchellh/hashstructure/v2/README.md

@@ -0,0 +1,76 @@
+# hashstructure [![GoDoc](https://godoc.org/github.com/mitchellh/hashstructure?status.svg)](https://godoc.org/github.com/mitchellh/hashstructure)
+
+hashstructure is a Go library for creating a unique hash value
+for arbitrary values in Go.
+
+This can be used to key values in a hash (for use in a map, set, etc.)
+that are complex. The most common use case is comparing two values without
+sending data across the network, caching values locally (de-dup), and so on.
+
+## Features
+
+  * Hash any arbitrary Go value, including complex types.
+
+  * Tag a struct field to ignore it and not affect the hash value.
+
+  * Tag a slice type struct field to treat it as a set where ordering
+    doesn't affect the hash code but the field itself is still taken into
+    account to create the hash value.
+
+  * Optionally, specify a custom hash function to optimize for speed, collision
+    avoidance for your data set, etc.
+
+  * Optionally, hash the output of `.String()` on structs that implement fmt.Stringer,
+    allowing effective hashing of time.Time
+
+  * Optionally, override the hashing process by implementing `Hashable`.
+
+## Installation
+
+Standard `go get`:
+
+```
+$ go get github.com/mitchellh/hashstructure/v2
+```
+
+**Note on v2:** It is highly recommended you use the "v2" release since this
+fixes some significant hash collisions issues from v1. In practice, we used
+v1 for many years in real projects at HashiCorp and never had issues, but it
+is highly dependent on the shape of the data you're hashing and how you use
+those hashes.
+
+When using v2+, you can still generate weaker v1 hashes by using the
+`FormatV1` format when calling `Hash`.
+
+## Usage & Example
+
+For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/hashstructure).
+
+A quick code example is shown below:
+
+```go
+type ComplexStruct struct {
+    Name     string
+    Age      uint
+    Metadata map[string]interface{}
+}
+
+v := ComplexStruct{
+    Name: "mitchellh",
+    Age:  64,
+    Metadata: map[string]interface{}{
+        "car":      true,
+        "location": "California",
+        "siblings": []string{"Bob", "John"},
+    },
+}
+
+hash, err := hashstructure.Hash(v, nil)
+if err != nil {
+    panic(err)
+}
+
+fmt.Printf("%d", hash)
+// Output:
+// 2307517237273902113
+```

vendor/github.com/mitchellh/hashstructure/v2/errors.go

@@ -0,0 +1,22 @@
+package hashstructure
+
+import (
+	"fmt"
+)
+
+// ErrNotStringer is returned when there's an error with hash:"string"
+type ErrNotStringer struct {
+	Field string
+}
+
+// Error implements error for ErrNotStringer
+func (ens *ErrNotStringer) Error() string {
+	return fmt.Sprintf("hashstructure: %s has hash:\"string\" set, but does not implement fmt.Stringer", ens.Field)
+}
+
+// ErrFormat is returned when an invalid format is given to the Hash function.
+type ErrFormat struct{}
+
+func (*ErrFormat) Error() string {
+	return "format must be one of the defined Format values in the hashstructure library"
+}

vendor/github.com/mitchellh/hashstructure/v2/go.mod

@@ -0,0 +1,3 @@
+module github.com/mitchellh/hashstructure/v2
+
+go 1.14

vendor/github.com/mitchellh/hashstructure/v2/hashstructure.go

@@ -0,0 +1,483 @@
+package hashstructure
+
+import (
+	"encoding/binary"
+	"fmt"
+	"hash"
+	"hash/fnv"
+	"reflect"
+	"time"
+)
+
+// HashOptions are options that are available for hashing.
+type HashOptions struct {
+	// Hasher is the hash function to use. If this isn't set, it will
+	// default to FNV.
+	Hasher hash.Hash64
+
+	// TagName is the struct tag to look at when hashing the structure.
+	// By default this is "hash".
+	TagName string
+
+	// ZeroNil is flag determining if nil pointer should be treated equal
+	// to a zero value of pointed type. By default this is false.
+	ZeroNil bool
+
+	// IgnoreZeroValue is determining if zero value fields should be
+	// ignored for hash calculation.
+	IgnoreZeroValue bool
+
+	// SlicesAsSets assumes that a `set` tag is always present for slices.
+	// Default is false (in which case the tag is used instead)
+	SlicesAsSets bool
+
+	// UseStringer will attempt to use fmt.Stringer aways. If the struct
+	// doesn't implement fmt.Stringer, it'll fall back to trying usual tricks.
+	// If this is true, and the "string" tag is also set, the tag takes
+	// precedense (meaning that if the type doesn't implement fmt.Stringer, we
+	// panic)
+	UseStringer bool
+}
+
+// Format specifies the hashing process used. Different formats typically
+// generate different hashes for the same value and have different properties.
+type Format uint
+
+const (
+	// To disallow the zero value
+	formatInvalid Format = iota
+
+	// FormatV1 is the format used in v1.x of this library. This has the
+	// downsides noted in issue #18 but allows simultaneous v1/v2 usage.
+	FormatV1
+
+	// FormatV2 is the current recommended format and fixes the issues
+	// noted in FormatV1.
+	FormatV2
+
+	formatMax // so we can easily find the end
+)
+
+// Hash returns the hash value of an arbitrary value.
+//
+// If opts is nil, then default options will be used. See HashOptions
+// for the default values. The same *HashOptions value cannot be used
+// concurrently. None of the values within a *HashOptions struct are
+// safe to read/write while hashing is being done.
+//
+// The "format" is required and must be one of the format values defined
+// by this library. You should probably just use "FormatV2". This allows
+// generated hashes uses alternate logic to maintain compatibility with
+// older versions.
+//
+// Notes on the value:
+//
+//   * Unexported fields on structs are ignored and do not affect the
+//     hash value.
+//
+//   * Adding an exported field to a struct with the zero value will change
+//     the hash value.
+//
+// For structs, the hashing can be controlled using tags. For example:
+//
+//    struct {
+//        Name string
+//        UUID string `hash:"ignore"`
+//    }
+//
+// The available tag values are:
+//
+//   * "ignore" or "-" - The field will be ignored and not affect the hash code.
+//
+//   * "set" - The field will be treated as a set, where ordering doesn't
+//             affect the hash code. This only works for slices.
+//
+//   * "string" - The field will be hashed as a string, only works when the
+//                field implements fmt.Stringer
+//
+func Hash(v interface{}, format Format, opts *HashOptions) (uint64, error) {
+	// Validate our format
+	if format <= formatInvalid || format >= formatMax {
+		return 0, &ErrFormat{}
+	}
+
+	// Create default options
+	if opts == nil {
+		opts = &HashOptions{}
+	}
+	if opts.Hasher == nil {
+		opts.Hasher = fnv.New64()
+	}
+	if opts.TagName == "" {
+		opts.TagName = "hash"
+	}
+
+	// Reset the hash
+	opts.Hasher.Reset()
+
+	// Create our walker and walk the structure
+	w := &walker{
+		format:          format,
+		h:               opts.Hasher,
+		tag:             opts.TagName,
+		zeronil:         opts.ZeroNil,
+		ignorezerovalue: opts.IgnoreZeroValue,
+		sets:            opts.SlicesAsSets,
+		stringer:        opts.UseStringer,
+	}
+	return w.visit(reflect.ValueOf(v), nil)
+}
+
+type walker struct {
+	format          Format
+	h               hash.Hash64
+	tag             string
+	zeronil         bool
+	ignorezerovalue bool
+	sets            bool
+	stringer        bool
+}
+
+type visitOpts struct {
+	// Flags are a bitmask of flags to affect behavior of this visit
+	Flags visitFlag
+
+	// Information about the struct containing this field
+	Struct      interface{}
+	StructField string
+}
+
+var timeType = reflect.TypeOf(time.Time{})
+
+func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
+	t := reflect.TypeOf(0)
+
+	// Loop since these can be wrapped in multiple layers of pointers
+	// and interfaces.
+	for {
+		// If we have an interface, dereference it. We have to do this up
+		// here because it might be a nil in there and the check below must
+		// catch that.
+		if v.Kind() == reflect.Interface {
+			v = v.Elem()
+			continue
+		}
+
+		if v.Kind() == reflect.Ptr {
+			if w.zeronil {
+				t = v.Type().Elem()
+			}
+			v = reflect.Indirect(v)
+			continue
+		}
+
+		break
+	}
+
+	// If it is nil, treat it like a zero.
+	if !v.IsValid() {
+		v = reflect.Zero(t)
+	}
+
+	// Binary writing can use raw ints, we have to convert to
+	// a sized-int, we'll choose the largest...
+	switch v.Kind() {
+	case reflect.Int:
+		v = reflect.ValueOf(int64(v.Int()))
+	case reflect.Uint:
+		v = reflect.ValueOf(uint64(v.Uint()))
+	case reflect.Bool:
+		var tmp int8
+		if v.Bool() {
+			tmp = 1
+		}
+		v = reflect.ValueOf(tmp)
+	}
+
+	k := v.Kind()
+
+	// We can shortcut numeric values by directly binary writing them
+	if k >= reflect.Int && k <= reflect.Complex64 {
+		// A direct hash calculation
+		w.h.Reset()
+		err := binary.Write(w.h, binary.LittleEndian, v.Interface())
+		return w.h.Sum64(), err
+	}
+
+	switch v.Type() {
+	case timeType:
+		w.h.Reset()
+		b, err := v.Interface().(time.Time).MarshalBinary()
+		if err != nil {
+			return 0, err
+		}
+
+		err = binary.Write(w.h, binary.LittleEndian, b)
+		return w.h.Sum64(), err
+	}
+
+	switch k {
+	case reflect.Array:
+		var h uint64
+		l := v.Len()
+		for i := 0; i < l; i++ {
+			current, err := w.visit(v.Index(i), nil)
+			if err != nil {
+				return 0, err
+			}
+
+			h = hashUpdateOrdered(w.h, h, current)
+		}
+
+		return h, nil
+
+	case reflect.Map:
+		var includeMap IncludableMap
+		if opts != nil && opts.Struct != nil {
+			if v, ok := opts.Struct.(IncludableMap); ok {
+				includeMap = v
+			}
+		}
+
+		// Build the hash for the map. We do this by XOR-ing all the key
+		// and value hashes. This makes it deterministic despite ordering.
+		var h uint64
+		for _, k := range v.MapKeys() {
+			v := v.MapIndex(k)
+			if includeMap != nil {
+				incl, err := includeMap.HashIncludeMap(
+					opts.StructField, k.Interface(), v.Interface())
+				if err != nil {
+					return 0, err
+				}
+				if !incl {
+					continue
+				}
+			}
+
+			kh, err := w.visit(k, nil)
+			if err != nil {
+				return 0, err
+			}
+			vh, err := w.visit(v, nil)
+			if err != nil {
+				return 0, err
+			}
+
+			fieldHash := hashUpdateOrdered(w.h, kh, vh)
+			h = hashUpdateUnordered(h, fieldHash)
+		}
+
+		if w.format != FormatV1 {
+			// Important: read the docs for hashFinishUnordered
+			h = hashFinishUnordered(w.h, h)
+		}
+
+		return h, nil
+
+	case reflect.Struct:
+		parent := v.Interface()
+		var include Includable
+		if impl, ok := parent.(Includable); ok {
+			include = impl
+		}
+
+		if impl, ok := parent.(Hashable); ok {
+			return impl.Hash()
+		}
+
+		// If we can address this value, check if the pointer value
+		// implements our interfaces and use that if so.
+		if v.CanAddr() {
+			vptr := v.Addr()
+			parentptr := vptr.Interface()
+			if impl, ok := parentptr.(Includable); ok {
+				include = impl
+			}
+
+			if impl, ok := parentptr.(Hashable); ok {
+				return impl.Hash()
+			}
+		}
+
+		t := v.Type()
+		h, err := w.visit(reflect.ValueOf(t.Name()), nil)
+		if err != nil {
+			return 0, err
+		}
+
+		l := v.NumField()
+		for i := 0; i < l; i++ {
+			if innerV := v.Field(i); v.CanSet() || t.Field(i).Name != "_" {
+				var f visitFlag
+				fieldType := t.Field(i)
+				if fieldType.PkgPath != "" {
+					// Unexported
+					continue
+				}
+
+				tag := fieldType.Tag.Get(w.tag)
+				if tag == "ignore" || tag == "-" {
+					// Ignore this field
+					continue
+				}
+
+				if w.ignorezerovalue {
+					zeroVal := reflect.Zero(reflect.TypeOf(innerV.Interface())).Interface()
+					if innerV.Interface() == zeroVal {
+						continue
+					}
+				}
+
+				// if string is set, use the string value
+				if tag == "string" || w.stringer {
+					if impl, ok := innerV.Interface().(fmt.Stringer); ok {
+						innerV = reflect.ValueOf(impl.String())
+					} else if tag == "string" {
+						// We only show this error if the tag explicitly
+						// requests a stringer.
+						return 0, &ErrNotStringer{
+							Field: v.Type().Field(i).Name,
+						}
+					}
+				}
+
+				// Check if we implement includable and check it
+				if include != nil {
+					incl, err := include.HashInclude(fieldType.Name, innerV)
+					if err != nil {
+						return 0, err
+					}
+					if !incl {
+						continue
+					}
+				}
+
+				switch tag {
+				case "set":
+					f |= visitFlagSet
+				}
+
+				kh, err := w.visit(reflect.ValueOf(fieldType.Name), nil)
+				if err != nil {
+					return 0, err
+				}
+
+				vh, err := w.visit(innerV, &visitOpts{
+					Flags:       f,
+					Struct:      parent,
+					StructField: fieldType.Name,
+				})
+				if err != nil {
+					return 0, err
+				}
+
+				fieldHash := hashUpdateOrdered(w.h, kh, vh)
+				h = hashUpdateUnordered(h, fieldHash)
+			}
+
+			if w.format != FormatV1 {
+				// Important: read the docs for hashFinishUnordered
+				h = hashFinishUnordered(w.h, h)
+			}
+		}
+
+		return h, nil
+
+	case reflect.Slice:
+		// We have two behaviors here. If it isn't a set, then we just
+		// visit all the elements. If it is a set, then we do a deterministic
+		// hash code.
+		var h uint64
+		var set bool
+		if opts != nil {
+			set = (opts.Flags & visitFlagSet) != 0
+		}
+		l := v.Len()
+		for i := 0; i < l; i++ {
+			current, err := w.visit(v.Index(i), nil)
+			if err != nil {
+				return 0, err
+			}
+
+			if set || w.sets {
+				h = hashUpdateUnordered(h, current)
+			} else {
+				h = hashUpdateOrdered(w.h, h, current)
+			}
+		}
+
+		if set && w.format != FormatV1 {
+			// Important: read the docs for hashFinishUnordered
+			h = hashFinishUnordered(w.h, h)
+		}
+
+		return h, nil
+
+	case reflect.String:
+		// Directly hash
+		w.h.Reset()
+		_, err := w.h.Write([]byte(v.String()))
+		return w.h.Sum64(), err
+
+	default:
+		return 0, fmt.Errorf("unknown kind to hash: %s", k)
+	}
+
+}
+
+func hashUpdateOrdered(h hash.Hash64, a, b uint64) uint64 {
+	// For ordered updates, use a real hash function
+	h.Reset()
+
+	// We just panic if the binary writes fail because we are writing
+	// an int64 which should never be fail-able.
+	e1 := binary.Write(h, binary.LittleEndian, a)
+	e2 := binary.Write(h, binary.LittleEndian, b)
+	if e1 != nil {
+		panic(e1)
+	}
+	if e2 != nil {
+		panic(e2)
+	}
+
+	return h.Sum64()
+}
+
+func hashUpdateUnordered(a, b uint64) uint64 {
+	return a ^ b
+}
+
+// After mixing a group of unique hashes with hashUpdateUnordered, it's always
+// necessary to call hashFinishUnordered. Why? Because hashUpdateUnordered
+// is a simple XOR, and calling hashUpdateUnordered on hashes produced by
+// hashUpdateUnordered can effectively cancel out a previous change to the hash
+// result if the same hash value appears later on. For example, consider:
+//
+//   hashUpdateUnordered(hashUpdateUnordered("A", "B"), hashUpdateUnordered("A", "C")) =
+//   H("A") ^ H("B")) ^ (H("A") ^ H("C")) =
+//   (H("A") ^ H("A")) ^ (H("B") ^ H(C)) =
+//   H(B) ^ H(C) =
+//   hashUpdateUnordered(hashUpdateUnordered("Z", "B"), hashUpdateUnordered("Z", "C"))
+//
+// hashFinishUnordered "hardens" the result, so that encountering partially
+// overlapping input data later on in a different context won't cancel out.
+func hashFinishUnordered(h hash.Hash64, a uint64) uint64 {
+	h.Reset()
+
+	// We just panic if the writes fail
+	e1 := binary.Write(h, binary.LittleEndian, a)
+	if e1 != nil {
+		panic(e1)
+	}
+
+	return h.Sum64()
+}
+
+// visitFlag is used as a bitmask for affecting visit behavior
+type visitFlag uint
+
+const (
+	visitFlagInvalid visitFlag = iota
+	visitFlagSet               = iota << 1
+)

vendor/github.com/mitchellh/hashstructure/v2/include.go

@@ -0,0 +1,22 @@
+package hashstructure
+
+// Includable is an interface that can optionally be implemented by
+// a struct. It will be called for each field in the struct to check whether
+// it should be included in the hash.
+type Includable interface {
+	HashInclude(field string, v interface{}) (bool, error)
+}
+
+// IncludableMap is an interface that can optionally be implemented by
+// a struct. It will be called when a map-type field is found to ask the
+// struct if the map item should be included in the hash.
+type IncludableMap interface {
+	HashIncludeMap(field string, k, v interface{}) (bool, error)
+}
+
+// Hashable is an interface that can optionally be implemented by a struct
+// to override the hash value. This value will override the hash value for
+// the entire struct. Entries in the struct will not be hashed.
+type Hashable interface {
+	Hash() (uint64, error)
+}

vendor/modules.txt

@@ -376,6 +376,9 @@ github.com/mitchellh/colorstring
 github.com/mitchellh/copystructure
 # github.com/mitchellh/hashstructure v1.0.0
 github.com/mitchellh/hashstructure
+# github.com/mitchellh/hashstructure/v2 v2.0.1
+## explicit
+github.com/mitchellh/hashstructure/v2
 # github.com/mitchellh/mapstructure v1.1.2
 github.com/mitchellh/mapstructure
 # github.com/mitchellh/reflectwalk v1.0.0