Add join keyword to generate parent image from final artifacts

A new keyword `join` is introduced to generate the parent image. It
takes precedence over a `requires` or a `image` already defined in a
spec.

It will generate all the artifacts from the packages listed and join
them in a single image which will be used as parent for the package
build process.

This is a change which invalidates priorly generated hashes.

Fixes #173

pkg/compiler/compiler.go

@@ -16,7 +16,9 @@
 package compiler
 
 import (
+	"crypto/md5"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -506,7 +508,7 @@ func oneOfImagesAvailable(images []string, b CompilerBackend) (bool, string) {
 	return false, ""
 }
 
-func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
+func (cs *LuetCompiler) findImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
 	var resolvedImage string
 	Debug("Resolving image hash for", p.Package.HumanReadableString(), "hash", imageHash, "Pull repositories", p.BuildOptions.PullImageRepository)
 	toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)},
@@ -519,6 +521,11 @@ func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilersp
 			resolvedImage = which
 		}
 	}
+	return resolvedImage
+}
+
+func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
+	resolvedImage := cs.findImageHash(imageHash, p)
 
 	if resolvedImage == "" {
 		resolvedImage = fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)
@@ -745,6 +752,102 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS
 	Debug(p.GetPackage().HumanReadableString(), "Build options after inherit", p.BuildOptions)
 }
 
+func (cs *LuetCompiler) getSpecHash(pkgs pkg.DefaultPackages, salt string) (string, error) {
+	ht := NewHashTree(cs.Database)
+	overallFp := ""
+
+	for _, p := range pkgs {
+		compileSpec, err := cs.FromPackage(p)
+		if err != nil {
+			return "", errors.Wrap(err, "Error while generating compilespec for "+p.GetName())
+		}
+		packageHashTree, err := ht.Query(cs, compileSpec)
+		if err != nil {
+			return "nil", errors.Wrap(err, "failed querying hashtree")
+		}
+		overallFp = overallFp + packageHashTree.Target.Hash.PackageHash + p.GetFingerPrint()
+	}
+
+	h := md5.New()
+	io.WriteString(h, fmt.Sprintf("%s-%s", overallFp, salt))
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
+func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+	if len(p.Join) != 0 {
+		Info("Generating a joint parent image from final packages")
+	} else {
+		return nil
+	}
+
+	// First compute a hash and check if image is available. if it is, then directly consume that
+	overallFp, err := cs.getSpecHash(p.Join, "join")
+	if err != nil {
+		return errors.Wrap(err, "could not generate image hash")
+	}
+	image := cs.findImageHash(overallFp, p)
+	if image != "" {
+		Info("Image already found", image)
+		p.SetImage(image)
+		return nil
+	}
+	Info("Generating image with hash ", image)
+
+	// otherwise, generate it and push it aside
+	joinDir, err := ioutil.TempDir(p.GetOutputPath(), "join")
+	if err != nil {
+		return errors.Wrap(err, "Could not create tempdir")
+	}
+	defer os.RemoveAll(joinDir) // clean up
+
+	for _, c := range p.Join {
+		if c != nil && c.Name != "" && c.Version != "" {
+			Info(" :droplet: generating", c.HumanReadableString())
+			spec, err := cs.FromPackage(c)
+			if err != nil {
+				return errors.Wrap(err, "while generating images to join from")
+			}
+			wantsArtifact := true
+			artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, spec)
+			if err != nil {
+				return errors.Wrap(err, "failed building join image")
+			}
+
+			err = artifact.Unpack(joinDir, keepPermissions)
+			if err != nil {
+				return errors.Wrap(err, "failed building join image")
+			}
+		}
+	}
+
+	artifactDir, err := ioutil.TempDir(p.GetOutputPath(), "artifact")
+	if err != nil {
+		return errors.Wrap(err, "Could not create tempdir")
+	}
+	defer os.RemoveAll(joinDir) // clean up
+
+	// After unpack, create a new artifact and a new final image from it.
+	// no need to compress, as we are going to toss it away.
+	a := artifact.NewPackageArtifact(filepath.Join(artifactDir, p.GetPackage().GetFingerPrint()+".join.tar"))
+	if err := a.Compress(joinDir, concurrency); err != nil {
+		return errors.Wrap(err, "Error met while creating package archive")
+	}
+
+	joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp)
+	opts, err := a.GenerateFinalImage(joinImageName, cs.Backend, keepPermissions)
+	if err != nil {
+		return errors.Wrap(err, "Could not create tempdir")
+	}
+	if cs.Options.Push {
+		if err = cs.Backend.Push(opts); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
+		}
+	}
+	Info("Using image ", joinImageName)
+	p.SetImage(joinImageName)
+	return nil
+}
+
 func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
 	resolvedCopyFields := []compilerspec.CopyField{}
 	if len(p.Copy) != 0 {
@@ -780,6 +883,12 @@ func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
 	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
+	//Before multistage : join - same as multistage, but keep artifacts, join them, create a new one and generate a final image.
+	// When the image is there, use it as a source here, in place of GetImage().
+	if err := cs.resolveJoinImages(concurrency, keepPermissions, p); err != nil {
+		return nil, errors.Wrap(err, "while resolving join images")
+	}
+
 	if err := cs.resolveMultiStageImages(concurrency, keepPermissions, p); err != nil {
 		return nil, errors.Wrap(err, "while resolving multi-stage images")
 	}

pkg/compiler/compiler_test.go

@@ -75,6 +75,68 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Copy and Join", func() {
+		It("Compiles it correctly with Copy", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/copy")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("result"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("bina/busybox"))).To(BeTrue())
+		})
+
+		It("Compiles it correctly with Join", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/join")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.2"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+			Expect(helpers.Exists(spec.Rel("newc"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
+		})
+	})
+
 	Context("Simple package build definition", func() {
 		It("Compiles it in parallel", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))

pkg/compiler/imagehashtree.go

@@ -25,11 +25,22 @@ import (
 	"github.com/pkg/errors"
 )
 
+// ImageHashTree is holding the Database
+// and the options to resolve PackageImageHashTrees
+// for a given specfile
+// It is responsible of returning a concrete result
+// which identifies a Package in a HashTree
 type ImageHashTree struct {
 	Database      pkg.PackageDatabase
 	SolverOptions config.LuetSolverOptions
 }
 
+// PackageImageHashTree represent the Package into a given image hash tree
+// The hash tree is constructed by a set of images representing
+// the package during its build stage. A Hash is assigned to each image
+// from the package fingerprint, plus the SAT solver assertion result (which is hashed as well)
+// and the specfile signatures. This guarantees that each image of the build stage
+// is unique and can be identified later on.
 type PackageImageHashTree struct {
 	Target                       *solver.PackageAssert
 	Dependencies                 solver.PackagesAssertions
@@ -53,11 +64,19 @@ func (ht *PackageImageHashTree) DependencyBuildImage(p pkg.Package) (string, err
 	return found, nil
 }
 
-// TODO: ___ When computing the hash per package (and evaluating the sat solver solution tree part)
-// we should use the hash of each package  + its fingerprint  instead as a salt.
-// That's because the hash will be salted with its `build.yaml`.
-// In this way, we trigger recompilations if some dep of a target changes
-// a build.yaml, without touching the version
+func (ht *PackageImageHashTree) String() string {
+	return fmt.Sprintf(
+		"Target buildhash: %s\nTarget packagehash: %s\nBuilder Imagehash: %s\nSource Imagehash: %s\n",
+		ht.Target.Hash.BuildHash,
+		ht.Target.Hash.PackageHash,
+		ht.BuilderImageHash,
+		ht.SourceHash,
+	)
+}
+
+// Query takes a compiler and a compilation spec and returns a PackageImageHashTree tied to it.
+// PackageImageHashTree contains all the informations to resolve the spec build images in order to
+// reproducibly re-build images from packages
 func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) {
 	assertions, err := ht.resolve(cs, p)
 	if err != nil {

pkg/compiler/imagehashtree_test.go

@@ -46,14 +46,13 @@ var _ = Describe("ImageHashTree", func() {
 
 			packageHash, err := hashtree.Query(compiler, spec)
 			Expect(err).ToNot(HaveOccurred())
-
-			Expect(packageHash.Target.Hash.BuildHash).To(Equal("ec62e3e2cfb4c520c8b2561797c005d248c2659295f3660fa1a66582fc4dc280"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("5fa15a0eb0534eaa78ef1b4e32fe72704effaa5e54399b7cab6d630aa0aeac5c"))
-			Expect(packageHash.BuilderImageHash).To(Equal("builder-96e0c42b5741376ebcf0a47c8ec1c481"))
+			Expect(packageHash.Target.Hash.BuildHash).To(Equal("4db24406e8db30a3310a1cf8c4d4e19597745e6d41b189dc51a73ac4a50cc9e6"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("4c867c9bab6c71d9420df75806e7a2f171dbc08487852ab4e2487bab04066cf2"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-e6f9c5552a67c463215b0a9e4f7c7fc8"))
 		})
 	})
 
-	expectedPackageHash := "bc6d354e8b9480b70c6f17eafa34cef387b8443ad150b7c9528fb7e94b764e90"
+	expectedPackageHash := "15811d83d0f8360318c54d91dcae3714f8efb39bf872572294834880f00ee7a8"
 
 	Context("complex package definition", func() {
 		BeforeEach(func() {
@@ -74,24 +73,23 @@ var _ = Describe("ImageHashTree", func() {
 
 			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(expectedPackageHash))
 			Expect(packageHash.SourceHash).To(Equal(expectedPackageHash))
-			Expect(packageHash.BuilderImageHash).To(Equal("builder-9b2bc16985446c41eca8f7922ec98078"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-3d739cab442aec15a6da238481df73c5"))
 
 			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("bb84a30ced857725fcb575e87fe33d4aefe911abfdd5f9063bbaeb9e4b94e9e2"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("99c4ebb4bc4754985fcc28677badf90f525aa231b1db0fe75659f11b86dc20e8"))
 			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
 			hash, err := packageHash.DependencyBuildImage(a)
 			Expect(err).ToNot(HaveOccurred())
-
-			Expect(hash).To(Equal("484f14294d96fd3b51cec1f2db37a269b7b903f3516b74b0cb0771b65d85b799"))
+			Expect(hash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
 
 			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
 			Expect(assertionA.Hash.PackageHash).To(Equal(expectedPackageHash))
 			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
 			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
-			Expect(assertionB.Hash.PackageHash).To(Equal("484f14294d96fd3b51cec1f2db37a269b7b903f3516b74b0cb0771b65d85b799"))
+			Expect(assertionB.Hash.PackageHash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
 			hashB, err := packageHash.DependencyBuildImage(b)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(hashB).To(Equal("828c983e755353190540565a29e71c9eb4c48d6303e1fd2c523235b7c2339c73"))
+			Expect(hashB).To(Equal("2668e418eab6861404834ad617713e39b8e58f68016a1fbfcc9384efdd037376"))
 		})
 	})
 
@@ -114,33 +112,33 @@ var _ = Describe("ImageHashTree", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).ToNot(Equal(expectedPackageHash))
-			sourceHash := "ed1bd90e696904982a1f51998646a335067329e1a262994b5ae15c579106ac81"
+			sourceHash := "1d91b13d0246fa000085a1071c63397d21546300b17f69493f22315a64b717d4"
 			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(sourceHash))
 			Expect(packageHash.SourceHash).To(Equal(sourceHash))
 			Expect(packageHash.SourceHash).ToNot(Equal(expectedPackageHash))
 
-			Expect(packageHash.BuilderImageHash).To(Equal("builder-f4b0e366e0a42774428fbdc9aa325648"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-03ee108a7c56b17ee568ace0800dd16d"))
 
 			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
-			Expect(packageHash.Target.Hash.PackageHash).To(Equal("2618f12851a596f6801e2665e07147da98a0a151f44500a54ca8b76b869e378d"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("7677da23b2cc866c2d07aa4a58fbf703340f2f78c0efbb1ba9faf8979f250c87"))
 			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
 			hash, err := packageHash.DependencyBuildImage(a)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(hash).To(Equal("484f14294d96fd3b51cec1f2db37a269b7b903f3516b74b0cb0771b65d85b799"))
+			Expect(hash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
 
 			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
 
-			Expect(assertionA.Hash.PackageHash).To(Equal("ed1bd90e696904982a1f51998646a335067329e1a262994b5ae15c579106ac81"))
+			Expect(assertionA.Hash.PackageHash).To(Equal("1d91b13d0246fa000085a1071c63397d21546300b17f69493f22315a64b717d4"))
 			Expect(assertionA.Hash.PackageHash).ToNot(Equal(expectedPackageHash))
 
 			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
 			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
 
-			Expect(assertionB.Hash.PackageHash).To(Equal("484f14294d96fd3b51cec1f2db37a269b7b903f3516b74b0cb0771b65d85b799"))
+			Expect(assertionB.Hash.PackageHash).To(Equal("f48f28ab62f1379a4247ec763681ccede68ea1e5c25aae8fb72459c0b2f8742e"))
 			hashB, err := packageHash.DependencyBuildImage(b)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(hashB).To(Equal("828c983e755353190540565a29e71c9eb4c48d6303e1fd2c523235b7c2339c73"))
+			Expect(hashB).To(Equal("2668e418eab6861404834ad617713e39b8e58f68016a1fbfcc9384efdd037376"))
 		})
 	})
 

pkg/compiler/types/spec/spec.go

@@ -114,6 +114,41 @@ type LuetCompilationSpec struct {
 	BuildOptions *options.Compiler `json:"build_options"`
 
 	Copy []CopyField `json:"copy"`
+
+	Join pkg.DefaultPackages `json:"join"`
+}
+
+// Signature is a portion of the spec that yields a signature for the hash
+type Signature struct {
+	Image      string
+	Steps      []string
+	PackageDir string
+	Prelude    []string
+	Seed       string
+	Env        []string
+	Retrieve   []string
+	Unpack     bool
+	Includes   []string
+	Excludes   []string
+	Copy       []CopyField
+	Join       pkg.DefaultPackages
+}
+
+func (cs *LuetCompilationSpec) signature() Signature {
+	return Signature{
+		Image:      cs.Image,
+		Steps:      cs.Steps,
+		PackageDir: cs.PackageDir,
+		Prelude:    cs.Prelude,
+		Seed:       cs.Seed,
+		Env:        cs.Env,
+		Retrieve:   cs.Retrieve,
+		Unpack:     cs.Unpack,
+		Includes:   cs.Includes,
+		Excludes:   cs.Excludes,
+		Copy:       cs.Copy,
+		Join:       cs.Join,
+	}
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
@@ -224,38 +259,7 @@ func (cs *LuetCompilationSpec) UnpackedPackage() bool {
 // a compilation spec has an image source when it depends on other packages or have a source image
 // explictly supplied
 func (cs *LuetCompilationSpec) HasImageSource() bool {
-	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != ""
-}
-
-// Signature is a portion of the spec that yields a signature for the hash
-type Signature struct {
-	Image      string
-	Steps      []string
-	PackageDir string
-	Prelude    []string
-	Seed       string
-	Env        []string
-	Retrieve   []string
-	Unpack     bool
-	Includes   []string
-	Excludes   []string
-	Copy       []CopyField
-}
-
-func (cs *LuetCompilationSpec) signature() Signature {
-	return Signature{
-		Image:      cs.Image,
-		Steps:      cs.Steps,
-		PackageDir: cs.PackageDir,
-		Prelude:    cs.Prelude,
-		Seed:       cs.Seed,
-		Env:        cs.Env,
-		Retrieve:   cs.Retrieve,
-		Unpack:     cs.Unpack,
-		Includes:   cs.Includes,
-		Excludes:   cs.Excludes,
-		Copy:       cs.Copy,
-	}
+	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != "" || len(cs.Join) != 0
 }
 
 func (cs *LuetCompilationSpec) Hash() (string, error) {

pkg/package/package.go

@@ -133,6 +133,19 @@ type Tree interface {
 
 type Packages []Package
 
+type DefaultPackages []*DefaultPackage
+
+func (d DefaultPackages) Hash(salt string) string {
+
+	overallFp := ""
+	for _, c := range d {
+		overallFp = overallFp + c.HashFingerprint("join")
+	}
+	h := md5.New()
+	io.WriteString(h, fmt.Sprintf("%s-%s", overallFp, salt))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
 // >> Unmarshallers
 // DefaultPackageFromYaml decodes a package from yaml bytes
 func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) {

tests/fixtures/join/c/a/build.yaml

@@ -0,0 +1,8 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"
+unpack: true

tests/fixtures/join/c/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.2"

tests/fixtures/join/cat/a/a/build.yaml

@@ -0,0 +1,7 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/join/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/join/cat/b-1.1/build.yaml

@@ -0,0 +1,13 @@
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0"
+
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/join/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/join/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/integration/28_nobuildtreedocker.sh

@@ -51,9 +51,9 @@ testBuild() {
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available remotely for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:7dd6062f45e78c1fe36d0f48fc21bc8c8219edfc9759f117527677a15ae42717"
-    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available remotely for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2338dc4dc4b3d9657eb26597ad79569fe60f5529dd05676df266ad982ba2b1ba"
-    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2edc4b6f8fc08a0fc958132dfcf2813e1ab220c2bbf60b988f1039082d61ff3e"
+    assertContains 'Does use the upstream cache without specifying it test/c' "$build_output" "Images available remotely for test/c-1.0 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:ac34beb3fc2752bab54549db095dd6994d7531b88e1ff7f902d01ae80fdd030d"
+    assertContains 'Does use the upstream cache without specifying it test/z' "$build_output" "Images available remotely for test/z-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:d8b8881d50ae83646728bb28ad678fc14e4e003e4b0d0a66f8e6167c6116e024"
+    assertContains 'Does use the upstream cache without specifying it test/interpolated' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2aaccd929ebe5683f95c70c6ec8d68f240ceea8633f3350b94904ad73da5fd47"
 }
 
 testRepo() {

tests/integration/29_nobuildtreeinheritvalues.sh

@@ -56,8 +56,8 @@ EOF
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertNotContains 'Does NOT use the upstream cache without specifying it' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2edc4b6f8fc08a0fc958132dfcf2813e1ab220c2bbf60b988f1039082d61ff3e"
-    assertContains 'Does generate a new hash as values changed build.yaml' "$build_output" "Building image luet/cache:a249d16764168baf32b592dc05c33e499fa3685edda72dfb188a51191709de5a done"
+    assertNotContains 'Does NOT use the upstream cache without specifying it' "$build_output" "Images available remotely for test/interpolated-1.0+2 generating artifact from remote images: quay.io/mocaccinoos/integration-test-cache:2aaccd929ebe5683f95c70c6ec8d68f240ceea8633f3350b94904ad73da5fd47"
+    assertContains 'Does generate a new hash as values changed build.yaml for test/interpolated-1.0+2 package image' "$build_output" "Building image luet/cache:c34b533cf76886c332fe9b2d3208f04265360a465a90c996cb4fcdaf959dee36 done"
 }
 
 testRepo() {

tests/integration/29_nobuildtreeinheritvalues_noignored.sh

@@ -56,7 +56,7 @@ EOF
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
     assertTrue 'create package Z' "[ -e '$tmpdir/testbuild/z-test-1.0+2.package.tar.zst' ]"
     assertTrue 'create package interpolated' "[ -e '$tmpdir/testbuild/interpolated-test-1.0+2.package.tar.zst' ]"
-    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Downloading image quay.io/mocaccinoos/integration-test-cache:ec62e3e2cfb4c520c8b2561797c005d248c2659295f3660fa1a66582fc4dc280"
+    assertContains 'Does use the upstream cache without specifying it' "$build_output" "Downloading image quay.io/mocaccinoos/integration-test-cache:4db24406e8db30a3310a1cf8c4d4e19597745e6d41b189dc51a73ac4a50cc9e6"
 }
 
 testRepo() {

tests/integration/31_join.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    cat <<EOF > $tmpdir/default.yaml
+extra: "bar"
+foo: "baz"
+EOF
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/join" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --compression gzip --values $tmpdir/default.yaml \
+               test/c
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package c' "[ -e '$tmpdir/testbuild/c-test-1.2.package.tar.gz' ]"
+    mkdir $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/c-test-1.2.package.tar.gz -C $tmpdir/extract
+    assertTrue 'create result from join' "[ -e '$tmpdir/extract/test3' ]"
+    assertTrue 'create result from join' "[ -f '$tmpdir/extract/newc' ]"
+    assertTrue 'create result from join' "[ -e '$tmpdir/extract/test4' ]"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+