Tag 0.16.3

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.16.0"
+	LuetCLIVersion = "0.16.3"
 	LuetEnvPrefix  = "LUET"
 )
 

pkg/compiler/compiler.go

@@ -774,8 +774,10 @@ func (cs *LuetCompiler) getSpecHash(pkgs pkg.DefaultPackages, salt string) (stri
 }
 
 func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+
+	joinTag := ">:loop: join<"
 	if len(p.Join) != 0 {
-		Info("Generating a joint parent image from final packages")
+		Info(joinTag, "Generating a joint parent image from final packages")
 	} else {
 		return nil
 	}
@@ -785,29 +787,47 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 	if err != nil {
 		return errors.Wrap(err, "could not generate image hash")
 	}
+
+	Info(joinTag, "Searching existing image with hash ", overallFp)
+
 	image := cs.findImageHash(overallFp, p)
 	if image != "" {
 		Info("Image already found", image)
 		p.SetImage(image)
 		return nil
 	}
-	Info("Generating image with hash ", image)
+	Info(joinTag, "Image not found. Generating image join with hash ", overallFp)
+
+	// Make sure there is an output path
+	if err := os.MkdirAll(p.GetOutputPath(), os.ModePerm); err != nil {
+		return errors.Wrap(err, "while creating output path")
+	}
 
 	// otherwise, generate it and push it aside
 	joinDir, err := ioutil.TempDir(p.GetOutputPath(), "join")
 	if err != nil {
-		return errors.Wrap(err, "Could not create tempdir")
+		return errors.Wrap(err, "could not create tempdir for joining images")
 	}
 	defer os.RemoveAll(joinDir) // clean up
 
+	for _, p := range p.Join { //highly dependent on the order
+		Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
+	}
+
+	current := 0
 	for _, c := range p.Join {
+		current++
 		if c != nil && c.Name != "" && c.Version != "" {
-			Info(" :droplet: generating", c.HumanReadableString())
+			joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(p.Join), c.HumanReadableString())
+
+			Info(joinTag2, "compilation starts")
 			spec, err := cs.FromPackage(c)
 			if err != nil {
 				return errors.Wrap(err, "while generating images to join from")
 			}
 			wantsArtifact := true
+			spec.SetOutputPath(p.GetOutputPath())
+
 			artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, spec)
 			if err != nil {
 				return errors.Wrap(err, "failed building join image")
@@ -817,52 +837,67 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 			if err != nil {
 				return errors.Wrap(err, "failed building join image")
 			}
+			Info(joinTag2, ":white_check_mark: Done")
 		}
 	}
 
 	artifactDir, err := ioutil.TempDir(p.GetOutputPath(), "artifact")
 	if err != nil {
-		return errors.Wrap(err, "Could not create tempdir")
+		return errors.Wrap(err, "could not create tempdir for final artifact")
 	}
 	defer os.RemoveAll(joinDir) // clean up
 
+	Info(joinTag, ":droplet: generating artifact for source image of", p.GetPackage().HumanReadableString())
+
 	// After unpack, create a new artifact and a new final image from it.
 	// no need to compress, as we are going to toss it away.
 	a := artifact.NewPackageArtifact(filepath.Join(artifactDir, p.GetPackage().GetFingerPrint()+".join.tar"))
 	if err := a.Compress(joinDir, concurrency); err != nil {
-		return errors.Wrap(err, "Error met while creating package archive")
+		return errors.Wrap(err, "error met while creating package archive")
 	}
 
 	joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp)
+	Info(joinTag, ":droplet: generating image from artifact", joinImageName)
 	opts, err := a.GenerateFinalImage(joinImageName, cs.Backend, keepPermissions)
 	if err != nil {
-		return errors.Wrap(err, "Could not create tempdir")
+		return errors.Wrap(err, "could not create final image")
 	}
 	if cs.Options.Push {
+		Info(joinTag, ":droplet: pushing image from artifact", joinImageName)
 		if err = cs.Backend.Push(opts); err != nil {
 			return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
 		}
 	}
-	Info("Using image ", joinImageName)
+	Info(joinTag, ":droplet: Consuming image", joinImageName)
 	p.SetImage(joinImageName)
 	return nil
 }
 
 func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
 	resolvedCopyFields := []compilerspec.CopyField{}
+	copyTag := ">:droplet: copy<"
+
 	if len(p.Copy) != 0 {
-		Info("Package has multi-stage copy, generating required images")
+		Info(copyTag, "Package has multi-stage copy, generating required images")
 	}
+
+	current := 0
+	// TODO: we should run this only if we are going to build the image
 	for _, c := range p.Copy {
+		current++
 		if c.Package != nil && c.Package.Name != "" && c.Package.Version != "" {
-			Info(" :droplet: generating multi-stage images for", c.Package.HumanReadableString())
+			copyTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", copyTag, current, len(p.Copy), c.Package.HumanReadableString())
+
+			Info(copyTag2, "generating multi-stage images for", c.Package.HumanReadableString())
 			spec, err := cs.FromPackage(c.Package)
 			if err != nil {
 				return errors.Wrap(err, "while generating images to copy from")
 			}
-			noArtifact := false
-			artifact, err := cs.compile(concurrency, keepPermissions, &noArtifact, spec)
 
+			// If we specify --only-target package, we don't want any artifact, otherwise we do
+			genArtifact := !cs.Options.PackageTargetOnly
+			spec.SetOutputPath(p.GetOutputPath())
+			artifact, err := cs.compile(concurrency, keepPermissions, &genArtifact, spec)
 			if err != nil {
 				return errors.Wrap(err, "failed building multi-stage image")
 			}
@@ -872,6 +907,7 @@ func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions
 				Source:      c.Source,
 				Destination: c.Destination,
 			})
+			Info(copyTag2, ":white_check_mark: Done")
 		} else {
 			resolvedCopyFields = append(resolvedCopyFields, c)
 		}
@@ -933,11 +969,13 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateA
 		if generateArtifact != nil {
 			localGenerateArtifact = *generateArtifact
 		}
+
 		a, err := cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
 		if err != nil {
 			return nil, errors.Wrap(err, "building direct image")
 		}
 		a.SourceAssertion = p.GetSourceAssertion()
+
 		a.PackageCacheImage = targetAssertion.Hash.PackageHash
 		return a, nil
 	}
@@ -987,6 +1025,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateA
 				Assert:      assertion,
 			})
 
+			if err := cs.resolveJoinImages(concurrency, keepPermissions, compileSpec); err != nil {
+				return nil, errors.Wrap(err, "while resolving join images")
+			}
+
+			if err := cs.resolveMultiStageImages(concurrency, keepPermissions, compileSpec); err != nil {
+				return nil, errors.Wrap(err, "while resolving multi-stage images")
+			}
+
 			buildHash, err := packageHashTree.DependencyBuildImage(assertion.Package)
 			if err != nil {
 				return nil, errors.Wrap(err, "failed looking for dependency in hashtree")

tests/fixtures/join_complex/c/c1/build.yaml

@@ -0,0 +1,7 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"

tests/fixtures/join_complex/c/c1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.2"

tests/fixtures/join_complex/c/f/build.yaml

@@ -0,0 +1,8 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"
+unpack: true

tests/fixtures/join_complex/c/f/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "f"
+version: "1.2"

tests/fixtures/join_complex/cat/a/a/build.yaml

@@ -0,0 +1,2 @@
+image: "alpine"
+unpack: true

tests/fixtures/join_complex/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/join_complex/cat/b-1.1/build.yaml

@@ -0,0 +1,13 @@
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0"
+
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/join_complex/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/join_complex/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/fixtures/join_complex/x/build.yaml

@@ -0,0 +1,10 @@
+join:
+- category: "test"
+  name: "f"
+  version: ">=0"
+# this is actually a virtual, no files shipped in c
+- category: "test"
+  name: "c"
+  version: ">=0"
+steps:
+- mv /newnewc /x

tests/fixtures/join_complex/x/definition.yaml

@@ -0,0 +1,3 @@
+name: "x"
+category: "test"
+version: "0.1"

tests/fixtures/join_complex/z/build.yaml

@@ -0,0 +1,6 @@
+requires:
+- category: "test"
+  name: "c"
+  version: ">=0"
+steps:
+- mv /newnewc /z

tests/fixtures/join_complex/z/definition.yaml

@@ -0,0 +1,3 @@
+name: "z"
+category: "test"
+version: "0.1"

tests/integration/31_join_complex.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/join_complex" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --compression gzip \
+               test/z test/x
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package z' "[ -e '$tmpdir/testbuild/z-test-0.1.package.tar.gz' ]"
+    assertTrue 'create package z' "[ -e '$tmpdir/testbuild/x-test-0.1.package.tar.gz' ]"
+
+    mkdir $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/x-test-0.1.package.tar.gz -C $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/z-test-0.1.package.tar.gz -C $tmpdir/extract
+    assertTrue 'create result from a package that requires a join' "[ -e '$tmpdir/extract/z' ]"
+    assertTrue 'create result from join of a join' "[ -e '$tmpdir/extract/x' ]"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+