Tag 0.16.4

Distinguish rather if we should generate artifact of the deps of a
package or not, and ensure that we just force artifact generation for
targets when computing the join image.

Fixes #217

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.16.0"
+	LuetCLIVersion = "0.16.4"
 	LuetEnvPrefix  = "LUET"
 )
 

pkg/compiler/backend/simpledocker.go

@@ -181,6 +181,12 @@ func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
 	name := opts.ImageName
 	dst := opts.Destination
 
+	if !b.ImageExists(name) {
+		if err := b.DownloadImage(opts); err != nil {
+			return errors.Wrap(err, "failed pulling image "+name+" during extraction")
+		}
+	}
+
 	tempexport, err := ioutil.TempDir(dst, "tmprootfs")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for rootfs")

pkg/compiler/compiler.go

@@ -91,7 +91,7 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *c
 	defer wg.Done()
 
 	for s := range cspecs {
-		ar, err := cs.compile(concurrency, keepPermissions, nil, s)
+		ar, err := cs.compile(concurrency, keepPermissions, nil, nil, s)
 		if err != nil {
 			errors <- err
 		}
@@ -723,7 +723,7 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompi
 // Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
 // an artifact
 func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	return cs.compile(cs.Options.Concurrency, keepPermissions, nil, p)
+	return cs.compile(cs.Options.Concurrency, keepPermissions, nil, nil, p)
 }
 
 func genImageList(refs []string, hash string) []string {
@@ -774,8 +774,10 @@ func (cs *LuetCompiler) getSpecHash(pkgs pkg.DefaultPackages, salt string) (stri
 }
 
 func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+
+	joinTag := ">:loop: join<"
 	if len(p.Join) != 0 {
-		Info("Generating a joint parent image from final packages")
+		Info(joinTag, "Generating a joint parent image from final packages")
 	} else {
 		return nil
 	}
@@ -785,30 +787,50 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 	if err != nil {
 		return errors.Wrap(err, "could not generate image hash")
 	}
+
+	Info(joinTag, "Searching existing image with hash ", overallFp)
+
 	image := cs.findImageHash(overallFp, p)
 	if image != "" {
 		Info("Image already found", image)
 		p.SetImage(image)
 		return nil
 	}
-	Info("Generating image with hash ", image)
+	Info(joinTag, "Image not found. Generating image join with hash ", overallFp)
+
+	// Make sure there is an output path
+	if err := os.MkdirAll(p.GetOutputPath(), os.ModePerm); err != nil {
+		return errors.Wrap(err, "while creating output path")
+	}
 
 	// otherwise, generate it and push it aside
 	joinDir, err := ioutil.TempDir(p.GetOutputPath(), "join")
 	if err != nil {
-		return errors.Wrap(err, "Could not create tempdir")
+		return errors.Wrap(err, "could not create tempdir for joining images")
 	}
 	defer os.RemoveAll(joinDir) // clean up
 
+	for _, p := range p.Join { //highly dependent on the order
+		Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
+	}
+
+	current := 0
 	for _, c := range p.Join {
+		current++
 		if c != nil && c.Name != "" && c.Version != "" {
-			Info(" :droplet: generating", c.HumanReadableString())
+			joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(p.Join), c.HumanReadableString())
+
+			Info(joinTag2, "compilation starts")
 			spec, err := cs.FromPackage(c)
 			if err != nil {
 				return errors.Wrap(err, "while generating images to join from")
 			}
 			wantsArtifact := true
-			artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, spec)
+			genDepsArtifact := !cs.Options.PackageTargetOnly
+
+			spec.SetOutputPath(p.GetOutputPath())
+
+			artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, &genDepsArtifact, spec)
 			if err != nil {
 				return errors.Wrap(err, "failed building join image")
 			}
@@ -817,52 +839,67 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 			if err != nil {
 				return errors.Wrap(err, "failed building join image")
 			}
+			Info(joinTag2, ":white_check_mark: Done")
 		}
 	}
 
 	artifactDir, err := ioutil.TempDir(p.GetOutputPath(), "artifact")
 	if err != nil {
-		return errors.Wrap(err, "Could not create tempdir")
+		return errors.Wrap(err, "could not create tempdir for final artifact")
 	}
 	defer os.RemoveAll(joinDir) // clean up
 
+	Info(joinTag, ":droplet: generating artifact for source image of", p.GetPackage().HumanReadableString())
+
 	// After unpack, create a new artifact and a new final image from it.
 	// no need to compress, as we are going to toss it away.
 	a := artifact.NewPackageArtifact(filepath.Join(artifactDir, p.GetPackage().GetFingerPrint()+".join.tar"))
 	if err := a.Compress(joinDir, concurrency); err != nil {
-		return errors.Wrap(err, "Error met while creating package archive")
+		return errors.Wrap(err, "error met while creating package archive")
 	}
 
 	joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp)
+	Info(joinTag, ":droplet: generating image from artifact", joinImageName)
 	opts, err := a.GenerateFinalImage(joinImageName, cs.Backend, keepPermissions)
 	if err != nil {
-		return errors.Wrap(err, "Could not create tempdir")
+		return errors.Wrap(err, "could not create final image")
 	}
 	if cs.Options.Push {
+		Info(joinTag, ":droplet: pushing image from artifact", joinImageName)
 		if err = cs.Backend.Push(opts); err != nil {
 			return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
 		}
 	}
-	Info("Using image ", joinImageName)
+	Info(joinTag, ":droplet: Consuming image", joinImageName)
 	p.SetImage(joinImageName)
 	return nil
 }
 
 func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
 	resolvedCopyFields := []compilerspec.CopyField{}
+	copyTag := ">:droplet: copy<"
+
 	if len(p.Copy) != 0 {
-		Info("Package has multi-stage copy, generating required images")
+		Info(copyTag, "Package has multi-stage copy, generating required images")
 	}
+
+	current := 0
+	// TODO: we should run this only if we are going to build the image
 	for _, c := range p.Copy {
+		current++
 		if c.Package != nil && c.Package.Name != "" && c.Package.Version != "" {
-			Info(" :droplet: generating multi-stage images for", c.Package.HumanReadableString())
+			copyTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", copyTag, current, len(p.Copy), c.Package.HumanReadableString())
+
+			Info(copyTag2, "generating multi-stage images for", c.Package.HumanReadableString())
 			spec, err := cs.FromPackage(c.Package)
 			if err != nil {
 				return errors.Wrap(err, "while generating images to copy from")
 			}
-			noArtifact := false
-			artifact, err := cs.compile(concurrency, keepPermissions, &noArtifact, spec)
 
+			// If we specify --only-target package, we don't want any artifact, otherwise we do
+			genArtifact := !cs.Options.PackageTargetOnly
+			spec.SetOutputPath(p.GetOutputPath())
+			artifact, err := cs.compile(concurrency, keepPermissions, &genArtifact, &genArtifact, spec)
 			if err != nil {
 				return errors.Wrap(err, "failed building multi-stage image")
 			}
@@ -872,6 +909,7 @@ func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions
 				Source:      c.Source,
 				Destination: c.Destination,
 			})
+			Info(copyTag2, ":white_check_mark: Done")
 		} else {
 			resolvedCopyFields = append(resolvedCopyFields, c)
 		}
@@ -880,7 +918,7 @@ func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions
 	return nil
 }
 
-func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
+func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateFinalArtifact *bool, generateDependenciesFinalArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
 	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
 	//Before multistage : join - same as multistage, but keep artifacts, join them, create a new one and generate a final image.
@@ -930,14 +968,16 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateA
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
 		localGenerateArtifact := true
-		if generateArtifact != nil {
-			localGenerateArtifact = *generateArtifact
+		if generateFinalArtifact != nil {
+			localGenerateArtifact = *generateFinalArtifact
 		}
+
 		a, err := cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
 		if err != nil {
 			return nil, errors.Wrap(err, "building direct image")
 		}
 		a.SourceAssertion = p.GetSourceAssertion()
+
 		a.PackageCacheImage = targetAssertion.Hash.PackageHash
 		return a, nil
 	}
@@ -952,8 +992,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateA
 	currentN := 0
 
 	packageDeps := !cs.Options.PackageTargetOnly
-	if generateArtifact != nil {
-		packageDeps = *generateArtifact
+	if generateDependenciesFinalArtifact != nil {
+		packageDeps = *generateDependenciesFinalArtifact
 	}
 
 	buildDeps := !cs.Options.NoDeps
@@ -987,6 +1027,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateA
 				Assert:      assertion,
 			})
 
+			if err := cs.resolveJoinImages(concurrency, keepPermissions, compileSpec); err != nil {
+				return nil, errors.Wrap(err, "while resolving join images")
+			}
+
+			if err := cs.resolveMultiStageImages(concurrency, keepPermissions, compileSpec); err != nil {
+				return nil, errors.Wrap(err, "while resolving multi-stage images")
+			}
+
 			buildHash, err := packageHashTree.DependencyBuildImage(assertion.Package)
 			if err != nil {
 				return nil, errors.Wrap(err, "failed looking for dependency in hashtree")
@@ -1040,8 +1088,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateA
 
 	if buildTarget {
 		localGenerateArtifact := true
-		if generateArtifact != nil {
-			localGenerateArtifact = *generateArtifact
+		if generateFinalArtifact != nil {
+			localGenerateArtifact = *generateFinalArtifact
 		}
 		resolvedSourceImage := cs.resolveExistingImageHash(packageHashTree.SourceHash, p)
 		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())

pkg/installer/repository_docker.go

@@ -61,7 +61,7 @@ func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDataba
 			return nil
 		}
 
-		if err := l.pushImageFromArtifact(artifact.NewPackageArtifact(currentpath), l.b); err != nil {
+		if err := l.pushImageFromArtifact(artifact.NewPackageArtifact(currentpath), l.b, true); err != nil {
 			return errors.Wrap(err, "while pushing metadata file associated to the artifact")
 		}
 
@@ -159,7 +159,7 @@ func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec string, r *LuetSys
 	return nil
 }
 
-func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, b compiler.CompilerBackend) error {
+func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, b compiler.CompilerBackend, checkIfExists bool) error {
 	// we generate a new archive containing the required compressed file.
 	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
 	treeArchive, err := artifact.CreateArtifactForFile(a.Path)
@@ -167,8 +167,12 @@ func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArt
 		return errors.Wrap(err, "failed generating checksums for tree")
 	}
 	imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, helpers.StripInvalidStringsFromImage(a.GetFileName()))
-
-	return d.pushFileFromArtifact(treeArchive, imageTree)
+	if checkIfExists && d.imagePush && d.b.ImageAvailable(imageTree) && !d.force {
+		Info("Image", imageTree, "already present, skipping. use --force-push to override")
+		return nil
+	} else {
+		return d.pushFileFromArtifact(treeArchive, imageTree)
+	}
 }
 
 // Generate creates a Docker luet repository
@@ -225,7 +229,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 
 	// we generate a new archive containing the required compressed file.
 	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
-	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+	if err := d.pushImageFromArtifact(a, d.b, false); err != nil {
 		return errors.Wrap(err, "error met while pushing runtime tree")
 	}
 
@@ -235,7 +239,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 	}
 	// we generate a new archive containing the required compressed file.
 	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
-	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+	if err := d.pushImageFromArtifact(a, d.b, false); err != nil {
 		return errors.Wrap(err, "error met while pushing compiler tree")
 	}
 
@@ -251,7 +255,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 		return errors.Wrap(err, "failed adding Metadata file to repository")
 	}
 
-	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+	if err := d.pushImageFromArtifact(a, d.b, false); err != nil {
 		return errors.Wrap(err, "error met while pushing docker image from artifact")
 	}
 

tests/fixtures/join_complex/c/c1/build.yaml

@@ -0,0 +1,7 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"

tests/fixtures/join_complex/c/c1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.2"

tests/fixtures/join_complex/c/f/build.yaml

@@ -0,0 +1,8 @@
+join:
+  - name: "a"
+    category: "test"
+    version: ">=0"
+  - name: "b"
+    category: "test"
+    version: ">=0"
+unpack: true

tests/fixtures/join_complex/c/f/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "f"
+version: "1.2"

tests/fixtures/join_complex/cat/a/a/build.yaml

@@ -0,0 +1,2 @@
+image: "alpine"
+unpack: true

tests/fixtures/join_complex/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/join_complex/cat/b-1.1/build.yaml

@@ -0,0 +1,13 @@
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0"
+
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/join_complex/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/join_complex/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/fixtures/join_complex/x/build.yaml

@@ -0,0 +1,10 @@
+join:
+- category: "test"
+  name: "f"
+  version: ">=0"
+# this is actually a virtual, no files shipped in c
+- category: "test"
+  name: "c"
+  version: ">=0"
+steps:
+- mv /newnewc /x

tests/fixtures/join_complex/x/definition.yaml

@@ -0,0 +1,3 @@
+name: "x"
+category: "test"
+version: "0.1"

tests/fixtures/join_complex/z/build.yaml

@@ -0,0 +1,6 @@
+requires:
+- category: "test"
+  name: "c"
+  version: ">=0"
+steps:
+- mv /newnewc /z

tests/fixtures/join_complex/z/definition.yaml

@@ -0,0 +1,3 @@
+name: "z"
+category: "test"
+version: "0.1"

tests/integration/31_join_complex.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/join_complex" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --compression gzip \
+               test/z test/x
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package z' "[ -e '$tmpdir/testbuild/z-test-0.1.package.tar.gz' ]"
+    assertTrue 'create package z' "[ -e '$tmpdir/testbuild/x-test-0.1.package.tar.gz' ]"
+
+    mkdir $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/x-test-0.1.package.tar.gz -C $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/z-test-0.1.package.tar.gz -C $tmpdir/extract
+    assertTrue 'create result from a package that requires a join' "[ -e '$tmpdir/extract/z' ]"
+    assertTrue 'create result from join of a join' "[ -e '$tmpdir/extract/x' ]"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+