Tag 0.16.4

Distinguish rather if we should generate artifact of the deps of a
package or not, and ensure that we just force artifact generation for
targets when computing the join image.

Fixes #217

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.16.2"
+	LuetCLIVersion = "0.16.4"
 	LuetEnvPrefix  = "LUET"
 )
 

pkg/compiler/backend/simpledocker.go

@@ -181,6 +181,12 @@ func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
 	name := opts.ImageName
 	dst := opts.Destination
 
+	if !b.ImageExists(name) {
+		if err := b.DownloadImage(opts); err != nil {
+			return errors.Wrap(err, "failed pulling image "+name+" during extraction")
+		}
+	}
+
 	tempexport, err := ioutil.TempDir(dst, "tmprootfs")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for rootfs")

pkg/compiler/compiler.go

@@ -91,7 +91,7 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *c
 	defer wg.Done()
 
 	for s := range cspecs {
-		ar, err := cs.compile(concurrency, keepPermissions, nil, s)
+		ar, err := cs.compile(concurrency, keepPermissions, nil, nil, s)
 		if err != nil {
 			errors <- err
 		}
@@ -723,7 +723,7 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompi
 // Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
 // an artifact
 func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	return cs.compile(cs.Options.Concurrency, keepPermissions, nil, p)
+	return cs.compile(cs.Options.Concurrency, keepPermissions, nil, nil, p)
 }
 
 func genImageList(refs []string, hash string) []string {
@@ -774,8 +774,10 @@ func (cs *LuetCompiler) getSpecHash(pkgs pkg.DefaultPackages, salt string) (stri
 }
 
 func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+
+	joinTag := ">:loop: join<"
 	if len(p.Join) != 0 {
-		Info("Generating a joint parent image from final packages")
+		Info(joinTag, "Generating a joint parent image from final packages")
 	} else {
 		return nil
 	}
@@ -785,13 +787,16 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 	if err != nil {
 		return errors.Wrap(err, "could not generate image hash")
 	}
+
+	Info(joinTag, "Searching existing image with hash ", overallFp)
+
 	image := cs.findImageHash(overallFp, p)
 	if image != "" {
 		Info("Image already found", image)
 		p.SetImage(image)
 		return nil
 	}
-	Info("Generating image with hash ", image)
+	Info(joinTag, "Image not found. Generating image join with hash ", overallFp)
 
 	// Make sure there is an output path
 	if err := os.MkdirAll(p.GetOutputPath(), os.ModePerm); err != nil {
@@ -805,15 +810,27 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 	}
 	defer os.RemoveAll(joinDir) // clean up
 
+	for _, p := range p.Join { //highly dependent on the order
+		Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
+	}
+
+	current := 0
 	for _, c := range p.Join {
+		current++
 		if c != nil && c.Name != "" && c.Version != "" {
-			Info(" :droplet: generating", c.HumanReadableString())
+			joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(p.Join), c.HumanReadableString())
+
+			Info(joinTag2, "compilation starts")
 			spec, err := cs.FromPackage(c)
 			if err != nil {
 				return errors.Wrap(err, "while generating images to join from")
 			}
 			wantsArtifact := true
-			artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, spec)
+			genDepsArtifact := !cs.Options.PackageTargetOnly
+
+			spec.SetOutputPath(p.GetOutputPath())
+
+			artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, &genDepsArtifact, spec)
 			if err != nil {
 				return errors.Wrap(err, "failed building join image")
 			}
@@ -822,6 +839,7 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 			if err != nil {
 				return errors.Wrap(err, "failed building join image")
 			}
+			Info(joinTag2, ":white_check_mark: Done")
 		}
 	}
 
@@ -831,6 +849,8 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 	}
 	defer os.RemoveAll(joinDir) // clean up
 
+	Info(joinTag, ":droplet: generating artifact for source image of", p.GetPackage().HumanReadableString())
+
 	// After unpack, create a new artifact and a new final image from it.
 	// no need to compress, as we are going to toss it away.
 	a := artifact.NewPackageArtifact(filepath.Join(artifactDir, p.GetPackage().GetFingerPrint()+".join.tar"))
@@ -839,35 +859,47 @@ func (cs *LuetCompiler) resolveJoinImages(concurrency int, keepPermissions bool,
 	}
 
 	joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp)
+	Info(joinTag, ":droplet: generating image from artifact", joinImageName)
 	opts, err := a.GenerateFinalImage(joinImageName, cs.Backend, keepPermissions)
 	if err != nil {
 		return errors.Wrap(err, "could not create final image")
 	}
 	if cs.Options.Push {
+		Info(joinTag, ":droplet: pushing image from artifact", joinImageName)
 		if err = cs.Backend.Push(opts); err != nil {
 			return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
 		}
 	}
-	Info("Using image ", joinImageName)
+	Info(joinTag, ":droplet: Consuming image", joinImageName)
 	p.SetImage(joinImageName)
 	return nil
 }
 
 func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
 	resolvedCopyFields := []compilerspec.CopyField{}
+	copyTag := ">:droplet: copy<"
+
 	if len(p.Copy) != 0 {
-		Info("Package has multi-stage copy, generating required images")
+		Info(copyTag, "Package has multi-stage copy, generating required images")
 	}
+
+	current := 0
+	// TODO: we should run this only if we are going to build the image
 	for _, c := range p.Copy {
+		current++
 		if c.Package != nil && c.Package.Name != "" && c.Package.Version != "" {
-			Info(" :droplet: generating multi-stage images for", c.Package.HumanReadableString())
+			copyTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", copyTag, current, len(p.Copy), c.Package.HumanReadableString())
+
+			Info(copyTag2, "generating multi-stage images for", c.Package.HumanReadableString())
 			spec, err := cs.FromPackage(c.Package)
 			if err != nil {
 				return errors.Wrap(err, "while generating images to copy from")
 			}
-			noArtifact := false
-			artifact, err := cs.compile(concurrency, keepPermissions, &noArtifact, spec)
 
+			// If we specify --only-target package, we don't want any artifact, otherwise we do
+			genArtifact := !cs.Options.PackageTargetOnly
+			spec.SetOutputPath(p.GetOutputPath())
+			artifact, err := cs.compile(concurrency, keepPermissions, &genArtifact, &genArtifact, spec)
 			if err != nil {
 				return errors.Wrap(err, "failed building multi-stage image")
 			}
@@ -877,6 +909,7 @@ func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions
 				Source:      c.Source,
 				Destination: c.Destination,
 			})
+			Info(copyTag2, ":white_check_mark: Done")
 		} else {
 			resolvedCopyFields = append(resolvedCopyFields, c)
 		}
@@ -885,7 +918,7 @@ func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions
 	return nil
 }
 
-func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
+func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateFinalArtifact *bool, generateDependenciesFinalArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
 	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
 	//Before multistage : join - same as multistage, but keep artifacts, join them, create a new one and generate a final image.
@@ -935,14 +968,16 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateA
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
 		localGenerateArtifact := true
-		if generateArtifact != nil {
-			localGenerateArtifact = *generateArtifact
+		if generateFinalArtifact != nil {
+			localGenerateArtifact = *generateFinalArtifact
 		}
+
 		a, err := cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
 		if err != nil {
 			return nil, errors.Wrap(err, "building direct image")
 		}
 		a.SourceAssertion = p.GetSourceAssertion()
+
 		a.PackageCacheImage = targetAssertion.Hash.PackageHash
 		return a, nil
 	}
@@ -957,8 +992,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateA
 	currentN := 0
 
 	packageDeps := !cs.Options.PackageTargetOnly
-	if generateArtifact != nil {
-		packageDeps = *generateArtifact
+	if generateDependenciesFinalArtifact != nil {
+		packageDeps = *generateDependenciesFinalArtifact
 	}
 
 	buildDeps := !cs.Options.NoDeps
@@ -1053,8 +1088,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateA
 
 	if buildTarget {
 		localGenerateArtifact := true
-		if generateArtifact != nil {
-			localGenerateArtifact = *generateArtifact
+		if generateFinalArtifact != nil {
+			localGenerateArtifact = *generateFinalArtifact
 		}
 		resolvedSourceImage := cs.resolveExistingImageHash(packageHashTree.SourceHash, p)
 		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())

pkg/installer/repository_docker.go

@@ -61,7 +61,7 @@ func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDataba
 			return nil
 		}
 
-		if err := l.pushImageFromArtifact(artifact.NewPackageArtifact(currentpath), l.b); err != nil {
+		if err := l.pushImageFromArtifact(artifact.NewPackageArtifact(currentpath), l.b, true); err != nil {
 			return errors.Wrap(err, "while pushing metadata file associated to the artifact")
 		}
 
@@ -159,7 +159,7 @@ func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec string, r *LuetSys
 	return nil
 }
 
-func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, b compiler.CompilerBackend) error {
+func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, b compiler.CompilerBackend, checkIfExists bool) error {
 	// we generate a new archive containing the required compressed file.
 	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
 	treeArchive, err := artifact.CreateArtifactForFile(a.Path)
@@ -167,8 +167,12 @@ func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArt
 		return errors.Wrap(err, "failed generating checksums for tree")
 	}
 	imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, helpers.StripInvalidStringsFromImage(a.GetFileName()))
-
-	return d.pushFileFromArtifact(treeArchive, imageTree)
+	if checkIfExists && d.imagePush && d.b.ImageAvailable(imageTree) && !d.force {
+		Info("Image", imageTree, "already present, skipping. use --force-push to override")
+		return nil
+	} else {
+		return d.pushFileFromArtifact(treeArchive, imageTree)
+	}
 }
 
 // Generate creates a Docker luet repository
@@ -225,7 +229,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 
 	// we generate a new archive containing the required compressed file.
 	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
-	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+	if err := d.pushImageFromArtifact(a, d.b, false); err != nil {
 		return errors.Wrap(err, "error met while pushing runtime tree")
 	}
 
@@ -235,7 +239,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 	}
 	// we generate a new archive containing the required compressed file.
 	// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
-	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+	if err := d.pushImageFromArtifact(a, d.b, false); err != nil {
 		return errors.Wrap(err, "error met while pushing compiler tree")
 	}
 
@@ -251,7 +255,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 		return errors.Wrap(err, "failed adding Metadata file to repository")
 	}
 
-	if err := d.pushImageFromArtifact(a, d.b); err != nil {
+	if err := d.pushImageFromArtifact(a, d.b, false); err != nil {
 		return errors.Wrap(err, "error met while pushing docker image from artifact")
 	}