Add KeepImg and PullFirst options

They allow to keep a cache,define an endpoint and pull it first.

Also slim logic in favor of more freedom on backend side

pkg/compiler/compiler.go

@@ -37,6 +37,8 @@ type LuetCompiler struct {
 	*tree.CompilerRecipe
 	Backend            CompilerBackend
 	Database           pkg.PackageDatabase
+	ImageRepository    string
+	PullFirst, KeepImg bool
 }
 
 func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase) Compiler {
@@ -47,6 +49,9 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase) Compiler {
 			tree.Recipe{Database: db},
 		},
 		Database:        db,
+		ImageRepository: "luet/cache",
+		PullFirst:       true,
+		KeepImg:         true,
 	}
 }
 
@@ -153,13 +158,12 @@ func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, p
 	return artifacts, allErrors
 }
 
-func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
+func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec) (Artifact, error) {
 	pkgTag := ":package:  " + p.GetPackage().GetName()
 
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
 	// and we build all the images first.
-	keepImg := true
+
-	keepPackageImg := true
 	err := os.MkdirAll(p.Rel("build"), os.ModePerm)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for building")
@@ -177,12 +181,16 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 
 	}
 	if buildertaggedImage == "" {
-		keepImg = false
+		buildertaggedImage = cs.ImageRepository + "-" + p.GetPackage().GetFingerPrint() + "-builder"
-		buildertaggedImage = "luet/" + p.GetPackage().GetFingerPrint() + "-builder"
 	}
 	if packageImage == "" {
-		keepPackageImg = false
+		packageImage = cs.ImageRepository + "-" + p.GetPackage().GetFingerPrint()
-		packageImage = "luet/" + p.GetPackage().GetFingerPrint()
+	}
+
+	if cs.PullFirst {
+		//Best effort pull
+		cs.Backend.DownloadImage(CompilerBackendOptions{ImageName: buildertaggedImage})
+		cs.Backend.DownloadImage(CompilerBackendOptions{ImageName: packageImage})
 	}
 
 	Info(pkgTag, "Generating :whale: definition")
@@ -215,19 +223,19 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 	}
 
-	if !keepPackageImg {
+	// if !keepPackageImg {
-		err = cs.Backend.ImageDefinitionToTar(runnerOpts)
+	// 	err = cs.Backend.ImageDefinitionToTar(runnerOpts)
-		if err != nil {
+	// 	if err != nil {
-			return nil, errors.Wrap(err, "Could not export image to tar")
+	// 		return nil, errors.Wrap(err, "Could not export image to tar")
-		}
+	// 	}
-	} else {
+	// } else {
 	if err := cs.Backend.BuildImage(runnerOpts); err != nil {
 		return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
 	}
 	if err := cs.Backend.ExportImage(runnerOpts); err != nil {
 		return nil, errors.Wrap(err, "Failed exporting image")
 	}
-	}
+	//	}
 
 	var diffs []ArtifactLayer
 	var artifact Artifact
@@ -240,6 +248,20 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 	}
 
+	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not create tempdir")
+	}
+	defer os.RemoveAll(rootfs) // clean up
+
+	// TODO: Compression and such
+	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
+		ImageName:  packageImage,
+		SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not extract rootfs")
+	}
+
 	if !keepImg {
 		// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
 		// TODO: Handle caching and optionally do not remove things
@@ -249,17 +271,12 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 			Warning("Could not remove image ", builderOpts.ImageName)
 			//	return nil, errors.Wrap(err, "Could not remove image")
 		}
-	}
+		err = cs.Backend.RemoveImage(runnerOpts)
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
 		if err != nil {
-		return nil, errors.Wrap(err, "Could not create tempdir")
+			// TODO: Have a --fatal flag which enables Warnings to exit.
+			Warning("Could not remove image ", builderOpts.ImageName)
+			//	return nil, errors.Wrap(err, "Could not remove image")
 		}
-	defer os.RemoveAll(rootfs) // clean up
-
-	// TODO: Compression and such
-	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not extract rootfs")
 	}
 
 	if p.ImageUnpack() {
@@ -289,7 +306,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	return artifact, nil
 }
 
-func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions bool) (Artifact, error) {
+func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions, keepImg bool) (Artifact, error) {
 	pkgTag := ":package:  " + p.GetPackage().GetName()
 
 	Info(pkgTag, "   🍩 Build starts 🔨 🔨 🔨 ")
@@ -321,7 +338,9 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 	defer os.RemoveAll(rootfs) // clean up
 
 	// TODO: Compression and such
-	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: builderOpts.Destination, Destination: rootfs}, keepPermissions)
+	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
+		ImageName:  p.GetImage(),
+		SourcePath: builderOpts.Destination, Destination: rootfs}, keepPermissions)
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not extract rootfs")
 	}
@@ -331,6 +350,17 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}
 
+	if !keepImg {
+		// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
+		// TODO: Handle caching and optionally do not remove things
+		err = cs.Backend.RemoveImage(builderOpts)
+		if err != nil {
+			// TODO: Have a --fatal flag which enables Warnings to exit.
+			Warning("Could not remove image ", builderOpts.ImageName)
+			//	return nil, errors.Wrap(err, "Could not remove image")
+		}
+	}
+
 	Info(pkgTag, "   :white_check_mark: Done")
 	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 	artifact.SetCompileSpec(p)
@@ -393,10 +423,10 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
 		if p.ImageUnpack() { // If it is just an entire image, create a package from it
-			return cs.packageFromImage(p, "", keepPermissions)
+			return cs.packageFromImage(p, "", keepPermissions, cs.KeepImg)
 		}
 
-		return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, p)
+		return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, cs.KeepImg, p)
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@@ -427,8 +457,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 		}
 		compileSpec.SetOutputPath(p.GetOutputPath())
 
-		buildImageHash := "luet/cache:" + assertion.Hash.BuildHash
+		buildImageHash := cs.ImageRepository + ":" + assertion.Hash.BuildHash
-		currentPackageImageHash := "luet/cache:" + assertion.Hash.PackageHash
+		currentPackageImageHash := cs.ImageRepository + ":" + assertion.Hash.PackageHash
 		Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image name", buildImageHash)
 		Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
 
@@ -440,7 +470,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 					return nil, errors.New("No image defined for package: " + assertion.Package.GetName())
 				}
 				Info(pkgTag, ":whale: Sourcing package from image", compileSpec.GetImage())
-				artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions)
+				artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions, cs.KeepImg)
 				if err != nil {
 					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
 				}
@@ -449,7 +479,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 			}
 
 			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().GetFingerPrint()+" from image")
-			artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, compileSpec)
+			artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
 			if err != nil {
 				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
 			}
@@ -459,7 +489,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 		}
 
 		Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().GetFingerPrint()+" from tree")
-		artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, compileSpec)
+		artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
 		if err != nil {
 			return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
 			//	deperrs = append(deperrs, err)
@@ -470,7 +500,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	}
 
 	Info(":package:", p.GetPackage().GetName(), ":cyclone:  Building package target from:", lastHash)
-	artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, p)
+	artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, cs.KeepImg, p)
 	if err != nil {
 		return artifact, err
 	}