WIP Compiler for images by solving the deptree

pkg/compiler/compiler.go

@@ -21,8 +21,11 @@ import (
 	"path/filepath"
 	"sync"
 
+	. "github.com/mudler/luet/pkg/logger"
+
 	"github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	"github.com/mudler/luet/pkg/tree"
 	"github.com/pkg/errors"
 )
@@ -87,20 +90,11 @@ func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, p
 	return artifacts, allErrors
 }
 
-func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
+func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
-
+	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
-	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
-	// on how we compute the resolvable tree.
-	// This means to recursively build all the build-images needed to reach that tree part.
-	// - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image.
-	// - If image is set we just generate a plain dockerfile
-
-	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
-
-	if p.GetImage() != "" {
-		p.SetSeedImage(p.GetImage()) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
 	// and we build all the images first.
-
+	keepImg := true
+	keepPackageImg := true
 	buildDir := p.Rel("build")
 
 	// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
@@ -109,18 +103,27 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 		return nil, errors.Wrap(err, "Could not copy package sources")
 
 	}
+	if buildertaggedImage == "" {
+		keepImg = false
+		buildertaggedImage = "luet/" + p.GetPackage().GetFingerPrint() + "-builder"
+	}
+	if packageImage == "" {
+		keepPackageImg = false
+		packageImage = "luet/" + p.GetPackage().GetFingerPrint()
+	}
 
 	// First we create the builder image
 	p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile"))
 	builderOpts := CompilerBackendOptions{
-			ImageName:      "luet/" + p.GetPackage().GetFingerPrint() + "-builder",
+		ImageName:      buildertaggedImage,
 		SourcePath:     buildDir,
 		DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 	}
+
 	err = cs.Backend.BuildImage(builderOpts)
 	if err != nil {
-			return nil, errors.Wrap(err, "Could not build image")
+		return nil, errors.Wrap(err, "Could not build image: "+image)
 	}
 
 	err = cs.Backend.ExportImage(builderOpts)
@@ -129,29 +132,41 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 	}
 
 	// Then we write the step image, which uses the builder one
-		p.WriteStepImageDefinition("luet/"+p.GetPackage().GetFingerPrint()+"-builder", filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
+	p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
 	runnerOpts := CompilerBackendOptions{
-			ImageName:      "luet/" + p.GetPackage().GetFingerPrint(),
+		ImageName:      packageImage,
 		SourcePath:     buildDir,
 		DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 	}
+
+	if !keepPackageImg {
 		err = cs.Backend.ImageDefinitionToTar(runnerOpts)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not export image to tar")
 		}
+	} else {
+		if err := cs.Backend.BuildImage(runnerOpts); err != nil {
+			return nil, errors.Wrap(err, "Failed building image")
+		}
+		if err := cs.Backend.ExportImage(runnerOpts); err != nil {
+			return nil, errors.Wrap(err, "Failed exporting image")
+		}
+	}
 
 	diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not generate changes from layers")
 	}
 
+	if !keepImg {
+		// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
 		// TODO: Handle caching and optionally do not remove things
 		err = cs.Backend.RemoveImage(builderOpts)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not remove image")
 		}
-
+	}
 	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
 	defer os.RemoveAll(rootfs) // clean up
 
@@ -168,8 +183,82 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 	return artifact, nil
 }
 
-	// TODO: Solve - hash
+func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
-	return nil, errors.New("Image build with a seed image is not implemented yet")
+
+	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
+	// on how we compute the resolvable tree.
+	// This means to recursively build all the build-images needed to reach that tree part.
+	// - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image.
+	// - If image is set we just generate a plain dockerfile
+
+	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
+	if p.GetImage() != "" {
+		return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, p)
+	}
+
+	// Get build deps tree (ordered)
+	world, err := cs.Tree().World()
+	if err != nil {
+		return nil, errors.Wrap(err, "While computing tree world")
+	}
+
+	s := solver.NewSolver([]pkg.Package{}, world)
+	solution, err := s.Install([]pkg.Package{p.GetPackage()})
+	if err != nil {
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+	}
+
+	dependencies := solution.Drop(p.GetPackage()).Order() // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
+	departifacts := []Artifact{}                          // TODO: Return this somehow
+	deperrs := []error{}
+	var lastHash string
+	Info("Build dependencies:", dependencies.Explain())
+
+	if len(dependencies[0].Package.GetRequires()) != 0 {
+		return nil, errors.New("The first dependency of the deptree doesn't have an image base")
+	}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value && assertion.Package.Flagged() {
+			Info("Building", assertion.Package.GetName())
+			compileSpec, err := cs.FromPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.New("Error while generating compilespec for " + assertion.Package.GetName())
+			}
+			compileSpec.SetOutputPath(p.GetOutputPath())
+			pack, err := cs.Tree().FindPackage(p.GetPackage())
+			if err != nil {
+				return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+			}
+			//TODO: Generate image name of builder image - it should match with the hash up to this point - package
+			nthsolution, err := s.Install([]pkg.Package{pack})
+			if err != nil {
+				return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+			}
+
+			buildImageHash := "luet/cache-" + nthsolution.Drop(p.GetPackage()).AssertionHash()
+			currentPackageImageHash := "luet/cache-" + nthsolution.AssertionHash()
+			lastHash = currentPackageImageHash
+			if compileSpec.GetImage() != "" {
+				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, compileSpec)
+				if err != nil {
+					deperrs = append(deperrs, err)
+					break // stop at first error
+				}
+				departifacts = append(departifacts, artifact)
+				continue
+			}
+
+			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, compileSpec)
+			if err != nil {
+				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
+				//	deperrs = append(deperrs, err)
+				//		break // stop at first error
+			}
+			departifacts = append(departifacts, artifact)
+		}
+	}
+
+	return cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, p)
 }
 
 func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {

pkg/compiler/compiler_test.go

@@ -104,4 +104,38 @@ var _ = Describe("Compiler", func() {
 
 		})
 	})
+
+	Context("Reconstruct image tree", func() {
+		FIt("Compiles it", func() {
+			generalRecipe := tree.NewCompilerRecipe()
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/buildableseed")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
+
+			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+			artifacts, errs := compiler.CompileParallel(2, false, []CompilationSpec{spec, spec2})
+			Expect(errs).To(Equal(""))
+			Expect(len(errs)).To(Equal(0))
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+
+		})
+	})
 })