diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 67919a39..033ffdfd 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -21,8 +21,11 @@ import ( "path/filepath" "sync" + . "github.com/mudler/luet/pkg/logger" + "github.com/mudler/luet/pkg/helpers" pkg "github.com/mudler/luet/pkg/package" + "github.com/mudler/luet/pkg/solver" "github.com/mudler/luet/pkg/tree" "github.com/pkg/errors" ) @@ -87,6 +90,99 @@ func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, p return artifacts, allErrors } +func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { + p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver, + // and we build all the images first. + keepImg := true + keepPackageImg := true + buildDir := p.Rel("build") + + // First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow) + err := helpers.CopyDir(p.GetPackage().GetPath(), buildDir) + if err != nil { + return nil, errors.Wrap(err, "Could not copy package sources") + + } + if buildertaggedImage == "" { + keepImg = false + buildertaggedImage = "luet/" + p.GetPackage().GetFingerPrint() + "-builder" + } + if packageImage == "" { + keepPackageImg = false + packageImage = "luet/" + p.GetPackage().GetFingerPrint() + } + + // First we create the builder image + p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")) + builderOpts := CompilerBackendOptions{ + ImageName: buildertaggedImage, + SourcePath: buildDir, + DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile", + Destination: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"), + } + + err = cs.Backend.BuildImage(builderOpts) + if err != nil { + return nil, errors.Wrap(err, "Could not build image: "+image) + } + + err = cs.Backend.ExportImage(builderOpts) + if err != nil { + return nil, errors.Wrap(err, "Could not export image") + } + + // Then we write the step image, which uses the builder one + p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")) + runnerOpts := CompilerBackendOptions{ + ImageName: packageImage, + SourcePath: buildDir, + DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile", + Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"), + } + + if !keepPackageImg { + err = cs.Backend.ImageDefinitionToTar(runnerOpts) + if err != nil { + return nil, errors.Wrap(err, "Could not export image to tar") + } + } else { + if err := cs.Backend.BuildImage(runnerOpts); err != nil { + return nil, errors.Wrap(err, "Failed building image") + } + if err := cs.Backend.ExportImage(runnerOpts); err != nil { + return nil, errors.Wrap(err, "Failed exporting image") + } + } + + diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar")) + if err != nil { + return nil, errors.Wrap(err, "Could not generate changes from layers") + } + + if !keepImg { + // We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers + // TODO: Handle caching and optionally do not remove things + err = cs.Backend.RemoveImage(builderOpts) + if err != nil { + return nil, errors.Wrap(err, "Could not remove image") + } + } + rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs") + defer os.RemoveAll(rootfs) // clean up + + // TODO: Compression and such + err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions) + if err != nil { + return nil, errors.Wrap(err, "Could not extract rootfs") + } + artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions) + if err != nil { + return nil, errors.Wrap(err, "Could not generate deltas") + } + + return artifact, nil +} + func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { // - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based @@ -96,80 +192,73 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila // - If image is set we just generate a plain dockerfile // Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above - if p.GetImage() != "" { - p.SetSeedImage(p.GetImage()) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver, - // and we build all the images first. - - buildDir := p.Rel("build") - - // First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow) - err := helpers.CopyDir(p.GetPackage().GetPath(), buildDir) - if err != nil { - return nil, errors.Wrap(err, "Could not copy package sources") - - } - - // First we create the builder image - p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")) - builderOpts := CompilerBackendOptions{ - ImageName: "luet/" + p.GetPackage().GetFingerPrint() + "-builder", - SourcePath: buildDir, - DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile", - Destination: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"), - } - err = cs.Backend.BuildImage(builderOpts) - if err != nil { - return nil, errors.Wrap(err, "Could not build image") - } - - err = cs.Backend.ExportImage(builderOpts) - if err != nil { - return nil, errors.Wrap(err, "Could not export image") - } - - // Then we write the step image, which uses the builder one - p.WriteStepImageDefinition("luet/"+p.GetPackage().GetFingerPrint()+"-builder", filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")) - runnerOpts := CompilerBackendOptions{ - ImageName: "luet/" + p.GetPackage().GetFingerPrint(), - SourcePath: buildDir, - DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile", - Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"), - } - err = cs.Backend.ImageDefinitionToTar(runnerOpts) - if err != nil { - return nil, errors.Wrap(err, "Could not export image to tar") - } - - diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar")) - if err != nil { - return nil, errors.Wrap(err, "Could not generate changes from layers") - } - - // TODO: Handle caching and optionally do not remove things - err = cs.Backend.RemoveImage(builderOpts) - if err != nil { - return nil, errors.Wrap(err, "Could not remove image") - } - - rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs") - defer os.RemoveAll(rootfs) // clean up - - // TODO: Compression and such - err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions) - if err != nil { - return nil, errors.Wrap(err, "Could not extract rootfs") - } - artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions) - if err != nil { - return nil, errors.Wrap(err, "Could not generate deltas") - } - - return artifact, nil + return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, p) } - // TODO: Solve - hash - return nil, errors.New("Image build with a seed image is not implemented yet") + // Get build deps tree (ordered) + world, err := cs.Tree().World() + if err != nil { + return nil, errors.Wrap(err, "While computing tree world") + } + + s := solver.NewSolver([]pkg.Package{}, world) + solution, err := s.Install([]pkg.Package{p.GetPackage()}) + if err != nil { + return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) + } + + dependencies := solution.Drop(p.GetPackage()).Order() // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already) + departifacts := []Artifact{} // TODO: Return this somehow + deperrs := []error{} + var lastHash string + Info("Build dependencies:", dependencies.Explain()) + + if len(dependencies[0].Package.GetRequires()) != 0 { + return nil, errors.New("The first dependency of the deptree doesn't have an image base") + } + for _, assertion := range dependencies { //highly dependent on the order + if assertion.Value && assertion.Package.Flagged() { + Info("Building", assertion.Package.GetName()) + compileSpec, err := cs.FromPackage(assertion.Package) + if err != nil { + return nil, errors.New("Error while generating compilespec for " + assertion.Package.GetName()) + } + compileSpec.SetOutputPath(p.GetOutputPath()) + pack, err := cs.Tree().FindPackage(p.GetPackage()) + if err != nil { + return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) + } + //TODO: Generate image name of builder image - it should match with the hash up to this point - package + nthsolution, err := s.Install([]pkg.Package{pack}) + if err != nil { + return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) + } + + buildImageHash := "luet/cache-" + nthsolution.Drop(p.GetPackage()).AssertionHash() + currentPackageImageHash := "luet/cache-" + nthsolution.AssertionHash() + lastHash = currentPackageImageHash + if compileSpec.GetImage() != "" { + artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, compileSpec) + if err != nil { + deperrs = append(deperrs, err) + break // stop at first error + } + departifacts = append(departifacts, artifact) + continue + } + + artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, compileSpec) + if err != nil { + return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName()) + // deperrs = append(deperrs, err) + // break // stop at first error + } + departifacts = append(departifacts, artifact) + } + } + + return cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, p) } func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) { diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index 71acc8f1..235d579d 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -104,4 +104,38 @@ var _ = Describe("Compiler", func() { }) }) + + Context("Reconstruct image tree", func() { + FIt("Compiles it", func() { + generalRecipe := tree.NewCompilerRecipe() + tmpdir, err := ioutil.TempDir("", "package") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + + err = generalRecipe.Load("../../tests/fixtures/buildableseed") + Expect(err).ToNot(HaveOccurred()) + Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point + + Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3)) + + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree()) + spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"}) + Expect(err).ToNot(HaveOccurred()) + spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"}) + Expect(err).ToNot(HaveOccurred()) + + Expect(spec.GetPackage().GetPath()).ToNot(Equal("")) + + spec.SetOutputPath(tmpdir) + spec2.SetOutputPath(tmpdir) + artifacts, errs := compiler.CompileParallel(2, false, []CompilationSpec{spec, spec2}) + Expect(errs).To(Equal("")) + Expect(len(errs)).To(Equal(0)) + for _, artifact := range artifacts { + Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) + Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + } + + }) + }) })