// Copyright ยฉ 2019 Ettore Di Giacinto // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License along // with this program; if not, see . package compiler import ( "fmt" "io/ioutil" "os" "path/filepath" "sync" . "github.com/mudler/luet/pkg/logger" "github.com/mudler/luet/pkg/helpers" pkg "github.com/mudler/luet/pkg/package" "github.com/mudler/luet/pkg/solver" "github.com/mudler/luet/pkg/tree" "github.com/pkg/errors" ) const BuildFile = "build.yaml" type LuetCompiler struct { *tree.CompilerRecipe Backend CompilerBackend Database pkg.PackageDatabase } func NewLuetCompiler(backend CompilerBackend, t pkg.Tree, db pkg.PackageDatabase) Compiler { // The CompilerRecipe will gives us a tree with only build deps listed. return &LuetCompiler{ Backend: backend, CompilerRecipe: &tree.CompilerRecipe{ tree.Recipe{PackageTree: t}, }, Database: db, } } func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan CompilationSpec, a *[]Artifact, m *sync.Mutex, concurrency int, keepPermissions bool, errors chan error) { defer wg.Done() for s := range cspecs { ar, err := cs.compile(concurrency, keepPermissions, s) if err != nil { errors <- err } m.Lock() *a = append(*a, ar) m.Unlock() } } func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) { artifacts, err := cs.CompileParallel(concurrency, keepPermissions, ps) if len(err) != 0 { return artifacts, err } Info(":ant: Resolving reverse dependencies") toCompile := NewLuetCompilationspecs() for _, a := range artifacts { w, asserterr := cs.Tree().World() if asserterr != nil { return nil, append(err, asserterr) } revdeps := a.GetCompileSpec().GetPackage().Revdeps(&w) for _, r := range revdeps { spec, asserterr := cs.FromPackage(r) if err != nil { return nil, append(err, asserterr) } spec.SetOutputPath(ps.All()[0].GetOutputPath()) toCompile.Add(spec) } // for _, assertion := range a.GetSourceAssertion() { // if assertion.Value && assertion.Package.Flagged() { // spec, asserterr := cs.FromPackage(assertion.Package) // if err != nil { // return nil, append(err, asserterr) // } // w, asserterr := cs.Tree().World() // if err != nil { // return nil, append(err, asserterr) // } // revdeps := spec.GetPackage().Revdeps(&w) // for _, r := range revdeps { // spec, asserterr := cs.FromPackage(r) // if asserterr != nil { // return nil, append(err, asserterr) // } // spec.SetOutputPath(ps.All()[0].GetOutputPath()) // toCompile.Add(spec) // } // } // } } uniques := toCompile.Unique().Remove(ps) for _, u := range uniques.All() { Info(" :arrow_right_hook:", u.GetPackage().GetName(), ":leaves:", u.GetPackage().GetVersion(), "(", u.GetPackage().GetCategory(), ")") } artifacts2, err := cs.CompileParallel(concurrency, keepPermissions, uniques) return append(artifacts, artifacts2...), err } func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) { Spinner(22) defer SpinnerStop() all := make(chan CompilationSpec) artifacts := []Artifact{} mutex := &sync.Mutex{} errors := make(chan error, ps.Len()) var wg = new(sync.WaitGroup) for i := 0; i < concurrency; i++ { wg.Add(1) go cs.compilerWorker(i, wg, all, &artifacts, mutex, concurrency, keepPermissions, errors) } for _, p := range ps.All() { asserts, err := cs.ComputeDepTree(p) if err != nil { panic(err) } p.SetSourceAssertion(asserts) all <- p } close(all) wg.Wait() close(errors) var allErrors []error for e := range errors { allErrors = append(allErrors, e) } return artifacts, allErrors } func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver, // and we build all the images first. keepImg := true keepPackageImg := true err := os.MkdirAll(p.Rel("build"), os.ModePerm) if err != nil { return nil, errors.Wrap(err, "Error met while creating tempdir for building") } buildDir, err := ioutil.TempDir(p.Rel("build"), "pack") if err != nil { return nil, errors.Wrap(err, "Error met while creating tempdir for building") } defer os.RemoveAll(buildDir) // clean up // First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow) err = helpers.CopyDir(p.GetPackage().GetPath(), buildDir) if err != nil { return nil, errors.Wrap(err, "Could not copy package sources") } if buildertaggedImage == "" { keepImg = false buildertaggedImage = "luet/" + p.GetPackage().GetFingerPrint() + "-builder" } if packageImage == "" { keepPackageImg = false packageImage = "luet/" + p.GetPackage().GetFingerPrint() } // First we create the builder image p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")) builderOpts := CompilerBackendOptions{ ImageName: buildertaggedImage, SourcePath: buildDir, DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile", Destination: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"), } err = cs.Backend.BuildImage(builderOpts) if err != nil { return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName) } err = cs.Backend.ExportImage(builderOpts) if err != nil { return nil, errors.Wrap(err, "Could not export image") } // Then we write the step image, which uses the builder one p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")) runnerOpts := CompilerBackendOptions{ ImageName: packageImage, SourcePath: buildDir, DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile", Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"), } if !keepPackageImg { err = cs.Backend.ImageDefinitionToTar(runnerOpts) if err != nil { return nil, errors.Wrap(err, "Could not export image to tar") } } else { if err := cs.Backend.BuildImage(runnerOpts); err != nil { return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName) } if err := cs.Backend.ExportImage(runnerOpts); err != nil { return nil, errors.Wrap(err, "Failed exporting image") } } diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar")) if err != nil { return nil, errors.Wrap(err, "Could not generate changes from layers") } if !keepImg { // We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers // TODO: Handle caching and optionally do not remove things err = cs.Backend.RemoveImage(builderOpts) if err != nil { // TODO: Have a --fatal flag which enables Warnings to exit. Warning("Could not remove image ", builderOpts.ImageName) // return nil, errors.Wrap(err, "Could not remove image") } } rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs") if err != nil { return nil, errors.Wrap(err, "Could not create tempdir") } defer os.RemoveAll(rootfs) // clean up // TODO: Compression and such err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions) if err != nil { return nil, errors.Wrap(err, "Could not extract rootfs") } if p.ImageUnpack() { err = helpers.Tar(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar")) if err != nil { return nil, errors.Wrap(err, "Error met while creating package archive") } artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")) artifact.SetCompileSpec(p) return artifact, nil } artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes()) if err != nil { return nil, errors.Wrap(err, "Could not generate deltas") } artifact.SetCompileSpec(p) return artifact, nil } func (cs *LuetCompiler) Prepare(concurrency int) error { err := cs.Tree().ResolveDeps(concurrency) // FIXME: When done in parallel, this could be done on top before starting if err != nil { return errors.Wrap(err, "While resoolving tree world deps") } return nil } func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions bool) (Artifact, error) { pkgTag := ":package: " + p.GetPackage().GetName() Info(pkgTag, " ๐Ÿฉ Build starts ๐Ÿ”จ ๐Ÿ”จ ๐Ÿ”จ ") builderOpts := CompilerBackendOptions{ ImageName: p.GetImage(), Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"), } err := cs.Backend.DownloadImage(builderOpts) if err != nil { return nil, errors.Wrap(err, "Could not download image") } if tag != "" { err = cs.Backend.CopyImage(p.GetImage(), tag) if err != nil { return nil, errors.Wrap(err, "Could not download image") } } err = cs.Backend.ExportImage(builderOpts) if err != nil { return nil, errors.Wrap(err, "Could not export image") } rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs") if err != nil { return nil, errors.Wrap(err, "Could not create tempdir") } defer os.RemoveAll(rootfs) // clean up // TODO: Compression and such err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: builderOpts.Destination, Destination: rootfs}, keepPermissions) if err != nil { return nil, errors.Wrap(err, "Could not extract rootfs") } err = helpers.Tar(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar")) if err != nil { return nil, errors.Wrap(err, "Error met while creating package archive") } Info(pkgTag, " :white_check_mark: Done") artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")) artifact.SetCompileSpec(p) return artifact, nil } func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) { // Get build deps tree (ordered) world, err := cs.Tree().World() if err != nil { return nil, errors.Wrap(err, "While computing tree world") } s := solver.NewSolver([]pkg.Package{}, world, cs.Database) pack, err := cs.Tree().FindPackage(p.GetPackage()) if err != nil { return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) } solution, err := s.Install([]pkg.Package{pack}) if err != nil { return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) } dependencies := solution.Order(p.GetPackage().GetFingerPrint()) assertions := solver.PackagesAssertions{} for _, assertion := range dependencies { //highly dependent on the order if assertion.Value && assertion.Package.Flagged() { depPack, err := cs.Tree().FindPackage(assertion.Package) if err != nil { return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) } nthsolution, err := s.Install([]pkg.Package{depPack}) if err != nil { return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) } assertion.Hash = solver.PackageHash{ BuildHash: nthsolution.Order(depPack.GetFingerPrint()).Drop(depPack).AssertionHash(), PackageHash: nthsolution.Order(depPack.GetFingerPrint()).AssertionHash(), } assertions = append(assertions, assertion) } } p.SetSourceAssertion(assertions) return assertions, nil } // Compile is non-parallel func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { asserts, err := cs.ComputeDepTree(p) if err != nil { panic(err) } p.SetSourceAssertion(asserts) return cs.compile(concurrency, keepPermissions, p) } func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { Info(":package: Compiling", p.GetPackage().GetName(), "version", p.GetPackage().GetVersion(), ".... :coffee:") if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" { Error("Package with no deps and no seed image supplied, bailing out") return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() + "with no deps and no seed image supplied, bailing out") } // - If image is set we just generate a plain dockerfile // Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above if p.GetImage() != "" { if p.ImageUnpack() { // If it is just an entire image, create a package from it return cs.packageFromImage(p, "", keepPermissions) } return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, p) } // - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based // on how we compute the resolvable tree. // This means to recursively build all the build-images needed to reach that tree part. // - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image. dependencies := p.GetSourceAssertion().Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already) departifacts := []Artifact{} // TODO: Return this somehow deperrs := []error{} var lastHash string depsN := 0 currentN := 0 Info(":deciduous_tree: Build dependencies for " + p.GetPackage().GetName()) for _, assertion := range dependencies { //highly dependent on the order depsN++ Info(" :arrow_right_hook:", assertion.Package.GetName(), ":leaves:", assertion.Package.GetVersion(), "(", assertion.Package.GetCategory(), ")") } for _, assertion := range dependencies { //highly dependent on the order currentN++ pkgTag := fmt.Sprintf(":package: %d/%d %s โค‘ %s", currentN, depsN, p.GetPackage().GetName(), assertion.Package.GetName()) Info(pkgTag, " :zap: Building dependency") compileSpec, err := cs.FromPackage(assertion.Package) if err != nil { return nil, errors.New("Error while generating compilespec for " + assertion.Package.GetName()) } compileSpec.SetOutputPath(p.GetOutputPath()) buildImageHash := "luet/cache:" + assertion.Hash.BuildHash currentPackageImageHash := "luet/cache:" + assertion.Hash.PackageHash Debug(pkgTag, " :arrow_right_hook: :whale: Builder image name", buildImageHash) Debug(pkgTag, " :arrow_right_hook: :whale: Package image name", currentPackageImageHash) lastHash = currentPackageImageHash if compileSpec.GetImage() != "" { // TODO: Refactor this if compileSpec.ImageUnpack() { // If it is just an entire image, create a package from it if compileSpec.GetImage() == "" { return nil, errors.New("No image defined for package: " + assertion.Package.GetName()) } Info(":whale: Sourcing package from image", compileSpec.GetImage()) artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions) if err != nil { deperrs = append(deperrs, err) break // stop at first error } departifacts = append(departifacts, artifact) continue } Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().GetFingerPrint()+" from image") artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, compileSpec) if err != nil { deperrs = append(deperrs, err) break // stop at first error } departifacts = append(departifacts, artifact) Info(pkgTag, ":white_check_mark: Done") continue } artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, compileSpec) if err != nil { return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName()) // deperrs = append(deperrs, err) // break // stop at first error } departifacts = append(departifacts, artifact) Info(pkgTag, ":collision: Done") } Info(":package:", p.GetPackage().GetName(), ":cyclone: Building package target from:", lastHash) artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, p) if err != nil { return artifact, err } artifact.SetDependencies(departifacts) artifact.SetSourceAssertion(p.GetSourceAssertion()) return artifact, err } func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) { pack, err := cs.Tree().FindPackage(p) if err != nil { return nil, err } buildFile := pack.Rel(BuildFile) if !helpers.Exists(buildFile) { return nil, errors.New("No build file present for " + p.GetFingerPrint()) } dat, err := ioutil.ReadFile(buildFile) if err != nil { return nil, err } return NewLuetCompilationSpec(dat, pack) } func (cs *LuetCompiler) GetBackend() CompilerBackend { return cs.Backend } func (cs *LuetCompiler) SetBackend(b CompilerBackend) { cs.Backend = b }