diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 033ffdfd..164ce5c6 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -123,7 +123,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage err = cs.Backend.BuildImage(builderOpts) if err != nil { - return nil, errors.Wrap(err, "Could not build image: "+image) + return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName) } err = cs.Backend.ExportImage(builderOpts) @@ -147,7 +147,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage } } else { if err := cs.Backend.BuildImage(runnerOpts); err != nil { - return nil, errors.Wrap(err, "Failed building image") + return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName) } if err := cs.Backend.ExportImage(runnerOpts); err != nil { return nil, errors.Wrap(err, "Failed exporting image") @@ -168,6 +168,9 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage } } rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs") + if err != nil { + return nil, errors.Wrap(err, "Could not create tempdir") + } defer os.RemoveAll(rootfs) // clean up // TODO: Compression and such @@ -184,6 +187,12 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage } func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { + Debug("Compiling " + p.GetPackage().GetName()) + + if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" { + Error("Package with no deps and no seed image supplied, bailing out") + return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() + "with no deps and no seed image supplied, bailing out") + } // - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based // on how we compute the resolvable tree. @@ -203,42 +212,52 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila } s := solver.NewSolver([]pkg.Package{}, world) - solution, err := s.Install([]pkg.Package{p.GetPackage()}) + pack, err := cs.Tree().FindPackage(p.GetPackage()) if err != nil { return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) } + solution, err := s.Install([]pkg.Package{pack}) + if err != nil { + return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) + } + Info("Build dependencies: ( target "+p.GetPackage().GetName()+")", solution.Explain()) dependencies := solution.Drop(p.GetPackage()).Order() // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already) departifacts := []Artifact{} // TODO: Return this somehow deperrs := []error{} var lastHash string - Info("Build dependencies:", dependencies.Explain()) + //Info("Build dependencies: ( target "+p.GetPackage().GetName()+")", dependencies.Explain()) if len(dependencies[0].Package.GetRequires()) != 0 { return nil, errors.New("The first dependency of the deptree doesn't have an image base") } for _, assertion := range dependencies { //highly dependent on the order if assertion.Value && assertion.Package.Flagged() { - Info("Building", assertion.Package.GetName()) + Info("( target "+p.GetPackage().GetName()+") Building", assertion.Package.GetName()) compileSpec, err := cs.FromPackage(assertion.Package) if err != nil { return nil, errors.New("Error while generating compilespec for " + assertion.Package.GetName()) } compileSpec.SetOutputPath(p.GetOutputPath()) - pack, err := cs.Tree().FindPackage(p.GetPackage()) - if err != nil { - return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) - } - //TODO: Generate image name of builder image - it should match with the hash up to this point - package - nthsolution, err := s.Install([]pkg.Package{pack}) + depPack, err := cs.Tree().FindPackage(assertion.Package) if err != nil { return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) } - buildImageHash := "luet/cache-" + nthsolution.Drop(p.GetPackage()).AssertionHash() + // Generate image name of builder image - it should match with the hash up to this point - package + nthsolution, err := s.Install([]pkg.Package{depPack}) + if err != nil { + return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName()) + } + + buildImageHash := "luet/cache-" + nthsolution.Drop(depPack).AssertionHash() currentPackageImageHash := "luet/cache-" + nthsolution.AssertionHash() + Debug("("+p.GetPackage().GetName()+") Builder image name:", buildImageHash) + Debug("("+p.GetPackage().GetName()+") Package image name:", currentPackageImageHash) + lastHash = currentPackageImageHash if compileSpec.GetImage() != "" { + Debug("(" + p.GetPackage().GetName() + ") Compiling " + compileSpec.GetPackage().GetFingerPrint() + " from image") artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, compileSpec) if err != nil { deperrs = append(deperrs, err) @@ -257,6 +276,7 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila departifacts = append(departifacts, artifact) } } + Debug("("+p.GetPackage().GetName()+") Building target from", lastHash) return cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, p) } diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index 235d579d..d345b621 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -116,21 +116,25 @@ var _ = Describe("Compiler", func() { Expect(err).ToNot(HaveOccurred()) Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point - Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3)) + Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(4)) compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) + spec3, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"}) + Expect(err).ToNot(HaveOccurred()) Expect(spec.GetPackage().GetPath()).ToNot(Equal("")) spec.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir) - artifacts, errs := compiler.CompileParallel(2, false, []CompilationSpec{spec, spec2}) - Expect(errs).To(Equal("")) - Expect(len(errs)).To(Equal(0)) + spec3.SetOutputPath(tmpdir) + artifacts, errs := compiler.CompileParallel(2, false, []CompilationSpec{spec, spec2, spec3}) + Expect(errs).To(BeNil()) + Expect(len(artifacts)).To(Equal(3)) + for _, artifact := range artifacts { Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) diff --git a/pkg/solver/decoder.go b/pkg/solver/decoder.go index ce17ba12..37f71c73 100644 --- a/pkg/solver/decoder.go +++ b/pkg/solver/decoder.go @@ -71,10 +71,10 @@ func (assertions PackagesAssertions) Order() PackagesAssertions { tmpMap := map[string]PackageAssert{} for _, a := range assertions { + tmpMap[a.Package.GetFingerPrint()] = a if a.Package.Flagged() { unorderedAssertions = append(unorderedAssertions, a) // Build a list of the ones that must be ordered fingerprints = append(fingerprints, a.Package.GetFingerPrint()) - tmpMap[a.Package.GetFingerPrint()] = a } else { orderedAssertions = append(orderedAssertions, a) // Keep last the ones which are not meant to be installed } @@ -92,13 +92,12 @@ func (assertions PackagesAssertions) Order() PackagesAssertions { if !ok { panic("cycle detected") } - for _, res := range result { a, ok := tmpMap[res] if !ok { - panic("Sort order - this shouldn't happen") + continue } - orderedAssertions = append([]PackageAssert{a}, orderedAssertions...) // push upfront + orderedAssertions = append(PackagesAssertions{a}, orderedAssertions...) // push upfront } return orderedAssertions diff --git a/pkg/tree/compiler_recipe.go b/pkg/tree/compiler_recipe.go index 6294e5bd..2e38ef7d 100644 --- a/pkg/tree/compiler_recipe.go +++ b/pkg/tree/compiler_recipe.go @@ -55,7 +55,6 @@ func (r *CompilerRecipe) Load(path string) error { r.Tree().SetPackageSet(pkg.NewInMemoryDatabase(false)) //r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name())) // TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean() - // the function that handles each file or dir var ff = func(currentpath string, info os.FileInfo, err error) error { @@ -78,7 +77,7 @@ func (r *CompilerRecipe) Load(path string) error { // Instead of rdeps, have a different tree for build deps. compileDefPath := pack.Rel(CompilerDefinitionFile) if helpers.Exists(compileDefPath) { - dat, err = ioutil.ReadFile(currentpath) + dat, err = ioutil.ReadFile(compileDefPath) if err != nil { return errors.Wrap(err, "Error reading file "+currentpath) } @@ -104,3 +103,6 @@ func (r *CompilerRecipe) Load(path string) error { } return nil } + +func (r *CompilerRecipe) Tree() pkg.Tree { return r.PackageTree } +func (r *CompilerRecipe) WithTree(t pkg.Tree) { r.PackageTree = t }