diff --git a/cmd/build.go b/cmd/build.go index 78509363..8762fd54 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -247,11 +247,12 @@ Build packages specifying multiple definition trees: } for _, sp := range toCalculate { - packs, err := luetCompiler.ComputeDepTree(sp) + ht := compiler.NewHashTree(generalRecipe.GetDatabase()) + hashTree, err := ht.Query(luetCompiler, sp) if err != nil { errs = append(errs, err) } - for _, p := range packs { + for _, p := range hashTree.Dependencies { results.Packages = append(results.Packages, PackageResult{ Name: p.Package.GetName(), diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 04a4e724..29fec6d1 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -146,11 +146,6 @@ func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps *compilerspec.L } for _, p := range ps.All() { - asserts, err := cs.ComputeDepTree(p) - if err != nil { - panic(err) - } - p.SetSourceAssertion(asserts) all <- p } @@ -295,17 +290,6 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co return artifact, nil } -func (cs *LuetCompiler) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string { - // Use packageImage as salt into the fp being used - // so the hash is unique also in cases where - // some package deps does have completely different - // depgraphs - // TODO: We should use the image tag, or pass by the package assertion hash which is unique - // and identifies the deptree of the package. - return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(helpers.StripRegistryFromImage(packageImage))) - -} - func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (backend.Options, backend.Options, error) { @@ -680,33 +664,7 @@ func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst s } } -// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs -func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) { - // Generate a set with all the deps of the provided specs - // we will use that set to remove the deps from the list of provided compilation specs - allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps - result := []*compilerspec.LuetCompilationSpec{} - for _, spec := range p { - ass, err := cs.ComputeDepTree(spec) - if err != nil { - return result, errors.Wrap(err, "computin specs deptree") - } - - allDependencies = append(allDependencies, ass.Drop(spec.GetPackage())...) - } - - for _, spec := range p { - if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil { - result = append(result, spec) - } - } - return result, nil -} - -// ComputeDepTree computes the dependency tree of a compilation spec and returns solver assertions -// in order to be able to compile the spec. func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) { - s := solver.NewResolver(cs.Options.SolverOptions.Options, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver()) solution, err := s.Install(pkg.Packages{p.GetPackage()}) @@ -718,31 +676,34 @@ func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (sol if err != nil { return nil, errors.Wrap(err, "While order a solution for "+p.GetPackage().HumanReadableString()) } + return dependencies, nil +} - assertions := solver.PackagesAssertions{} - for _, assertion := range dependencies { //highly dependent on the order - if assertion.Value { - nthsolution := dependencies.Cut(assertion.Package) - assertion.Hash = solver.PackageHash{ - BuildHash: nthsolution.HashFrom(assertion.Package), - PackageHash: nthsolution.AssertionHash(), - } - assertion.Package.SetTreeDir(p.Package.GetTreeDir()) - assertions = append(assertions, assertion) +// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs +func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) { + // Generate a set with all the deps of the provided specs + // we will use that set to remove the deps from the list of provided compilation specs + allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps + result := []*compilerspec.LuetCompilationSpec{} + for _, spec := range p { + sol, err := cs.ComputeDepTree(spec) + if err != nil { + return nil, errors.Wrap(err, "failed querying hashtree") + } + allDependencies = append(allDependencies, sol.Drop(spec.GetPackage())...) + } + + for _, spec := range p { + if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil { + result = append(result, spec) } } - p.SetSourceAssertion(assertions) - return assertions, nil + return result, nil } // Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates // an artifact func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { - asserts, err := cs.ComputeDepTree(p) - if err != nil { - return nil, err - } - p.SetSourceAssertion(asserts) return cs.compile(cs.Options.Concurrency, keepPermissions, p) } @@ -773,11 +734,6 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS } func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { - // TODO: Racy, remove it - // Inherit build options from compilation specs metadata - // orig := cs.Options.PullImageRepository - // defer func() { cs.Options.PullImageRepository = orig }() - Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:") Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage())) @@ -789,36 +745,42 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil ) } - targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint()) + ht := NewHashTree(cs.Database) + + packageHashTree, err := ht.Query(cs, p) + if err != nil { + return nil, errors.Wrap(err, "failed querying hashtree") + } + + // This is in order to have the metadata in the yaml + p.SetSourceAssertion(packageHashTree.Solution) + targetAssertion := packageHashTree.Target bus.Manager.Publish(bus.EventPackagePreBuild, struct { - CompileSpec *compilerspec.LuetCompilationSpec - Assert solver.PackageAssert + CompileSpec *compilerspec.LuetCompilationSpec + Assert solver.PackageAssert + PackageHashTree *PackageImageHashTree }{ - CompileSpec: p, - Assert: *targetAssertion, + CompileSpec: p, + Assert: *targetAssertion, + PackageHashTree: packageHashTree, }) // Update compilespec build options - it will be then serialized into the compilation metadata file - //p.SetBuildOptions(cs.Options) p.BuildOptions.PushImageRepository = cs.Options.PushImageRepository - //p.BuildOptions.BuildValues = cs.Options.BuildValues - //p.BuildOptions.BuildValuesFile = cs.Options.BuildValuesFile // - If image is set we just generate a plain dockerfile // Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above if p.GetImage() != "" { - return cs.compileWithImage(p.GetImage(), cs.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true) + return cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true) } // - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based // on how we compute the resolvable tree. // This means to recursively build all the build-images needed to reach that tree part. // - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image. - - dependencies := p.GetSourceAssertion().Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already) - departifacts := []*artifact.PackageArtifact{} // TODO: Return this somehow - var lastHash string + dependencies := packageHashTree.Dependencies // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already) + departifacts := []*artifact.PackageArtifact{} // TODO: Return this somehow depsN := 0 currentN := 0 @@ -845,8 +807,6 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository) compileSpec.SetOutputPath(p.GetOutputPath()) - Debug(pkgTag, " :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash) - Debug(pkgTag, " :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash) bus.Manager.Publish(bus.EventPackagePreBuild, struct { CompileSpec *compilerspec.LuetCompilationSpec @@ -856,29 +816,43 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil Assert: assertion, }) - lastHash = assertion.Hash.PackageHash - // for the source instead, pick an image and a buildertaggedImage from hashes if they exists. - // otherways fallback to the pushed repo - // Resolve images from the hashtree - resolvedBuildImage := cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec) - if compileSpec.GetImage() != "" { - Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image") - - a, err := cs.compileWithImage(compileSpec.GetImage(), assertion.Hash.BuildHash, assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps) - if err != nil { - return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString()) - } - departifacts = append(departifacts, a) - Info(pkgTag, ":white_check_mark: Done") - continue + buildHash, err := packageHashTree.DependencyBuildImage(assertion.Package) + if err != nil { + return nil, errors.Wrap(err, "failed looking for dependency in hashtree") } - Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree") - a, err := cs.compileWithImage(resolvedBuildImage, cs.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash), assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps) + Debug(pkgTag, " :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash) + Debug(pkgTag, " :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash) + + var sourceImage string + + if compileSpec.GetImage() != "" { + Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image") + sourceImage = compileSpec.GetImage() + } else { + // for the source instead, pick an image and a buildertaggedImage from hashes if they exists. + // otherways fallback to the pushed repo + // Resolve images from the hashtree + sourceImage = cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec) + Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree") + } + + a, err := cs.compileWithImage( + sourceImage, + buildHash, + assertion.Hash.PackageHash, + concurrency, + keepPermissions, + cs.Options.KeepImg, + compileSpec, + packageDeps, + ) if err != nil { return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString()) } + Info(pkgTag, ":white_check_mark: Done") + bus.Manager.Publish(bus.EventPackagePostBuild, struct { CompileSpec *compilerspec.LuetCompilationSpec Artifact *artifact.PackageArtifact @@ -888,18 +862,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil }) departifacts = append(departifacts, a) - Info(pkgTag, ":white_check_mark: Done") } - - } else if len(dependencies) > 0 { - lastHash = dependencies[len(dependencies)-1].Hash.PackageHash } if buildTarget { - resolvedBuildImage := cs.resolveExistingImageHash(lastHash, p) + resolvedSourceImage := cs.resolveExistingImageHash(packageHashTree.SourceHash, p) Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString()) - Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedBuildImage) - a, err := cs.compileWithImage(resolvedBuildImage, cs.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true) + Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedSourceImage) + a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true) if err != nil { return a, err } diff --git a/pkg/compiler/imagehashtree.go b/pkg/compiler/imagehashtree.go new file mode 100644 index 00000000..6e885e08 --- /dev/null +++ b/pkg/compiler/imagehashtree.go @@ -0,0 +1,126 @@ +// Copyright © 2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package compiler + +import ( + "fmt" + + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" + "github.com/mudler/luet/pkg/config" + pkg "github.com/mudler/luet/pkg/package" + "github.com/mudler/luet/pkg/solver" + "github.com/pkg/errors" +) + +type ImageHashTree struct { + Database pkg.PackageDatabase + SolverOptions config.LuetSolverOptions +} + +type PackageImageHashTree struct { + Target *solver.PackageAssert + Dependencies solver.PackagesAssertions + Solution solver.PackagesAssertions + dependencyBuilderImageHashes map[string]string + SourceHash string + BuilderImageHash string +} + +func NewHashTree(db pkg.PackageDatabase) *ImageHashTree { + return &ImageHashTree{ + Database: db, + } +} + +func (ht *PackageImageHashTree) DependencyBuildImage(p pkg.Package) (string, error) { + found, ok := ht.dependencyBuilderImageHashes[p.GetFingerPrint()] + if !ok { + return "", errors.New("package hash not found") + } + return found, nil +} + +// TODO: ___ When computing the hash per package (and evaluating the sat solver solution tree part) +// we should use the hash of each package + its fingerprint instead as a salt. +// That's because the hash will be salted with its `build.yaml`. +// In this way, we trigger recompilations if some dep of a target changes +// a build.yaml, without touching the version +func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) { + assertions, err := ht.resolve(cs, p) + if err != nil { + return nil, err + } + targetAssertion := assertions.Search(p.GetPackage().GetFingerPrint()) + + dependencies := assertions.Drop(p.GetPackage()) + var sourceHash string + imageHashes := map[string]string{} + for _, assertion := range dependencies { + var depbuildImageTag string + compileSpec, err := cs.FromPackage(assertion.Package) + if err != nil { + return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName()) + } + if compileSpec.GetImage() != "" { + depbuildImageTag = assertion.Hash.BuildHash + } else { + depbuildImageTag = ht.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash) + } + imageHashes[assertion.Package.GetFingerPrint()] = depbuildImageTag + sourceHash = assertion.Hash.PackageHash + } + + return &PackageImageHashTree{ + Dependencies: dependencies, + Target: targetAssertion, + SourceHash: sourceHash, + BuilderImageHash: ht.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), + dependencyBuilderImageHashes: imageHashes, + Solution: assertions, + }, nil +} + +func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string { + // Use packageImage as salt into the fp being used + // so the hash is unique also in cases where + // some package deps does have completely different + // depgraphs + return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(packageImage)) +} + +// resolve computes the dependency tree of a compilation spec and returns solver assertions +// in order to be able to compile the spec. +func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) { + dependencies, err := cs.ComputeDepTree(p) + if err != nil { + return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString()) + } + + assertions := solver.PackagesAssertions{} + for _, assertion := range dependencies { //highly dependent on the order + if assertion.Value { + nthsolution := dependencies.Cut(assertion.Package) + assertion.Hash = solver.PackageHash{ + BuildHash: nthsolution.HashFrom(assertion.Package), + PackageHash: nthsolution.AssertionHash(), + } + assertion.Package.SetTreeDir(p.Package.GetTreeDir()) + assertions = append(assertions, assertion) + } + } + + return assertions, nil +} diff --git a/pkg/compiler/imagehashtree_test.go b/pkg/compiler/imagehashtree_test.go new file mode 100644 index 00000000..25781984 --- /dev/null +++ b/pkg/compiler/imagehashtree_test.go @@ -0,0 +1,94 @@ +// Copyright © 2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package compiler_test + +import ( + . "github.com/mudler/luet/pkg/compiler" + sd "github.com/mudler/luet/pkg/compiler/backend" + "github.com/mudler/luet/pkg/compiler/types/options" + pkg "github.com/mudler/luet/pkg/package" + "github.com/mudler/luet/pkg/tree" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("ImageHashTree", func() { + generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false)) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2)) + hashtree := NewHashTree(generalRecipe.GetDatabase()) + Context("Simple package definition", func() { + BeforeEach(func() { + generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false)) + err := generalRecipe.Load("../../tests/fixtures/buildable") + Expect(err).ToNot(HaveOccurred()) + compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2)) + hashtree = NewHashTree(generalRecipe.GetDatabase()) + + }) + + It("Calculates the hash correctly", func() { + + spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) + Expect(err).ToNot(HaveOccurred()) + + packageHash, err := hashtree.Query(compiler, spec) + Expect(err).ToNot(HaveOccurred()) + Expect(packageHash.Target.Hash.BuildHash).To(Equal("6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e")) + Expect(packageHash.Target.Hash.PackageHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281")) + Expect(packageHash.BuilderImageHash).To(Equal("builder-79462b60bf899ad79db63f194a3c9c2a")) + }) + }) + + Context("complex package definition", func() { + BeforeEach(func() { + generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false)) + + err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision") + Expect(err).ToNot(HaveOccurred()) + compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2)) + hashtree = NewHashTree(generalRecipe.GetDatabase()) + + }) + It("Calculates the hash correctly", func() { + spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"}) + Expect(err).ToNot(HaveOccurred()) + + packageHash, err := hashtree.Query(compiler, spec) + Expect(err).ToNot(HaveOccurred()) + + Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982")) + Expect(packageHash.SourceHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982")) + Expect(packageHash.BuilderImageHash).To(Equal("builder-37f4d05ba8a39525742ca364f69b4090")) + + //Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281")) + Expect(packageHash.Target.Hash.PackageHash).To(Equal("bb1d9a99c0c309a297c75b436504e664a42121fadbb4e035bda403cd418117aa")) + a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"} + hash, err := packageHash.DependencyBuildImage(a) + Expect(err).ToNot(HaveOccurred()) + Expect(hash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281")) + + assertionA := packageHash.Dependencies.Search(a.GetFingerPrint()) + Expect(assertionA.Hash.PackageHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982")) + b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"} + assertionB := packageHash.Dependencies.Search(b.GetFingerPrint()) + Expect(assertionB.Hash.PackageHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281")) + hashB, err := packageHash.DependencyBuildImage(b) + Expect(err).ToNot(HaveOccurred()) + Expect(hashB).To(Equal("6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e")) + }) + }) + +})