mirror of
https://github.com/mudler/luet.git
synced 2025-08-29 20:23:17 +00:00
Add hashtree and extract hash logic from compiler
Add unit tests, consume imagehashtree in compiler and cleanup Fixes: #203
This commit is contained in:
parent
fd12227d53
commit
9d2ee1b760
@ -247,11 +247,12 @@ Build packages specifying multiple definition trees:
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, sp := range toCalculate {
|
for _, sp := range toCalculate {
|
||||||
packs, err := luetCompiler.ComputeDepTree(sp)
|
ht := compiler.NewHashTree(generalRecipe.GetDatabase())
|
||||||
|
hashTree, err := ht.Query(luetCompiler, sp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errs = append(errs, err)
|
errs = append(errs, err)
|
||||||
}
|
}
|
||||||
for _, p := range packs {
|
for _, p := range hashTree.Dependencies {
|
||||||
results.Packages = append(results.Packages,
|
results.Packages = append(results.Packages,
|
||||||
PackageResult{
|
PackageResult{
|
||||||
Name: p.Package.GetName(),
|
Name: p.Package.GetName(),
|
||||||
|
@ -146,11 +146,6 @@ func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps *compilerspec.L
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, p := range ps.All() {
|
for _, p := range ps.All() {
|
||||||
asserts, err := cs.ComputeDepTree(p)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
p.SetSourceAssertion(asserts)
|
|
||||||
all <- p
|
all <- p
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -295,17 +290,6 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co
|
|||||||
return artifact, nil
|
return artifact, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cs *LuetCompiler) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
|
|
||||||
// Use packageImage as salt into the fp being used
|
|
||||||
// so the hash is unique also in cases where
|
|
||||||
// some package deps does have completely different
|
|
||||||
// depgraphs
|
|
||||||
// TODO: We should use the image tag, or pass by the package assertion hash which is unique
|
|
||||||
// and identifies the deptree of the package.
|
|
||||||
return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(helpers.StripRegistryFromImage(packageImage)))
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
|
func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
|
||||||
concurrency int, keepPermissions bool,
|
concurrency int, keepPermissions bool,
|
||||||
p *compilerspec.LuetCompilationSpec) (backend.Options, backend.Options, error) {
|
p *compilerspec.LuetCompilationSpec) (backend.Options, backend.Options, error) {
|
||||||
@ -680,33 +664,7 @@ func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst s
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
|
|
||||||
func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
|
|
||||||
// Generate a set with all the deps of the provided specs
|
|
||||||
// we will use that set to remove the deps from the list of provided compilation specs
|
|
||||||
allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
|
|
||||||
result := []*compilerspec.LuetCompilationSpec{}
|
|
||||||
for _, spec := range p {
|
|
||||||
ass, err := cs.ComputeDepTree(spec)
|
|
||||||
if err != nil {
|
|
||||||
return result, errors.Wrap(err, "computin specs deptree")
|
|
||||||
}
|
|
||||||
|
|
||||||
allDependencies = append(allDependencies, ass.Drop(spec.GetPackage())...)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, spec := range p {
|
|
||||||
if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
|
|
||||||
result = append(result, spec)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ComputeDepTree computes the dependency tree of a compilation spec and returns solver assertions
|
|
||||||
// in order to be able to compile the spec.
|
|
||||||
func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
|
func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
|
||||||
|
|
||||||
s := solver.NewResolver(cs.Options.SolverOptions.Options, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
|
s := solver.NewResolver(cs.Options.SolverOptions.Options, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
|
||||||
|
|
||||||
solution, err := s.Install(pkg.Packages{p.GetPackage()})
|
solution, err := s.Install(pkg.Packages{p.GetPackage()})
|
||||||
@ -718,31 +676,34 @@ func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (sol
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "While order a solution for "+p.GetPackage().HumanReadableString())
|
return nil, errors.Wrap(err, "While order a solution for "+p.GetPackage().HumanReadableString())
|
||||||
}
|
}
|
||||||
|
return dependencies, nil
|
||||||
|
}
|
||||||
|
|
||||||
assertions := solver.PackagesAssertions{}
|
// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
|
||||||
for _, assertion := range dependencies { //highly dependent on the order
|
func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
|
||||||
if assertion.Value {
|
// Generate a set with all the deps of the provided specs
|
||||||
nthsolution := dependencies.Cut(assertion.Package)
|
// we will use that set to remove the deps from the list of provided compilation specs
|
||||||
assertion.Hash = solver.PackageHash{
|
allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
|
||||||
BuildHash: nthsolution.HashFrom(assertion.Package),
|
result := []*compilerspec.LuetCompilationSpec{}
|
||||||
PackageHash: nthsolution.AssertionHash(),
|
for _, spec := range p {
|
||||||
|
sol, err := cs.ComputeDepTree(spec)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed querying hashtree")
|
||||||
}
|
}
|
||||||
assertion.Package.SetTreeDir(p.Package.GetTreeDir())
|
allDependencies = append(allDependencies, sol.Drop(spec.GetPackage())...)
|
||||||
assertions = append(assertions, assertion)
|
}
|
||||||
|
|
||||||
|
for _, spec := range p {
|
||||||
|
if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
|
||||||
|
result = append(result, spec)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
p.SetSourceAssertion(assertions)
|
return result, nil
|
||||||
return assertions, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
|
// Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
|
||||||
// an artifact
|
// an artifact
|
||||||
func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
|
func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
|
||||||
asserts, err := cs.ComputeDepTree(p)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
p.SetSourceAssertion(asserts)
|
|
||||||
return cs.compile(cs.Options.Concurrency, keepPermissions, p)
|
return cs.compile(cs.Options.Concurrency, keepPermissions, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -773,11 +734,6 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
|
func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
|
||||||
// TODO: Racy, remove it
|
|
||||||
// Inherit build options from compilation specs metadata
|
|
||||||
// orig := cs.Options.PullImageRepository
|
|
||||||
// defer func() { cs.Options.PullImageRepository = orig }()
|
|
||||||
|
|
||||||
Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
|
Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
|
||||||
|
|
||||||
Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
|
Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
|
||||||
@ -789,36 +745,42 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
|
ht := NewHashTree(cs.Database)
|
||||||
|
|
||||||
|
packageHashTree, err := ht.Query(cs, p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed querying hashtree")
|
||||||
|
}
|
||||||
|
|
||||||
|
// This is in order to have the metadata in the yaml
|
||||||
|
p.SetSourceAssertion(packageHashTree.Solution)
|
||||||
|
targetAssertion := packageHashTree.Target
|
||||||
|
|
||||||
bus.Manager.Publish(bus.EventPackagePreBuild, struct {
|
bus.Manager.Publish(bus.EventPackagePreBuild, struct {
|
||||||
CompileSpec *compilerspec.LuetCompilationSpec
|
CompileSpec *compilerspec.LuetCompilationSpec
|
||||||
Assert solver.PackageAssert
|
Assert solver.PackageAssert
|
||||||
|
PackageHashTree *PackageImageHashTree
|
||||||
}{
|
}{
|
||||||
CompileSpec: p,
|
CompileSpec: p,
|
||||||
Assert: *targetAssertion,
|
Assert: *targetAssertion,
|
||||||
|
PackageHashTree: packageHashTree,
|
||||||
})
|
})
|
||||||
|
|
||||||
// Update compilespec build options - it will be then serialized into the compilation metadata file
|
// Update compilespec build options - it will be then serialized into the compilation metadata file
|
||||||
//p.SetBuildOptions(cs.Options)
|
|
||||||
p.BuildOptions.PushImageRepository = cs.Options.PushImageRepository
|
p.BuildOptions.PushImageRepository = cs.Options.PushImageRepository
|
||||||
//p.BuildOptions.BuildValues = cs.Options.BuildValues
|
|
||||||
//p.BuildOptions.BuildValuesFile = cs.Options.BuildValuesFile
|
|
||||||
|
|
||||||
// - If image is set we just generate a plain dockerfile
|
// - If image is set we just generate a plain dockerfile
|
||||||
// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
|
// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
|
||||||
if p.GetImage() != "" {
|
if p.GetImage() != "" {
|
||||||
return cs.compileWithImage(p.GetImage(), cs.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
|
return cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
|
// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
|
||||||
// on how we compute the resolvable tree.
|
// on how we compute the resolvable tree.
|
||||||
// This means to recursively build all the build-images needed to reach that tree part.
|
// This means to recursively build all the build-images needed to reach that tree part.
|
||||||
// - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image.
|
// - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image.
|
||||||
|
dependencies := packageHashTree.Dependencies // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
|
||||||
dependencies := p.GetSourceAssertion().Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
|
|
||||||
departifacts := []*artifact.PackageArtifact{} // TODO: Return this somehow
|
departifacts := []*artifact.PackageArtifact{} // TODO: Return this somehow
|
||||||
var lastHash string
|
|
||||||
depsN := 0
|
depsN := 0
|
||||||
currentN := 0
|
currentN := 0
|
||||||
|
|
||||||
@ -845,8 +807,6 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
|
|||||||
Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
|
Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
|
||||||
|
|
||||||
compileSpec.SetOutputPath(p.GetOutputPath())
|
compileSpec.SetOutputPath(p.GetOutputPath())
|
||||||
Debug(pkgTag, " :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash)
|
|
||||||
Debug(pkgTag, " :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
|
|
||||||
|
|
||||||
bus.Manager.Publish(bus.EventPackagePreBuild, struct {
|
bus.Manager.Publish(bus.EventPackagePreBuild, struct {
|
||||||
CompileSpec *compilerspec.LuetCompilationSpec
|
CompileSpec *compilerspec.LuetCompilationSpec
|
||||||
@ -856,29 +816,43 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
|
|||||||
Assert: assertion,
|
Assert: assertion,
|
||||||
})
|
})
|
||||||
|
|
||||||
lastHash = assertion.Hash.PackageHash
|
buildHash, err := packageHashTree.DependencyBuildImage(assertion.Package)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed looking for dependency in hashtree")
|
||||||
|
}
|
||||||
|
|
||||||
|
Debug(pkgTag, " :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash)
|
||||||
|
Debug(pkgTag, " :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
|
||||||
|
|
||||||
|
var sourceImage string
|
||||||
|
|
||||||
|
if compileSpec.GetImage() != "" {
|
||||||
|
Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
|
||||||
|
sourceImage = compileSpec.GetImage()
|
||||||
|
} else {
|
||||||
// for the source instead, pick an image and a buildertaggedImage from hashes if they exists.
|
// for the source instead, pick an image and a buildertaggedImage from hashes if they exists.
|
||||||
// otherways fallback to the pushed repo
|
// otherways fallback to the pushed repo
|
||||||
// Resolve images from the hashtree
|
// Resolve images from the hashtree
|
||||||
resolvedBuildImage := cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec)
|
sourceImage = cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec)
|
||||||
if compileSpec.GetImage() != "" {
|
|
||||||
Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
|
|
||||||
|
|
||||||
a, err := cs.compileWithImage(compileSpec.GetImage(), assertion.Hash.BuildHash, assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
|
|
||||||
}
|
|
||||||
departifacts = append(departifacts, a)
|
|
||||||
Info(pkgTag, ":white_check_mark: Done")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
|
Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
|
||||||
a, err := cs.compileWithImage(resolvedBuildImage, cs.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash), assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps)
|
}
|
||||||
|
|
||||||
|
a, err := cs.compileWithImage(
|
||||||
|
sourceImage,
|
||||||
|
buildHash,
|
||||||
|
assertion.Hash.PackageHash,
|
||||||
|
concurrency,
|
||||||
|
keepPermissions,
|
||||||
|
cs.Options.KeepImg,
|
||||||
|
compileSpec,
|
||||||
|
packageDeps,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
|
return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Info(pkgTag, ":white_check_mark: Done")
|
||||||
|
|
||||||
bus.Manager.Publish(bus.EventPackagePostBuild, struct {
|
bus.Manager.Publish(bus.EventPackagePostBuild, struct {
|
||||||
CompileSpec *compilerspec.LuetCompilationSpec
|
CompileSpec *compilerspec.LuetCompilationSpec
|
||||||
Artifact *artifact.PackageArtifact
|
Artifact *artifact.PackageArtifact
|
||||||
@ -888,18 +862,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
|
|||||||
})
|
})
|
||||||
|
|
||||||
departifacts = append(departifacts, a)
|
departifacts = append(departifacts, a)
|
||||||
Info(pkgTag, ":white_check_mark: Done")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
} else if len(dependencies) > 0 {
|
|
||||||
lastHash = dependencies[len(dependencies)-1].Hash.PackageHash
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if buildTarget {
|
if buildTarget {
|
||||||
resolvedBuildImage := cs.resolveExistingImageHash(lastHash, p)
|
resolvedSourceImage := cs.resolveExistingImageHash(packageHashTree.SourceHash, p)
|
||||||
Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
|
Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
|
||||||
Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedBuildImage)
|
Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedSourceImage)
|
||||||
a, err := cs.compileWithImage(resolvedBuildImage, cs.genBuilderImageTag(p, targetAssertion.Hash.PackageHash), targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
|
a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return a, err
|
return a, err
|
||||||
}
|
}
|
||||||
|
126
pkg/compiler/imagehashtree.go
Normal file
126
pkg/compiler/imagehashtree.go
Normal file
@ -0,0 +1,126 @@
|
|||||||
|
// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License along
|
||||||
|
// with this program; if not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package compiler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
|
||||||
|
"github.com/mudler/luet/pkg/config"
|
||||||
|
pkg "github.com/mudler/luet/pkg/package"
|
||||||
|
"github.com/mudler/luet/pkg/solver"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ImageHashTree struct {
|
||||||
|
Database pkg.PackageDatabase
|
||||||
|
SolverOptions config.LuetSolverOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
type PackageImageHashTree struct {
|
||||||
|
Target *solver.PackageAssert
|
||||||
|
Dependencies solver.PackagesAssertions
|
||||||
|
Solution solver.PackagesAssertions
|
||||||
|
dependencyBuilderImageHashes map[string]string
|
||||||
|
SourceHash string
|
||||||
|
BuilderImageHash string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewHashTree(db pkg.PackageDatabase) *ImageHashTree {
|
||||||
|
return &ImageHashTree{
|
||||||
|
Database: db,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ht *PackageImageHashTree) DependencyBuildImage(p pkg.Package) (string, error) {
|
||||||
|
found, ok := ht.dependencyBuilderImageHashes[p.GetFingerPrint()]
|
||||||
|
if !ok {
|
||||||
|
return "", errors.New("package hash not found")
|
||||||
|
}
|
||||||
|
return found, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: ___ When computing the hash per package (and evaluating the sat solver solution tree part)
|
||||||
|
// we should use the hash of each package + its fingerprint instead as a salt.
|
||||||
|
// That's because the hash will be salted with its `build.yaml`.
|
||||||
|
// In this way, we trigger recompilations if some dep of a target changes
|
||||||
|
// a build.yaml, without touching the version
|
||||||
|
func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) {
|
||||||
|
assertions, err := ht.resolve(cs, p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
targetAssertion := assertions.Search(p.GetPackage().GetFingerPrint())
|
||||||
|
|
||||||
|
dependencies := assertions.Drop(p.GetPackage())
|
||||||
|
var sourceHash string
|
||||||
|
imageHashes := map[string]string{}
|
||||||
|
for _, assertion := range dependencies {
|
||||||
|
var depbuildImageTag string
|
||||||
|
compileSpec, err := cs.FromPackage(assertion.Package)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
|
||||||
|
}
|
||||||
|
if compileSpec.GetImage() != "" {
|
||||||
|
depbuildImageTag = assertion.Hash.BuildHash
|
||||||
|
} else {
|
||||||
|
depbuildImageTag = ht.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash)
|
||||||
|
}
|
||||||
|
imageHashes[assertion.Package.GetFingerPrint()] = depbuildImageTag
|
||||||
|
sourceHash = assertion.Hash.PackageHash
|
||||||
|
}
|
||||||
|
|
||||||
|
return &PackageImageHashTree{
|
||||||
|
Dependencies: dependencies,
|
||||||
|
Target: targetAssertion,
|
||||||
|
SourceHash: sourceHash,
|
||||||
|
BuilderImageHash: ht.genBuilderImageTag(p, targetAssertion.Hash.PackageHash),
|
||||||
|
dependencyBuilderImageHashes: imageHashes,
|
||||||
|
Solution: assertions,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
|
||||||
|
// Use packageImage as salt into the fp being used
|
||||||
|
// so the hash is unique also in cases where
|
||||||
|
// some package deps does have completely different
|
||||||
|
// depgraphs
|
||||||
|
return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(packageImage))
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolve computes the dependency tree of a compilation spec and returns solver assertions
|
||||||
|
// in order to be able to compile the spec.
|
||||||
|
func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
|
||||||
|
dependencies, err := cs.ComputeDepTree(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
|
||||||
|
}
|
||||||
|
|
||||||
|
assertions := solver.PackagesAssertions{}
|
||||||
|
for _, assertion := range dependencies { //highly dependent on the order
|
||||||
|
if assertion.Value {
|
||||||
|
nthsolution := dependencies.Cut(assertion.Package)
|
||||||
|
assertion.Hash = solver.PackageHash{
|
||||||
|
BuildHash: nthsolution.HashFrom(assertion.Package),
|
||||||
|
PackageHash: nthsolution.AssertionHash(),
|
||||||
|
}
|
||||||
|
assertion.Package.SetTreeDir(p.Package.GetTreeDir())
|
||||||
|
assertions = append(assertions, assertion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return assertions, nil
|
||||||
|
}
|
94
pkg/compiler/imagehashtree_test.go
Normal file
94
pkg/compiler/imagehashtree_test.go
Normal file
@ -0,0 +1,94 @@
|
|||||||
|
// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License along
|
||||||
|
// with this program; if not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package compiler_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
. "github.com/mudler/luet/pkg/compiler"
|
||||||
|
sd "github.com/mudler/luet/pkg/compiler/backend"
|
||||||
|
"github.com/mudler/luet/pkg/compiler/types/options"
|
||||||
|
pkg "github.com/mudler/luet/pkg/package"
|
||||||
|
"github.com/mudler/luet/pkg/tree"
|
||||||
|
. "github.com/onsi/ginkgo"
|
||||||
|
. "github.com/onsi/gomega"
|
||||||
|
)
|
||||||
|
|
||||||
|
var _ = Describe("ImageHashTree", func() {
|
||||||
|
generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
|
||||||
|
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
|
||||||
|
hashtree := NewHashTree(generalRecipe.GetDatabase())
|
||||||
|
Context("Simple package definition", func() {
|
||||||
|
BeforeEach(func() {
|
||||||
|
generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
|
||||||
|
err := generalRecipe.Load("../../tests/fixtures/buildable")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
|
||||||
|
hashtree = NewHashTree(generalRecipe.GetDatabase())
|
||||||
|
|
||||||
|
})
|
||||||
|
|
||||||
|
It("Calculates the hash correctly", func() {
|
||||||
|
|
||||||
|
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
|
||||||
|
packageHash, err := hashtree.Query(compiler, spec)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(packageHash.Target.Hash.BuildHash).To(Equal("6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"))
|
||||||
|
Expect(packageHash.Target.Hash.PackageHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
|
||||||
|
Expect(packageHash.BuilderImageHash).To(Equal("builder-79462b60bf899ad79db63f194a3c9c2a"))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
Context("complex package definition", func() {
|
||||||
|
BeforeEach(func() {
|
||||||
|
generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
|
||||||
|
|
||||||
|
err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
|
||||||
|
hashtree = NewHashTree(generalRecipe.GetDatabase())
|
||||||
|
|
||||||
|
})
|
||||||
|
It("Calculates the hash correctly", func() {
|
||||||
|
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
|
||||||
|
packageHash, err := hashtree.Query(compiler, spec)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
|
||||||
|
Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
|
||||||
|
Expect(packageHash.SourceHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
|
||||||
|
Expect(packageHash.BuilderImageHash).To(Equal("builder-37f4d05ba8a39525742ca364f69b4090"))
|
||||||
|
|
||||||
|
//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
|
||||||
|
Expect(packageHash.Target.Hash.PackageHash).To(Equal("bb1d9a99c0c309a297c75b436504e664a42121fadbb4e035bda403cd418117aa"))
|
||||||
|
a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
|
||||||
|
hash, err := packageHash.DependencyBuildImage(a)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(hash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
|
||||||
|
|
||||||
|
assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
|
||||||
|
Expect(assertionA.Hash.PackageHash).To(Equal("c46e653125d71ee3fd696b3941ec1ed6e8a0268f896204c7a222a5aa03eb9982"))
|
||||||
|
b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
|
||||||
|
assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
|
||||||
|
Expect(assertionB.Hash.PackageHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
|
||||||
|
hashB, err := packageHash.DependencyBuildImage(b)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(hashB).To(Equal("6490e800fe443b99328fc363529aee74bda513930fb27ce6ab814d692bba068e"))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
})
|
Loading…
Reference in New Issue
Block a user