kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-09-03 00:06:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixup on values interpolation and metadata retrieve

Browse Source 
- Fixup search path on metadata spec load. Previously we were reading
  the package being passed, and not the one resolved (it failed against
selectors)
- Do inherit first pushrepositories, so they take precedence over pull
- Add test cases to cover build values interpolation by remote
  repositories
- Enhance test cases to check image cache repository inheritance when
  --from-repositories is passed
- Fix race condition when inheriting buildspec options: Instead of consuming the compiler one, annotate the updates in the
package BuildOption spec which is passed by
- Update vendor
This commit is contained in:
Ettore Di Giacinto
2021-04-21 17:59:56 +02:00
parent 788b889d14
commit c9aed37fa7
9 changed files with 265 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
pkg/compiler/compiler.go
View File

@@ -518,10 +518,10 @@ func oneOfImagesAvailable(images []string, b CompilerBackend) (bool, string) {
return false, ""
}
func (cs *LuetCompiler) resolveExistingImageHash(imageHash string) string {
func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
var resolvedImage string
toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)},
genImageList(cs.Options.PullImageRepository, imageHash)...)
genImageList(p.BuildOptions.PullImageRepository, imageHash)...)
if exists, which := oneOfImagesExists(toChecklist, cs.Backend); exists {
resolvedImage = which
}
@@ -557,7 +557,7 @@ func (cs *LuetCompiler) getImageArtifact(hash string, p *compilerspec.LuetCompil
// we return a full artifact if can be loaded locally.
toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, hash)},
genImageList(cs.Options.PullImageRepository, hash)...)
genImageList(p.BuildOptions.PullImageRepository, hash)...)
exists, _ := oneOfImagesExists(toChecklist, cs.Backend)
if art, err := LoadArtifactFromYaml(p); err == nil && exists { // If YAML is correctly loaded, and both images exists, no reason to rebuild.
@@ -698,6 +698,7 @@ func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (sol
BuildHash: nthsolution.HashFrom(assertion.Package),
PackageHash: nthsolution.AssertionHash(),
}
assertion.Package.SetTreeDir(p.Package.GetTreeDir())
assertions = append(assertions, assertion)
}
}
@@ -710,7 +711,7 @@ func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (sol
func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
asserts, err := cs.ComputeDepTree(p)
if err != nil {
panic(err)
return nil, err
}
p.SetSourceAssertion(asserts)
return cs.compile(cs.Options.Concurrency, keepPermissions, p)
@@ -725,20 +726,28 @@ func genImageList(refs []string, hash string) []string {
}
func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationSpec) {
if len(p.BuildOptions.PullImageRepository) != 0 {
cs.Options.PullImageRepository = append(cs.Options.PullImageRepository, p.BuildOptions.PullImageRepository...)
}
Debug("Build options before inherit", p.BuildOptions)
// Append push repositories from buildpsec buildoptions as pull if found.
// This allows to resolve the hash automatically if we pulled the metadata from
// repositories that are advertizing their cache.
if len(p.BuildOptions.PushImageRepository) != 0 {
cs.Options.PullImageRepository = append(cs.Options.PullImageRepository, p.BuildOptions.PushImageRepository)
p.BuildOptions.PullImageRepository = append(p.BuildOptions.PullImageRepository, p.BuildOptions.PushImageRepository)
Debug("Inheriting pull repository from PushImageRepository buildoptions", p.BuildOptions.PullImageRepository)
}
if len(cs.Options.PullImageRepository) != 0 {
p.BuildOptions.PullImageRepository = append(p.BuildOptions.PullImageRepository, cs.Options.PullImageRepository...)
Debug("Inheriting pull repository from PullImageRepository buildoptions", p.BuildOptions.PullImageRepository)
}
Debug("Build options after inherit", p.BuildOptions)
}
func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
// TODO: Racy, remove it
// Inherit build options from compilation specs metadata
orig := cs.Options.PullImageRepository
defer func() { cs.Options.PullImageRepository = orig }()
cs.inheritSpecBuildOptions(p)
// orig := cs.Options.PullImageRepository
// defer func() { cs.Options.PullImageRepository = orig }()
Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
@@ -762,7 +771,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
})
// Update compilespec build options - it will be then serialized into the compilation metadata file
p.SetBuildOptions(cs.Options)
//p.SetBuildOptions(cs.Options)
cs.inheritSpecBuildOptions(p)
// - If image is set we just generate a plain dockerfile
// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
@@ -782,7 +792,10 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
currentN := 0
packageDeps := !cs.Options.PackageTargetOnly
if !cs.Options.NoDeps {
buildDeps := !cs.Options.NoDeps
buildTarget := !cs.Options.OnlyDeps
if buildDeps {
Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
for _, assertion := range dependencies { //highly dependent on the order
depsN++
@@ -797,6 +810,9 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
if err != nil {
return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
}
compileSpec.BuildOptions.PullImageRepository = append(compileSpec.BuildOptions.PullImageRepository, p.BuildOptions.PullImageRepository...)
Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
compileSpec.SetOutputPath(p.GetOutputPath())
Debug(pkgTag, " :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash)
Debug(pkgTag, " :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
@@ -813,7 +829,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
// for the source instead, pick an image and a buildertaggedImage from hashes if they exists.
// otherways fallback to the pushed repo
// Resolve images from the hashtree
resolvedBuildImage := cs.resolveExistingImageHash(assertion.Hash.BuildHash)
resolvedBuildImage := cs.resolveExistingImageHash(assertion.Hash.BuildHash, compileSpec)
if compileSpec.GetImage() != "" {
Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
@@ -848,8 +864,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
lastHash = dependencies[len(dependencies)-1].Hash.PackageHash
}
if !cs.Options.OnlyDeps {
resolvedBuildImage := cs.resolveExistingImageHash(lastHash)
if buildTarget {
resolvedBuildImage := cs.resolveExistingImageHash(lastHash, p)
Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedBuildImage)
a, err := cs.compileWithImage(resolvedBuildImage, "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
@@ -965,7 +981,7 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio
opts := options.Compiler{}
artifactMetadataFile := filepath.Join(p.GetTreeDir(), "..", p.GetMetadataFilePath())
artifactMetadataFile := filepath.Join(pack.GetTreeDir(), "..", pack.GetMetadataFilePath())
Debug("Checking if metadata file is present", artifactMetadataFile)
if _, err := os.Stat(artifactMetadataFile); err == nil {
f, err := os.Open(artifactMetadataFile)
@@ -981,12 +997,10 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio
return nil, errors.Wrap(err, "could not decode package from yaml")
}
Debug("Read build options:", art.CompileSpec.BuildOptions)
opts = art.CompileSpec.BuildOptions
opts.PushImageRepository = ""
Debug("Read build options:", art.CompileSpec.BuildOptions, "from", artifactMetadataFile)
opts = *art.CompileSpec.BuildOptions
} else if !os.IsNotExist(err) {
Debug("error reading already existing artifact metadata file: ", err.Error())
Debug("error reading artifact metadata file: ", err.Error())
} else if os.IsNotExist(err) {
Debug("metadata file not present, skipping", artifactMetadataFile)
}
@@ -1000,7 +1014,9 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio
if err != nil {
return nil, err
}
newSpec.BuildOptions = opts
newSpec.BuildOptions = &opts
cs.inheritSpecBuildOptions(newSpec)
return newSpec, err
}