kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-07-31 23:05:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow to pull from multiple repositories

Browse Source 
Adds a new cli flag to luet build `--pull-repository` which allows to
pass-by a list of docker image references which are used to pull the
cache from

Fixes #185
Fixes #184
Closes #161
This commit is contained in:
Ettore Di Giacinto 2021-04-07 15:54:38 +02:00
parent 88307b1912
commit ec19b34ca8
4 changed files with 169 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/build.go
View File

@ -154,6 +154,7 @@ Build packages specifying multiple definition trees:
discount := LuetCfg.Viper.GetFloat64("solver.discount")
rate := LuetCfg.Viper.GetFloat64("solver.rate")
attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
LuetCfg.GetSolverOptions().Type = stype
LuetCfg.GetSolverOptions().LearnRate = float32(rate)
@ -166,7 +167,8 @@ Build packages specifying multiple definition trees:
opts := compiler.NewDefaultCompilerOptions()
opts.SolverOptions = *LuetCfg.GetSolverOptions()
opts.ImageRepository = imageRepository
opts.PushImageRepository = imageRepository
opts.PullImageRepository = pullRepo
opts.PullFirst = pull
opts.KeepImg = keepImages
opts.Push = push
@ -330,6 +332,7 @@ func init() {
buildCmd.Flags().Bool("live-output", LuetCfg.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
buildCmd.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")
buildCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")

5
cmd/tree/images.go
View File

@ -57,6 +57,7 @@ func NewTreeImageCommand() *cobra.Command {
treePath, _ := cmd.Flags().GetStringArray("tree")
imageRepository := viper.GetString("image-repository")
pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
out, _ := cmd.Flags().GetString("output")
if out != "terminal" {
@ -75,7 +76,8 @@ func NewTreeImageCommand() *cobra.Command {
opts := compiler.NewDefaultCompilerOptions()
opts.SolverOptions = *LuetCfg.GetSolverOptions()
opts.ImageRepository = imageRepository
opts.PushImageRepository = imageRepository
opts.PullImageRepository = pullRepo
solverOpts := solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
luetCompiler := compiler.NewLuetCompiler(compilerBackend, reciper.GetDatabase(), opts, solverOpts)
@ -135,6 +137,7 @@ func NewTreeImageCommand() *cobra.Command {
ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
ans.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
ans.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")
return ans
}

225
pkg/compiler/compiler.go
View File

@ -1,4 +1,4 @@
// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -43,9 +43,11 @@ const CollectionFile = "collection.yaml"
type LuetCompiler struct {
*tree.CompilerRecipe
Backend CompilerBackend
Database pkg.PackageDatabase
ImageRepository string
Backend CompilerBackend
Database pkg.PackageDatabase
PushImageRepository string
PullImageRepository []string
PullFirst, KeepImg, Clean bool
Concurrency int
CompressionType CompressionImplementation
@ -56,28 +58,41 @@ type LuetCompiler struct {
func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions, solvopts solver.Options) Compiler {
// The CompilerRecipe will gives us a tree with only build deps listed.
if len(opt.PullImageRepository) == 0 {
opt.PullImageRepository = []string{opt.PushImageRepository}
}
return &LuetCompiler{
Backend: backend,
CompilerRecipe: &tree.CompilerRecipe{
tree.Recipe{Database: db},
Recipe: tree.Recipe{Database: db},
},
Database: db,
ImageRepository: opt.ImageRepository,
PullFirst: opt.PullFirst,
CompressionType: opt.CompressionType,
KeepImg: opt.KeepImg,
Concurrency: opt.Concurrency,
Options: *opt,
SolverOptions: solvopts,
Database: db,
PushImageRepository: opt.PushImageRepository,
PullImageRepository: opt.PullImageRepository,
PullFirst: opt.PullFirst,
CompressionType: opt.CompressionType,
KeepImg: opt.KeepImg,
Concurrency: opt.Concurrency,
Options: *opt,
SolverOptions: solvopts,
}
}
// SetBackendArgs sets arbitrary backend arguments.
// Those for example can be commands passed to the docker daemon during build phase,
// as build-args, etc.
func (cs *LuetCompiler) SetBackendArgs(args []string) {
cs.BackedArgs = args
}
// SetConcurrency sets the compiler concurrency
func (cs *LuetCompiler) SetConcurrency(i int) {
cs.Concurrency = i
}
// SetCompressionType sets the compiler compression type for resulting artifacts
func (cs *LuetCompiler) SetCompressionType(t CompressionImplementation) {
cs.CompressionType = t
}
@ -97,6 +112,7 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan Co
}
}
// CompileWithReverseDeps compiles the supplied compilationspecs and their reverse dependencies
func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
artifacts, err := cs.CompileParallel(keepPermissions, ps)
if len(err) != 0 {
@ -117,28 +133,6 @@ func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps Compilat
toCompile.Add(spec)
}
// for _, assertion := range a.GetSourceAssertion() {
// if assertion.Value && assertion.Package.Flagged() {
// spec, asserterr := cs.FromPackage(assertion.Package)
// if err != nil {
// return nil, append(err, asserterr)
// }
// w, asserterr := cs.Tree().World()
// if err != nil {
// return nil, append(err, asserterr)
// }
// revdeps := spec.GetPackage().Revdeps(&w)
// for _, r := range revdeps {
// spec, asserterr := cs.FromPackage(r)
// if asserterr != nil {
// return nil, append(err, asserterr)
// }
// spec.SetOutputPath(ps.All()[0].GetOutputPath())
// toCompile.Add(spec)
// }
// }
// }
}
uniques := toCompile.Unique().Remove(ps)
@ -150,6 +144,8 @@ func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps Compilat
return append(artifacts, artifacts2...), err
}
// CompileParallel compiles the supplied compilationspecs in parallel
// to note, no specific heuristic is implemented, and the specs are run in parallel as they are.
func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
all := make(chan CompilationSpec)
artifacts := []Artifact{}
@ -329,13 +325,13 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
fp := p.GetPackage().HashFingerprint(helpers.StripRegistryFromImage(packageImage))
if buildertaggedImage == "" {
buildertaggedImage = cs.ImageRepository + ":builder-" + fp
buildertaggedImage = cs.PushImageRepository + ":builder-" + fp
Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image)
}
// TODO: Cleanup, not actually hit
if packageImage == "" {
packageImage = cs.ImageRepository + ":builder-invalid" + fp
return runnerOpts, builderOpts, errors.New("no package image given")
}
p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
@ -407,11 +403,11 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
}
if buildImage {
if err := cs.Backend.BuildImage(opts); err != nil {
return errors.Wrap(err, "Could not build image: "+image+" "+opts.DockerFileName)
return errors.Wrapf(err, "Could not build image: %s %s", image, opts.DockerFileName)
}
if cs.Options.Push {
if err = cs.Backend.Push(opts); err != nil {
return errors.Wrap(err, "Could not push image: "+image+" "+opts.DockerFileName)
return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
}
}
}
@ -420,7 +416,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
if len(p.GetPreBuildSteps()) != 0 {
Info(pkgTag, ":whale: Generating 'builder' image from", image, "as", buildertaggedImage, "with prelude steps")
if err := buildAndPush(builderOpts); err != nil {
return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
return builderOpts, runnerOpts, errors.Wrapf(err, "Could not push image: %s %s", image, builderOpts.DockerFileName)
}
}
@ -428,7 +424,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
// acting as a docker tag.
Info(pkgTag, ":whale: Generating 'package' image from", buildertaggedImage, "as", packageImage, "with build steps")
if err := buildAndPush(runnerOpts); err != nil {
return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
return builderOpts, runnerOpts, errors.Wrapf(err, "Could not push image: %s %s", image, runnerOpts.DockerFileName)
}
return builderOpts, runnerOpts, nil
@ -501,19 +497,85 @@ func (cs *LuetCompiler) genArtifact(p CompilationSpec, builderOpts, runnerOpts C
return artifact, nil
}
func (cs *LuetCompiler) waitForImage(image string) {
if cs.Options.PullFirst && cs.Options.Wait && !cs.Backend.ImageAvailable(image) {
Info(fmt.Sprintf("Waiting for image %s to be available... :zzz:", image))
Spinner(22)
defer SpinnerStop()
for !cs.Backend.ImageAvailable(image) {
Info(fmt.Sprintf("Image %s not available yet, sleeping", image))
time.Sleep(5 * time.Second)
func (cs *LuetCompiler) waitForImages(images []string) {
if cs.Options.PullFirst && cs.Options.Wait {
available, _ := oneOfImagesAvailable(images, cs.Backend)
if !available {
Info(fmt.Sprintf("Waiting for image %s to be available... :zzz:", images))
Spinner(22)
defer SpinnerStop()
for !available {
available, _ = oneOfImagesAvailable(images, cs.Backend)
Info(fmt.Sprintf("Image %s not available yet, sleeping", images))
time.Sleep(5 * time.Second)
}
}
}
}
func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string,
func oneOfImagesExists(images []string, b CompilerBackend) (bool, string) {
for _, i := range images {
if exists := b.ImageExists(i); exists {
return true, i
}
}
return false, ""
}
func oneOfImagesAvailable(images []string, b CompilerBackend) (bool, string) {
for _, i := range images {
if exists := b.ImageAvailable(i); exists {
return true, i
}
}
return false, ""
}
func (cs *LuetCompiler) resolveExistingImageHash(imageHash string) string {
var resolvedImage string
toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.PushImageRepository, imageHash)},
genImageList(cs.PullImageRepository, imageHash)...)
if exists, which := oneOfImagesExists(toChecklist, cs.Backend); exists {
resolvedImage = which
}
if cs.Options.PullFirst {
if exists, which := oneOfImagesAvailable(toChecklist, cs.Backend); exists {
resolvedImage = which
}
}
if resolvedImage == "" {
resolvedImage = fmt.Sprintf("%s:%s", cs.PushImageRepository, imageHash)
}
return resolvedImage
}
func (cs *LuetCompiler) getImageArtifact(hash string, p CompilationSpec) (Artifact, error) {
// we check if there is an available image with the given hash and
// we return a full artifact if can be loaded locally.
toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.PushImageRepository, hash)},
genImageList(cs.PullImageRepository, hash)...)
exists, _ := oneOfImagesExists(toChecklist, cs.Backend)
if art, err := LoadArtifactFromYaml(p); err == nil && exists { // If YAML is correctly loaded, and both images exists, no reason to rebuild.
Debug("Artifact reloaded from YAML. Skipping build")
return art, nil
}
cs.waitForImages(toChecklist)
available, _ := oneOfImagesAvailable(toChecklist, cs.Backend)
if exists || (cs.Options.PullFirst && available) {
Debug("Image available, returning empty artifact")
return &PackageArtifact{}, nil
}
return nil, errors.New("artifact not found")
}
// compileWithImage compiles a PackageTagHash image using the image source, and tagging an indermediate
// image buildertaggedImage.
// Images that can be resolved from repositories are prefered over the local ones if PullFirst is set to true
// avoiding to rebuild images as much as possible
func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage string, packageTagHash string,
concurrency int,
keepPermissions, keepImg bool,
p CompilationSpec, generateArtifact bool) (Artifact, error) {
@ -526,17 +588,16 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
}
if !generateArtifact {
exists := cs.Backend.ImageExists(packageImage)
if art, err := LoadArtifactFromYaml(p); err == nil && exists { // If YAML is correctly loaded, and both images exists, no reason to rebuild.
Debug("Artifact reloaded from YAML. Skipping build")
return art, err
}
cs.waitForImage(packageImage)
if cs.Options.PullFirst && cs.Backend.ImageAvailable(packageImage) {
return &PackageArtifact{}, nil
// try to avoid regenerating the image if possible by checking the hash in the
// given repositories
// It is best effort. If we fail resolving, we will generate the images and keep going
if art, err := cs.getImageArtifact(packageTagHash, p); err == nil {
return art, nil
}
}
// always going to point at the destination from the repo defined
packageImage := fmt.Sprintf("%s:%s", cs.PushImageRepository, packageTagHash)
builderOpts, runnerOpts, err := cs.buildPackageImage(image, buildertaggedImage, packageImage, concurrency, keepPermissions, p)
if err != nil {
return nil, errors.Wrap(err, "failed building package image")
@ -561,6 +622,8 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
return cs.genArtifact(p, builderOpts, runnerOpts, concurrency, keepPermissions)
}
// FromDatabase returns all the available compilation specs from a database. If the minimum flag is returned
// it will be computed a minimal subset that will guarantees that all packages are built ( if not targeting a single package explictly )
func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) {
compilerSpecs := NewLuetCompilationspecs()
@ -608,6 +671,8 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]Com
return result, nil
}
// ComputeDepTree computes the dependency tree of a compilation spec and returns solver assertions
// in order to be able to compile the spec.
func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
s := solver.NewResolver(cs.SolverOptions, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
@ -637,7 +702,8 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
return assertions, nil
}
// Compile is non-parallel
// Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
// an artifact
func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifact, error) {
asserts, err := cs.ComputeDepTree(p)
if err != nil {
@ -647,6 +713,14 @@ func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifa
return cs.compile(cs.Concurrency, keepPermissions, p)
}
func genImageList(refs []string, hash string) []string {
var res []string
for _, r := range refs {
res = append(res, fmt.Sprintf("%s:%s", r, hash))
}
return res
}
func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
@ -660,7 +734,6 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
}
targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
targetPackageHash := cs.ImageRepository + ":" + targetAssertion.Hash.PackageHash
bus.Manager.Publish(bus.EventPackagePreBuild, struct {
CompileSpec CompilationSpec
@ -673,7 +746,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
// - If image is set we just generate a plain dockerfile
// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
if p.GetImage() != "" {
return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
return cs.compileWithImage(p.GetImage(), "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
}
// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@ -704,11 +777,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
}
compileSpec.SetOutputPath(p.GetOutputPath())
buildImageHash := cs.ImageRepository + ":" + assertion.Hash.BuildHash
currentPackageImageHash := cs.ImageRepository + ":" + assertion.Hash.PackageHash
Debug(pkgTag, " :arrow_right_hook: :whale: Builder image from", buildImageHash)
Debug(pkgTag, " :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
Debug(pkgTag, " :arrow_right_hook: :whale: Builder image from hash", assertion.Hash.BuildHash)
Debug(pkgTag, " :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
bus.Manager.Publish(bus.EventPackagePreBuild, struct {
CompileSpec CompilationSpec
@ -718,10 +788,15 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
Assert: assertion,
})
lastHash = currentPackageImageHash
lastHash = assertion.Hash.PackageHash
// for the source instead, pick an image and a buildertaggedImage from hashes if they exists.
// otherways fallback to the pushed repo
// Resolve images from the hashtree
resolvedBuildImage := cs.resolveExistingImageHash(assertion.Hash.BuildHash)
if compileSpec.GetImage() != "" {
Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
artifact, err := cs.compileWithImage(compileSpec.GetImage(), resolvedBuildImage, assertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
if err != nil {
return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
}
@ -731,11 +806,9 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
}
Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
artifact, err := cs.compileWithImage(resolvedBuildImage, "", assertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
if err != nil {
return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
// deperrs = append(deperrs, err)
// break // stop at first error
}
bus.Manager.Publish(bus.EventPackagePostBuild, struct {
@ -751,13 +824,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
}
} else if len(dependencies) > 0 {
lastHash = cs.ImageRepository + ":" + dependencies[len(dependencies)-1].Hash.PackageHash
lastHash = dependencies[len(dependencies)-1].Hash.PackageHash
}
if !cs.Options.OnlyDeps {
resolvedBuildImage := cs.resolveExistingImageHash(lastHash)
Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", lastHash)
artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedBuildImage)
artifact, err := cs.compileWithImage(resolvedBuildImage, "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
if err != nil {
return artifact, err
}
@ -780,6 +854,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
type templatedata map[string]interface{}
// FromPackage returns a compilation spec from a package definition
func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
pack, err := cs.Database.FindPackageCandidate(p)
@ -834,10 +909,12 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
return NewLuetCompilationSpec(dataresult, pack)
}
// GetBackend returns the current compilation backend
func (cs *LuetCompiler) GetBackend() CompilerBackend {
return cs.Backend
}
// SetBackend sets the compilation backend
func (cs *LuetCompiler) SetBackend(b CompilerBackend) {
cs.Backend = b
}

19
pkg/compiler/interface.go
View File

@ -49,7 +49,8 @@ type CompilerBackendOptions struct {
}
type CompilerOptions struct {
ImageRepository string
PushImageRepository string
PullImageRepository []string
PullFirst, KeepImg, Push bool
Concurrency int
CompressionType CompressionImplementation
@ -65,14 +66,14 @@ type CompilerOptions struct {
func NewDefaultCompilerOptions() *CompilerOptions {
return &CompilerOptions{
ImageRepository: "luet/cache",
PullFirst: false,
Push: false,
CompressionType: None,
KeepImg: true,
Concurrency: runtime.NumCPU(),
OnlyDeps: false,
NoDeps: false,
PushImageRepository: "luet/cache",
PullFirst: false,
Push: false,
CompressionType: None,
KeepImg: true,
Concurrency: runtime.NumCPU(),
OnlyDeps: false,
NoDeps: false,
}
}