kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-08-29 04:10:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

⚙️ Enhancements to final images building

Browse Source 
This commit is multi-fold and impacts several areas:

- Don't re-generate final artifact locally if already present while
building with `requires_final_images`.
- Expose to CLI a way to build final images without pushing them.
- The packages listed with `requires_final_images` now are evaluated by
  the solver so the full deptree is took into account

Fixes: https://github.com/mudler/luet/issues/294
This commit is contained in:
Ettore Di Giacinto 2022-04-13 18:04:50 +02:00
parent 18881c3283
commit c6170fabd6
No known key found for this signature in database
GPG Key ID: 965A712536341999
5 changed files with 251 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/build.go
View File

@ -114,15 +114,18 @@ Build packages specifying multiple definition trees:
pushFinalImages, _ := cmd.Flags().GetBool("push-final-images")
pushFinalImagesRepository, _ := cmd.Flags().GetString("push-final-images-repository")
pushFinalImagesForce, _ := cmd.Flags().GetBool("push-final-images-force")
generateImages, _ := cmd.Flags().GetBool("generate-final-images")
var results Results
backendArgs := viper.GetStringSlice("backend-args")
out, _ := cmd.Flags().GetString("output")
pretend, _ := cmd.Flags().GetBool("pretend")
fromRepo, _ := cmd.Flags().GetBool("from-repositories")
compilerSpecs := compilerspec.NewLuetCompilationspecs()
var db types.PackageDatabase
var results Results
var templateFolders []string
compilerBackend, err := compiler.NewBackend(util.DefaultContext, backendType)
helpers.CheckErr(err)
@ -136,7 +139,6 @@ Build packages specifying multiple definition trees:
util.DefaultContext.Info("Loading tree", src)
helpers.CheckErr(generalRecipe.Load(src))
}
templateFolders := []string{}
if fromRepo {
bt, err := installer.LoadBuildTree(generalRecipe, db, util.DefaultContext)
@ -191,6 +193,10 @@ Build packages specifying multiple definition trees:
}
}
if generateImages {
compileropts = append(compileropts, options.EnableGenerateFinalImages)
}
luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), compileropts...)
if full {
@ -238,7 +244,7 @@ Build packages specifying multiple definition trees:
artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
} else if pretend {
toCalculate := []*compilerspec.LuetCompilationSpec{}
var toCalculate []*compilerspec.LuetCompilationSpec
if full {
var err error
toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...)
@ -316,6 +322,7 @@ func init() {
buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
buildCmd.Flags().Bool("generate-final-images", false, "Generate final images while building")
buildCmd.Flags().Bool("push-final-images", false, "Push final images while building")
buildCmd.Flags().Bool("push-final-images-force", false, "Override existing images")
buildCmd.Flags().String("push-final-images-repository", "", "Repository where to push final images to")

4
docs/content/en/docs/Concepts/Packages/specfile.md
View File

@ -160,7 +160,7 @@ The above keywords cannot be present in the same spec **at the same time**, or t
When specifying `requires_final_images: true` luet builds an artifact for each of the packages listed from their compilation specs and it will later *squash* them together in a new container image which is then used in the build process to create an artifact.
The key difference is about *where* your build is going to run from. By specifying `requires_final_images` it will be constructed a new image with the content of each package - while if setting it to false, it will order the images appropriately and link them together with the Dockerfile `FROM` field. That allows to reuse the same images used to build the packages in the require section - or - create a new one from the result of each package compilation.
The key difference is about *where* your build is going to run from. By specifying `requires_final_images` it will be constructed a new image with the content of each package specified and its dependencies - while if setting it to false, it will order the images appropriately and link them together with the Dockerfile `FROM` field. That allows to reuse the same images used to build the packages in the require section - or - create a new one from the result of each package compilation.
## Keywords
@ -365,7 +365,7 @@ _since luet>=0.17.0_
(optional) A boolean flag which instruct luet to use the final images in the `requires` field.
By setting `requires_final_images: true` in the compilation spec, packages in the `requires` section will be first compiled, and afterwards the final packages are squashed together in a new image that will be used during build.
By setting `requires_final_images: true` in the compilation spec, packages in the `requires` section and its dependencies will be fetched if available or compiled, and afterwards the result is squashed together in a new image that will be used as a source of the build process of the package.
```yaml
requires:

170
pkg/compiler/compiler.go
View File

@ -481,11 +481,10 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
return a, errors.Wrap(err, "Failed while writing metadata file")
}
cs.Options.Context.Success(pkgTag, " :white_check_mark: done (empty virtual package)")
if cs.Options.PushFinalImages {
if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil {
return nil, err
}
if err := cs.finalizeImages(a, p, keepPermissions); err != nil {
return nil, err
}
return a, nil
}
@ -519,20 +518,49 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
}
cs.Options.Context.Success(pkgTag, " :white_check_mark: Done building")
if cs.Options.PushFinalImages {
if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil {
return nil, err
}
if err := cs.finalizeImages(a, p, keepPermissions); err != nil {
return nil, err
}
return a, nil
}
// TODO: A small readaptation of repository_docker.go pushImageFromArtifact()
// Move this to a common place
func (cs *LuetCompiler) pushFinalArtifact(a *artifact.PackageArtifact, p *compilerspec.LuetCompilationSpec, keepPermissions bool) error {
cs.Options.Context.Info("Pushing final image for", a.CompileSpec.Package.HumanReadableString())
// finalizeImages finalizes images and generates final artifacts (push them as well if necessary).
func (cs *LuetCompiler) finalizeImages(a *artifact.PackageArtifact, p *compilerspec.LuetCompilationSpec, keepPermissions bool) error {
// TODO: This is a small readaptation of repository_docker.go pushImageFromArtifact().
// Maybe can be moved to a common place.
// We either check if finalization is needed
// and push or generate final images here, anything else we just return successfully
if !cs.Options.PushFinalImages && !cs.Options.GenerateFinalImages {
return nil
}
imageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, a.CompileSpec.Package.ImageID())
metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, helpers.SanitizeImageString(a.CompileSpec.GetPackage().GetMetadataFilePath()))
// Do generate image only, might be required for local iteration without pushing to remote repository
if cs.Options.GenerateFinalImages && !cs.Options.PushFinalImages {
cs.Options.Context.Info("Generating final image for", a.CompileSpec.Package.HumanReadableString())
if err := a.GenerateFinalImage(cs.Options.Context, imageID, cs.GetBackend(), true); err != nil {
return errors.Wrap(err, "while creating final image")
}
a := artifact.NewPackageArtifact(filepath.Join(p.GetOutputPath(), a.CompileSpec.GetPackage().GetMetadataFilePath()))
metadataArchive, err := artifact.CreateArtifactForFile(cs.Options.Context, a.Path)
if err != nil {
return errors.Wrap(err, "failed generating checksums for tree")
}
if err := metadataArchive.GenerateFinalImage(cs.Options.Context, metadataImageID, cs.Backend, keepPermissions); err != nil {
return errors.Wrap(err, "Failed generating metadata tree "+metadataImageID)
}
return nil
}
cs.Options.Context.Info("Pushing final image for", a.CompileSpec.Package.HumanReadableString())
// First push the package image
if !cs.Backend.ImageAvailable(imageID) || cs.Options.PushFinalImagesForce {
@ -547,7 +575,6 @@ func (cs *LuetCompiler) pushFinalArtifact(a *artifact.PackageArtifact, p *compil
}
// Then the image ID
metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, helpers.SanitizeImageString(a.CompileSpec.GetPackage().GetMetadataFilePath()))
if !cs.Backend.ImageAvailable(metadataImageID) || cs.Options.PushFinalImagesForce {
cs.Options.Context.Info("Generating metadata image for", a.CompileSpec.Package.HumanReadableString(), metadataImageID)
@ -604,6 +631,10 @@ func (cs *LuetCompiler) findImageHash(imageHash string, p *compilerspec.LuetComp
cs.Options.Context.Debug("Resolving image hash for", p.Package.HumanReadableString(), "hash", imageHash, "Pull repositories", p.BuildOptions.PullImageRepository)
toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)},
genImageList(p.BuildOptions.PullImageRepository, imageHash)...)
if cs.Options.PushFinalImagesRepository != "" {
toChecklist = append(toChecklist, fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, imageHash))
}
if exists, which := oneOfImagesExists(toChecklist, cs.Backend); exists {
resolvedImage = which
}
@ -910,16 +941,40 @@ func (cs *LuetCompiler) getSpecHash(pkgs types.Packages, salt string) (string, e
}
func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
if !p.RequiresFinalImages {
return nil
}
joinTag := ">:loop: final images<"
var fromPackages types.Packages
if p.RequiresFinalImages {
cs.Options.Context.Info(joinTag, "Generating a parent image from final packages")
fromPackages = p.Package.GetRequires()
} else {
// No source image to resolve
return nil
cs.Options.Context.Info(joinTag, "Generating a parent image from final packages")
//fromPackages = p.Package.GetRequires() // (first level only)
pTarget := p
runtime, err := p.Package.GetRuntimePackage()
if err == nil {
spec, err := cs.FromPackage(runtime)
if err == nil {
cs.Options.Context.Info(joinTag, "Using runtime package for deptree computation")
pTarget = spec
}
}
// resolve deptree of runtime of p and use it in fromPackages
t, err := cs.ComputeDepTree(pTarget)
if err != nil {
return errors.Wrap(err, "failed querying hashtree")
}
for _, a := range t {
if !a.Value || a.Package.Matches(p.Package) {
continue
}
fromPackages = append(fromPackages, a.Package)
cs.Options.Context.Infof("Adding dependency '%s'.", a.Package.HumanReadableString())
}
// First compute a hash and check if image is available. if it is, then directly consume that
@ -930,10 +985,9 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
cs.Options.Context.Info(joinTag, "Searching existing image with hash", overallFp)
image := cs.findImageHash(overallFp, p)
if image != "" {
cs.Options.Context.Info("Image already found", image)
p.SetImage(image)
if img := cs.findImageHash(overallFp, p); img != "" {
cs.Options.Context.Info("Image already found", img)
p.SetImage(img)
return nil
}
cs.Options.Context.Info(joinTag, "Image not found. Generating image join with hash ", overallFp)
@ -958,28 +1012,58 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
for _, c := range fromPackages {
current++
if c != nil && c.Name != "" && c.Version != "" {
joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(p.Package.GetRequires()), c.HumanReadableString())
joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(fromPackages), c.HumanReadableString())
cs.Options.Context.Info(joinTag2, "compilation starts")
spec, err := cs.FromPackage(c)
if err != nil {
return errors.Wrap(err, "while generating images to join from")
// Search if we have already a final-image that was already pushed
// for this to work on the same repo, it is required to push final images during build
if img := cs.findImageHash(c.ImageID(), p); cs.Options.PullFirst && img != "" {
cs.Options.Context.Info("Final image already found", img)
if !cs.Backend.ImageExists(img) {
if err := cs.Backend.DownloadImage(backend.Options{ImageName: img}); err != nil {
return errors.Wrap(err, "failed pulling image "+img+" during extraction")
}
}
imgRef, err := cs.Backend.ImageReference(img, true)
if err != nil {
return err
}
ctx := cs.Options.Context.WithLoggingContext(fmt.Sprintf("final image extract %s", img))
_, _, err = image.ExtractTo(
ctx,
imgRef,
joinDir,
nil,
)
if err != nil {
return err
}
} else {
cs.Options.Context.Info("Final image not found for", c.HumanReadableString())
// If no image was found, we have to build it from scratch
cs.Options.Context.Info(joinTag2, "compilation starts")
spec, err := cs.FromPackage(c)
if err != nil {
return errors.Wrap(err, "while generating images to join from")
}
wantsArtifact := true
genDepsArtifact := !cs.Options.PackageTargetOnly
spec.SetOutputPath(p.GetOutputPath())
artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, &genDepsArtifact, spec)
if err != nil {
return errors.Wrap(err, "failed building join image")
}
err = artifact.Unpack(cs.Options.Context, joinDir, keepPermissions)
if err != nil {
return errors.Wrap(err, "failed building join image")
}
cs.Options.Context.Info(joinTag2, ":white_check_mark: Done")
}
wantsArtifact := true
genDepsArtifact := !cs.Options.PackageTargetOnly
spec.SetOutputPath(p.GetOutputPath())
artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, &genDepsArtifact, spec)
if err != nil {
return errors.Wrap(err, "failed building join image")
}
err = artifact.Unpack(cs.Options.Context, joinDir, keepPermissions)
if err != nil {
return errors.Wrap(err, "failed building join image")
}
cs.Options.Context.Info(joinTag2, ":white_check_mark: Done")
}
}

104
pkg/compiler/compiler_test.go
View File

@ -22,6 +22,7 @@ import (
"path/filepath"
"strings"
"github.com/mudler/luet/pkg/api/core/logger"
"github.com/mudler/luet/pkg/api/core/types"
helpers "github.com/mudler/luet/tests/helpers"
@ -1103,4 +1104,107 @@ var _ = Describe("Compiler", func() {
Expect(files).ToNot(ContainElement("bin/busybox"))
})
})
Context("final images", func() {
It("reuses final images", func() {
generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
err := generalRecipe.Load("../../tests/fixtures/join_complex")
Expect(err).ToNot(HaveOccurred())
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(6))
logdir, err := ioutil.TempDir("", "log")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(logdir) // clean up
logPath := filepath.Join(logdir, "logs")
var log string
readLogs := func() {
d, err := ioutil.ReadFile(logPath)
Expect(err).To(BeNil())
log = string(d)
}
l, err := logger.New(
logger.WithFileLogging(
logPath,
"",
),
)
Expect(err).ToNot(HaveOccurred())
c := context.NewContext(
context.WithLogger(l),
)
b := sd.NewSimpleDockerBackend(ctx)
joinImage := "luet/cache:586b36482e3f238c76d3536e7ca12cc4" //resulting join image
allImages := []string{
joinImage,
"test/test:c-test-1.2"}
cleanup := func(imgs ...string) {
// Remove the join hash so we force using final images
for _, toRemove := range imgs {
b.RemoveImage(sd.Options{ImageName: toRemove})
}
}
defer cleanup(allImages...)
compiler := NewLuetCompiler(b, generalRecipe.GetDatabase(),
options.WithFinalRepository("test/test"),
options.EnableGenerateFinalImages,
options.PullFirst(true),
options.WithContext(c))
spec, err := compiler.FromPackage(&types.Package{Name: "x", Category: "test", Version: "0.1"})
Expect(err).ToNot(HaveOccurred())
compiler.Options.CompressionType = compression.GZip
Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
tmpdir, err := ioutil.TempDir("", "tree")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpdir) // clean up
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
readLogs()
Expect(log).To(And(
ContainSubstring("Generating final image for"),
ContainSubstring("Adding dependency"),
ContainSubstring("Final image not found for test/c-1.2"),
))
Expect(log).ToNot(And(
ContainSubstring("Final image already found test/test:c-test-1.2"),
))
os.WriteFile(logPath, []byte{}, os.ModePerm) // cleanup logs
// Remove the join hash so we force using final images
cleanup(joinImage)
//compile again
By("Recompiling")
artifacts, errs = compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
// read logs again
readLogs()
Expect(log).To(And(
ContainSubstring("Final image already found test/test:a-test-1.2"),
))
Expect(log).ToNot(And(
ContainSubstring("build test/c-1.2 compilation starts"),
ContainSubstring("Final image not found for test/c-1.2"),
))
})
})
})

8
pkg/compiler/types/options/compiler_options.go
View File

@ -49,6 +49,9 @@ type Compiler struct {
// Tells wether to push final container images after building
PushFinalImages bool
PushFinalImagesForce bool
GenerateFinalImages bool
// Image repository to push to
PushFinalImagesRepository string
@ -99,6 +102,11 @@ func WithFinalRepository(r string) func(cfg *Compiler) error {
}
}
func EnableGenerateFinalImages(cfg *Compiler) error {
cfg.GenerateFinalImages = true
return nil
}
func EnablePushFinalImages(cfg *Compiler) error {
cfg.PushFinalImages = true
return nil