From c6170fabd6ac31fd4cade0a88e7c0591fbbc03b6 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 13 Apr 2022 18:04:50 +0200 Subject: [PATCH] :gear: Enhancements to final images building This commit is multi-fold and impacts several areas: - Don't re-generate final artifact locally if already present while building with `requires_final_images`. - Expose to CLI a way to build final images without pushing them. - The packages listed with `requires_final_images` now are evaluated by the solver so the full deptree is took into account Fixes: https://github.com/mudler/luet/issues/294 --- cmd/build.go | 13 +- .../en/docs/Concepts/Packages/specfile.md | 4 +- pkg/compiler/compiler.go | 170 +++++++++++++----- pkg/compiler/compiler_test.go | 104 +++++++++++ .../types/options/compiler_options.go | 8 + 5 files changed, 251 insertions(+), 48 deletions(-) diff --git a/cmd/build.go b/cmd/build.go index dde47813..0e82f6e1 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -114,15 +114,18 @@ Build packages specifying multiple definition trees: pushFinalImages, _ := cmd.Flags().GetBool("push-final-images") pushFinalImagesRepository, _ := cmd.Flags().GetString("push-final-images-repository") pushFinalImagesForce, _ := cmd.Flags().GetBool("push-final-images-force") + generateImages, _ := cmd.Flags().GetBool("generate-final-images") - var results Results backendArgs := viper.GetStringSlice("backend-args") out, _ := cmd.Flags().GetString("output") pretend, _ := cmd.Flags().GetBool("pretend") fromRepo, _ := cmd.Flags().GetBool("from-repositories") compilerSpecs := compilerspec.NewLuetCompilationspecs() + var db types.PackageDatabase + var results Results + var templateFolders []string compilerBackend, err := compiler.NewBackend(util.DefaultContext, backendType) helpers.CheckErr(err) @@ -136,7 +139,6 @@ Build packages specifying multiple definition trees: util.DefaultContext.Info("Loading tree", src) helpers.CheckErr(generalRecipe.Load(src)) } - templateFolders := []string{} if fromRepo { bt, err := installer.LoadBuildTree(generalRecipe, db, util.DefaultContext) @@ -191,6 +193,10 @@ Build packages specifying multiple definition trees: } } + if generateImages { + compileropts = append(compileropts, options.EnableGenerateFinalImages) + } + luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), compileropts...) if full { @@ -238,7 +244,7 @@ Build packages specifying multiple definition trees: artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs) } else if pretend { - toCalculate := []*compilerspec.LuetCompilationSpec{} + var toCalculate []*compilerspec.LuetCompilationSpec if full { var err error toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...) @@ -316,6 +322,7 @@ func init() { buildCmd.Flags().Bool("revdeps", false, "Build with revdeps") buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree") + buildCmd.Flags().Bool("generate-final-images", false, "Generate final images while building") buildCmd.Flags().Bool("push-final-images", false, "Push final images while building") buildCmd.Flags().Bool("push-final-images-force", false, "Override existing images") buildCmd.Flags().String("push-final-images-repository", "", "Repository where to push final images to") diff --git a/docs/content/en/docs/Concepts/Packages/specfile.md b/docs/content/en/docs/Concepts/Packages/specfile.md index a07b7e0a..f1bb817a 100644 --- a/docs/content/en/docs/Concepts/Packages/specfile.md +++ b/docs/content/en/docs/Concepts/Packages/specfile.md @@ -160,7 +160,7 @@ The above keywords cannot be present in the same spec **at the same time**, or t When specifying `requires_final_images: true` luet builds an artifact for each of the packages listed from their compilation specs and it will later *squash* them together in a new container image which is then used in the build process to create an artifact. -The key difference is about *where* your build is going to run from. By specifying `requires_final_images` it will be constructed a new image with the content of each package - while if setting it to false, it will order the images appropriately and link them together with the Dockerfile `FROM` field. That allows to reuse the same images used to build the packages in the require section - or - create a new one from the result of each package compilation. +The key difference is about *where* your build is going to run from. By specifying `requires_final_images` it will be constructed a new image with the content of each package specified and its dependencies - while if setting it to false, it will order the images appropriately and link them together with the Dockerfile `FROM` field. That allows to reuse the same images used to build the packages in the require section - or - create a new one from the result of each package compilation. ## Keywords @@ -365,7 +365,7 @@ _since luet>=0.17.0_ (optional) A boolean flag which instruct luet to use the final images in the `requires` field. -By setting `requires_final_images: true` in the compilation spec, packages in the `requires` section will be first compiled, and afterwards the final packages are squashed together in a new image that will be used during build. +By setting `requires_final_images: true` in the compilation spec, packages in the `requires` section and its dependencies will be fetched if available or compiled, and afterwards the result is squashed together in a new image that will be used as a source of the build process of the package. ```yaml requires: diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 7536e54d..139bb6ce 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -481,11 +481,10 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder return a, errors.Wrap(err, "Failed while writing metadata file") } cs.Options.Context.Success(pkgTag, " :white_check_mark: done (empty virtual package)") - if cs.Options.PushFinalImages { - if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil { - return nil, err - } + if err := cs.finalizeImages(a, p, keepPermissions); err != nil { + return nil, err } + return a, nil } @@ -519,20 +518,49 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder } cs.Options.Context.Success(pkgTag, " :white_check_mark: Done building") - if cs.Options.PushFinalImages { - if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil { - return nil, err - } + if err := cs.finalizeImages(a, p, keepPermissions); err != nil { + return nil, err } return a, nil } -// TODO: A small readaptation of repository_docker.go pushImageFromArtifact() -// Move this to a common place -func (cs *LuetCompiler) pushFinalArtifact(a *artifact.PackageArtifact, p *compilerspec.LuetCompilationSpec, keepPermissions bool) error { - cs.Options.Context.Info("Pushing final image for", a.CompileSpec.Package.HumanReadableString()) +// finalizeImages finalizes images and generates final artifacts (push them as well if necessary). +func (cs *LuetCompiler) finalizeImages(a *artifact.PackageArtifact, p *compilerspec.LuetCompilationSpec, keepPermissions bool) error { + + // TODO: This is a small readaptation of repository_docker.go pushImageFromArtifact(). + // Maybe can be moved to a common place. + + // We either check if finalization is needed + // and push or generate final images here, anything else we just return successfully + if !cs.Options.PushFinalImages && !cs.Options.GenerateFinalImages { + return nil + } + imageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, a.CompileSpec.Package.ImageID()) + metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, helpers.SanitizeImageString(a.CompileSpec.GetPackage().GetMetadataFilePath())) + + // Do generate image only, might be required for local iteration without pushing to remote repository + if cs.Options.GenerateFinalImages && !cs.Options.PushFinalImages { + cs.Options.Context.Info("Generating final image for", a.CompileSpec.Package.HumanReadableString()) + + if err := a.GenerateFinalImage(cs.Options.Context, imageID, cs.GetBackend(), true); err != nil { + return errors.Wrap(err, "while creating final image") + } + + a := artifact.NewPackageArtifact(filepath.Join(p.GetOutputPath(), a.CompileSpec.GetPackage().GetMetadataFilePath())) + metadataArchive, err := artifact.CreateArtifactForFile(cs.Options.Context, a.Path) + if err != nil { + return errors.Wrap(err, "failed generating checksums for tree") + } + if err := metadataArchive.GenerateFinalImage(cs.Options.Context, metadataImageID, cs.Backend, keepPermissions); err != nil { + return errors.Wrap(err, "Failed generating metadata tree "+metadataImageID) + } + + return nil + } + + cs.Options.Context.Info("Pushing final image for", a.CompileSpec.Package.HumanReadableString()) // First push the package image if !cs.Backend.ImageAvailable(imageID) || cs.Options.PushFinalImagesForce { @@ -547,7 +575,6 @@ func (cs *LuetCompiler) pushFinalArtifact(a *artifact.PackageArtifact, p *compil } // Then the image ID - metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, helpers.SanitizeImageString(a.CompileSpec.GetPackage().GetMetadataFilePath())) if !cs.Backend.ImageAvailable(metadataImageID) || cs.Options.PushFinalImagesForce { cs.Options.Context.Info("Generating metadata image for", a.CompileSpec.Package.HumanReadableString(), metadataImageID) @@ -604,6 +631,10 @@ func (cs *LuetCompiler) findImageHash(imageHash string, p *compilerspec.LuetComp cs.Options.Context.Debug("Resolving image hash for", p.Package.HumanReadableString(), "hash", imageHash, "Pull repositories", p.BuildOptions.PullImageRepository) toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)}, genImageList(p.BuildOptions.PullImageRepository, imageHash)...) + + if cs.Options.PushFinalImagesRepository != "" { + toChecklist = append(toChecklist, fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, imageHash)) + } if exists, which := oneOfImagesExists(toChecklist, cs.Backend); exists { resolvedImage = which } @@ -910,16 +941,40 @@ func (cs *LuetCompiler) getSpecHash(pkgs types.Packages, salt string) (string, e } func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error { + if !p.RequiresFinalImages { + return nil + } joinTag := ">:loop: final images<" + var fromPackages types.Packages - if p.RequiresFinalImages { - cs.Options.Context.Info(joinTag, "Generating a parent image from final packages") - fromPackages = p.Package.GetRequires() - } else { - // No source image to resolve - return nil + cs.Options.Context.Info(joinTag, "Generating a parent image from final packages") + + //fromPackages = p.Package.GetRequires() // (first level only) + pTarget := p + + runtime, err := p.Package.GetRuntimePackage() + if err == nil { + spec, err := cs.FromPackage(runtime) + if err == nil { + cs.Options.Context.Info(joinTag, "Using runtime package for deptree computation") + pTarget = spec + } + } + // resolve deptree of runtime of p and use it in fromPackages + t, err := cs.ComputeDepTree(pTarget) + if err != nil { + return errors.Wrap(err, "failed querying hashtree") + } + + for _, a := range t { + if !a.Value || a.Package.Matches(p.Package) { + continue + } + + fromPackages = append(fromPackages, a.Package) + cs.Options.Context.Infof("Adding dependency '%s'.", a.Package.HumanReadableString()) } // First compute a hash and check if image is available. if it is, then directly consume that @@ -930,10 +985,9 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool cs.Options.Context.Info(joinTag, "Searching existing image with hash", overallFp) - image := cs.findImageHash(overallFp, p) - if image != "" { - cs.Options.Context.Info("Image already found", image) - p.SetImage(image) + if img := cs.findImageHash(overallFp, p); img != "" { + cs.Options.Context.Info("Image already found", img) + p.SetImage(img) return nil } cs.Options.Context.Info(joinTag, "Image not found. Generating image join with hash ", overallFp) @@ -958,28 +1012,58 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool for _, c := range fromPackages { current++ if c != nil && c.Name != "" && c.Version != "" { - joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(p.Package.GetRequires()), c.HumanReadableString()) + joinTag2 := fmt.Sprintf("%s %d/%d ⤑ :hammer: build %s", joinTag, current, len(fromPackages), c.HumanReadableString()) - cs.Options.Context.Info(joinTag2, "compilation starts") - spec, err := cs.FromPackage(c) - if err != nil { - return errors.Wrap(err, "while generating images to join from") + // Search if we have already a final-image that was already pushed + // for this to work on the same repo, it is required to push final images during build + if img := cs.findImageHash(c.ImageID(), p); cs.Options.PullFirst && img != "" { + cs.Options.Context.Info("Final image already found", img) + if !cs.Backend.ImageExists(img) { + if err := cs.Backend.DownloadImage(backend.Options{ImageName: img}); err != nil { + return errors.Wrap(err, "failed pulling image "+img+" during extraction") + } + } + + imgRef, err := cs.Backend.ImageReference(img, true) + if err != nil { + return err + } + + ctx := cs.Options.Context.WithLoggingContext(fmt.Sprintf("final image extract %s", img)) + _, _, err = image.ExtractTo( + ctx, + imgRef, + joinDir, + nil, + ) + if err != nil { + return err + } + } else { + cs.Options.Context.Info("Final image not found for", c.HumanReadableString()) + + // If no image was found, we have to build it from scratch + cs.Options.Context.Info(joinTag2, "compilation starts") + spec, err := cs.FromPackage(c) + if err != nil { + return errors.Wrap(err, "while generating images to join from") + } + wantsArtifact := true + genDepsArtifact := !cs.Options.PackageTargetOnly + + spec.SetOutputPath(p.GetOutputPath()) + + artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, &genDepsArtifact, spec) + if err != nil { + return errors.Wrap(err, "failed building join image") + } + + err = artifact.Unpack(cs.Options.Context, joinDir, keepPermissions) + if err != nil { + return errors.Wrap(err, "failed building join image") + } + cs.Options.Context.Info(joinTag2, ":white_check_mark: Done") } - wantsArtifact := true - genDepsArtifact := !cs.Options.PackageTargetOnly - - spec.SetOutputPath(p.GetOutputPath()) - - artifact, err := cs.compile(concurrency, keepPermissions, &wantsArtifact, &genDepsArtifact, spec) - if err != nil { - return errors.Wrap(err, "failed building join image") - } - - err = artifact.Unpack(cs.Options.Context, joinDir, keepPermissions) - if err != nil { - return errors.Wrap(err, "failed building join image") - } - cs.Options.Context.Info(joinTag2, ":white_check_mark: Done") } } diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index 9d605896..49238371 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -22,6 +22,7 @@ import ( "path/filepath" "strings" + "github.com/mudler/luet/pkg/api/core/logger" "github.com/mudler/luet/pkg/api/core/types" helpers "github.com/mudler/luet/tests/helpers" @@ -1103,4 +1104,107 @@ var _ = Describe("Compiler", func() { Expect(files).ToNot(ContainElement("bin/busybox")) }) }) + + Context("final images", func() { + It("reuses final images", func() { + generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false)) + + err := generalRecipe.Load("../../tests/fixtures/join_complex") + Expect(err).ToNot(HaveOccurred()) + + Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(6)) + logdir, err := ioutil.TempDir("", "log") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(logdir) // clean up + + logPath := filepath.Join(logdir, "logs") + var log string + readLogs := func() { + d, err := ioutil.ReadFile(logPath) + Expect(err).To(BeNil()) + log = string(d) + } + + l, err := logger.New( + logger.WithFileLogging( + logPath, + "", + ), + ) + Expect(err).ToNot(HaveOccurred()) + + c := context.NewContext( + context.WithLogger(l), + ) + + b := sd.NewSimpleDockerBackend(ctx) + + joinImage := "luet/cache:586b36482e3f238c76d3536e7ca12cc4" //resulting join image + allImages := []string{ + joinImage, + "test/test:c-test-1.2"} + + cleanup := func(imgs ...string) { + // Remove the join hash so we force using final images + for _, toRemove := range imgs { + b.RemoveImage(sd.Options{ImageName: toRemove}) + } + } + defer cleanup(allImages...) + + compiler := NewLuetCompiler(b, generalRecipe.GetDatabase(), + options.WithFinalRepository("test/test"), + options.EnableGenerateFinalImages, + options.PullFirst(true), + options.WithContext(c)) + + spec, err := compiler.FromPackage(&types.Package{Name: "x", Category: "test", Version: "0.1"}) + Expect(err).ToNot(HaveOccurred()) + compiler.Options.CompressionType = compression.GZip + Expect(spec.GetPackage().GetPath()).ToNot(Equal("")) + + tmpdir, err := ioutil.TempDir("", "tree") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + + spec.SetOutputPath(tmpdir) + + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) + Expect(errs).To(BeNil()) + Expect(len(artifacts)).To(Equal(1)) + + readLogs() + Expect(log).To(And( + ContainSubstring("Generating final image for"), + ContainSubstring("Adding dependency"), + ContainSubstring("Final image not found for test/c-1.2"), + )) + + Expect(log).ToNot(And( + ContainSubstring("Final image already found test/test:c-test-1.2"), + )) + + os.WriteFile(logPath, []byte{}, os.ModePerm) // cleanup logs + // Remove the join hash so we force using final images + cleanup(joinImage) + + //compile again + By("Recompiling") + + artifacts, errs = compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) + Expect(errs).To(BeNil()) + Expect(len(artifacts)).To(Equal(1)) + + // read logs again + readLogs() + + Expect(log).To(And( + ContainSubstring("Final image already found test/test:a-test-1.2"), + )) + Expect(log).ToNot(And( + ContainSubstring("build test/c-1.2 compilation starts"), + ContainSubstring("Final image not found for test/c-1.2"), + )) + }) + }) }) diff --git a/pkg/compiler/types/options/compiler_options.go b/pkg/compiler/types/options/compiler_options.go index a1f882c8..d58a3dee 100644 --- a/pkg/compiler/types/options/compiler_options.go +++ b/pkg/compiler/types/options/compiler_options.go @@ -49,6 +49,9 @@ type Compiler struct { // Tells wether to push final container images after building PushFinalImages bool PushFinalImagesForce bool + + GenerateFinalImages bool + // Image repository to push to PushFinalImagesRepository string @@ -99,6 +102,11 @@ func WithFinalRepository(r string) func(cfg *Compiler) error { } } +func EnableGenerateFinalImages(cfg *Compiler) error { + cfg.GenerateFinalImages = true + return nil +} + func EnablePushFinalImages(cfg *Compiler) error { cfg.PushFinalImages = true return nil