From 1e78570c50c250cd2a8ce500d3cbe48184326a77 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 4 Nov 2021 22:31:03 +0100 Subject: [PATCH] Allow to push final images while compiling Add --push-final-images, --push-final-images-repository and --push-final-images-force to luet build. --push-final-images enables pushing final artifact during build. Each package built will be packed and pushed to the final repository specified with --push-final-images-repository. By default if no final-repository is specified and pushing final images is enabled will default to the cache repository. --push-final-images-force allows to force-push final images even if there are already available on the specified registry --- cmd/build.go | 28 ++++++-- pkg/compiler/compiler.go | 48 +++++++++++++- pkg/compiler/compiler_test.go | 66 +++++++++++++++++++ .../types/options/compiler_options.go | 27 ++++++++ 4 files changed, 163 insertions(+), 6 deletions(-) diff --git a/cmd/build.go b/cmd/build.go index 2c816cfe..3c931329 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -110,6 +110,9 @@ Build packages specifying multiple definition trees: onlyTarget, _ := cmd.Flags().GetBool("only-target-package") full, _ := cmd.Flags().GetBool("full") rebuild, _ := cmd.Flags().GetBool("rebuild") + pushFinalImages, _ := cmd.Flags().GetBool("push-final-images") + pushFinalImagesRepository, _ := cmd.Flags().GetString("push-final-images-repository") + pushFinalImagesForce, _ := cmd.Flags().GetBool("push-final-images-force") var results Results backendArgs := viper.GetStringSlice("backend-args") @@ -159,8 +162,7 @@ Build packages specifying multiple definition trees: opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency} - luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), - options.NoDeps(nodeps), + compileropts := []options.Option{options.NoDeps(nodeps), options.WithBackendType(backendType), options.PushImages(push), options.WithBuildValues(values), @@ -177,8 +179,21 @@ Build packages specifying multiple definition trees: options.WithContext(util.DefaultContext), options.BackendArgs(backendArgs), options.Concurrency(concurrency), - options.WithCompressionType(compression.Implementation(compressionType)), - ) + options.WithCompressionType(compression.Implementation(compressionType))} + + if pushFinalImages { + compileropts = append(compileropts, options.EnablePushFinalImages) + if pushFinalImagesForce { + compileropts = append(compileropts, options.ForcePushFinalImages) + } + if pushFinalImagesRepository != "" { + compileropts = append(compileropts, options.WithFinalRepository(pushFinalImagesRepository)) + } else if imageRepository != "" { + compileropts = append(compileropts, options.WithFinalRepository(imageRepository)) + } + } + + luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), compileropts...) if full { specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst) @@ -302,6 +317,11 @@ func init() { buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)") buildCmd.Flags().Bool("revdeps", false, "Build with revdeps") buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree") + + buildCmd.Flags().Bool("push-final-images", false, "Push final images while building") + buildCmd.Flags().Bool("push-final-images-force", false, "Override existing images") + buildCmd.Flags().String("push-final-images-repository", "", "Repository where to push final images to") + buildCmd.Flags().Bool("full", false, "Build all packages (optimized)") buildCmd.Flags().StringSlice("values", []string{}, "Build values file to interpolate with each package") buildCmd.Flags().StringSliceP("backend-args", "a", []string{}, "Backend args") diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index af9dbd02..15bdb055 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -488,12 +488,14 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder a.CompileSpec = p a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String()) - err = a.WriteYAML(p.GetOutputPath()) if err != nil { return a, errors.Wrap(err, "Failed while writing metadata file") } cs.Options.Context.Success(pkgTag, " :white_check_mark: done (empty virtual package)") + if cs.Options.PushFinalImages { + cs.pushFinalArtifact(a, p, keepPermissions) + } return a, nil } @@ -523,11 +525,53 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder if err != nil { return a, errors.Wrap(err, "Failed while writing metadata file") } - cs.Options.Context.Success(pkgTag, " :white_check_mark: Done") + cs.Options.Context.Success(pkgTag, " :white_check_mark: Done building") + + if cs.Options.PushFinalImages { + cs.pushFinalArtifact(a, p, keepPermissions) + } return a, nil } +// TODO: A small readaptation of repository_docker.go pushImageFromArtifact() +// Move this to a common place +func (cs *LuetCompiler) pushFinalArtifact(a *artifact.PackageArtifact, p *compilerspec.LuetCompilationSpec, keepPermissions bool) error { + cs.Options.Context.Info("Pushing final image for", a.CompileSpec.Package.HumanReadableString()) + imageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, a.CompileSpec.Package.ImageID()) + + // First push the package image + if !cs.Backend.ImageAvailable(imageID) || cs.Options.PushFinalImagesForce { + cs.Options.Context.Info("Generating and pushing final image for", a.CompileSpec.Package.HumanReadableString(), "as", imageID) + + if err := a.GenerateFinalImage(cs.Options.Context, imageID, cs.GetBackend(), true); err != nil { + return errors.Wrap(err, "while creating final image") + } + if err := cs.Backend.Push(backend.Options{ImageName: imageID}); err != nil { + return errors.Wrapf(err, "Could not push image: %s", imageID) + } + } + + // Then the image ID + metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, a.CompileSpec.GetPackage().GetMetadataFilePath()) + if !cs.Backend.ImageAvailable(metadataImageID) || cs.Options.PushFinalImagesForce { + cs.Options.Context.Info("Generating metadata image for", a.CompileSpec.Package.HumanReadableString(), metadataImageID) + + a := artifact.NewPackageArtifact(filepath.Join(p.GetOutputPath(), a.CompileSpec.GetPackage().GetMetadataFilePath())) + metadataArchive, err := artifact.CreateArtifactForFile(cs.Options.Context, a.Path) + if err != nil { + return errors.Wrap(err, "failed generating checksums for tree") + } + if err := metadataArchive.GenerateFinalImage(cs.Options.Context, metadataImageID, cs.Backend, keepPermissions); err != nil { + return errors.Wrap(err, "Failed generating metadata tree "+metadataImageID) + } + if err = cs.Backend.Push(backend.Options{ImageName: metadataImageID}); err != nil { + return errors.Wrapf(err, "Could not push image: %s", metadataImageID) + } + } + return nil +} + func (cs *LuetCompiler) waitForImages(images []string) { if cs.Options.PullFirst && cs.Options.Wait { available, _ := oneOfImagesAvailable(images, cs.Backend) diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index b6e081ca..24f4f4c1 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -16,10 +16,15 @@ package compiler_test import ( + "fmt" "io/ioutil" "os" "path/filepath" + "strings" + helpers "github.com/mudler/luet/tests/helpers" + + "github.com/mudler/luet/pkg/api/core/image" "github.com/mudler/luet/pkg/api/core/types" "github.com/mudler/luet/pkg/api/core/types/artifact" . "github.com/mudler/luet/pkg/compiler" @@ -853,6 +858,67 @@ var _ = Describe("Compiler", func() { Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue()) Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue()) }) + + It("Pushes final images along", func() { + generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false)) + + randString := strings.ToLower(helpers.String(10)) + imageName := fmt.Sprintf("ttl.sh/%s", randString) + b := sd.NewSimpleDockerBackend(ctx) + + err := generalRecipe.Load("../../tests/fixtures/packagelayers") + Expect(err).ToNot(HaveOccurred()) + + Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) + + compiler := NewLuetCompiler(b, generalRecipe.GetDatabase(), + options.EnablePushFinalImages, options.ForcePushFinalImages, options.WithFinalRepository(imageName)) + + spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"}) + Expect(err).ToNot(HaveOccurred()) + spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "build", Category: "layer", Version: "0.1"}) + Expect(err).ToNot(HaveOccurred()) + Expect(spec.GetPackage().GetPath()).ToNot(Equal("")) + + tmpdir, err := ioutil.TempDir("", "tree") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + + spec.SetOutputPath(tmpdir) + + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2)) + Expect(errs).To(BeNil()) + Expect(len(artifacts)).To(Equal(2)) + //Expect(len(artifacts[0].Dependencies)).To(Equal(1)) + + Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.ImageID()))).To(BeTrue()) + Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.GetMetadataFilePath()))).To(BeTrue()) + + Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.ImageID()))).To(BeTrue()) + Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.GetMetadataFilePath()))).To(BeTrue()) + + img, err := b.ImageReference(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.ImageID()), true) + Expect(err).ToNot(HaveOccurred()) + _, path, err := image.Extract(ctx, img, false, nil) + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(path) // clean up + + Expect(fileHelper.Exists(filepath.Join(path, "bin/busybox"))).To(BeTrue()) + + img, err = b.ImageReference(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.GetMetadataFilePath()), true) + Expect(err).ToNot(HaveOccurred()) + _, path, err = image.Extract(ctx, img, false, nil) + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(path) // clean up + + meta := filepath.Join(path, artifacts[1].Runtime.GetMetadataFilePath()) + Expect(fileHelper.Exists(meta)).To(BeTrue()) + + d, err := ioutil.ReadFile(meta) + Expect(err).ToNot(HaveOccurred()) + + Expect(string(d)).To(ContainSubstring(artifacts[1].CompileSpec.GetPackage().GetName())) + }) }) Context("Packages which conents are a package folder", func() { diff --git a/pkg/compiler/types/options/compiler_options.go b/pkg/compiler/types/options/compiler_options.go index 280fa1a5..6e2961ce 100644 --- a/pkg/compiler/types/options/compiler_options.go +++ b/pkg/compiler/types/options/compiler_options.go @@ -47,6 +47,12 @@ type Compiler struct { // TemplatesFolder. should default to tree/templates TemplatesFolder []string + // Tells wether to push final container images after building + PushFinalImages bool + PushFinalImagesForce bool + // Image repository to push to + PushFinalImagesRepository string + Context *types.Context } @@ -85,6 +91,25 @@ func WithOptions(opt *Compiler) func(cfg *Compiler) error { } } +// WithFinalRepository Sets the final repository where to push +// images of built artifacts +func WithFinalRepository(r string) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.PushFinalImagesRepository = r + return nil + } +} + +func EnablePushFinalImages(cfg *Compiler) error { + cfg.PushFinalImages = true + return nil +} + +func ForcePushFinalImages(cfg *Compiler) error { + cfg.PushFinalImagesForce = true + return nil +} + func WithBackendType(r string) func(cfg *Compiler) error { return func(cfg *Compiler) error { cfg.BackendType = r @@ -113,6 +138,8 @@ func WithPullRepositories(r []string) func(cfg *Compiler) error { } } +// WithPushRepository Sets the image reference where to push +// cache images func WithPushRepository(r string) func(cfg *Compiler) error { return func(cfg *Compiler) error { if len(cfg.PullImageRepository) == 0 {