From ad455edafc9d52921ae1d51a88a0078fa4c0e7de Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 19 Jan 2021 12:00:51 +0100 Subject: [PATCH] Allow to push images in create-repo Add also the --force flag to allow image overwrite Related to #169 --- cmd/create-repo.go | 13 ++++++-- pkg/installer/repository.go | 60 +++++++++++++++++++++++++++++-------- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/cmd/create-repo.go b/cmd/create-repo.go index 1c759d4b..06034c41 100644 --- a/cmd/create-repo.go +++ b/cmd/create-repo.go @@ -67,6 +67,10 @@ Create a repository from the metadata description defined in the luet.yaml confi viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename")) viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision")) viper.BindPFlag("repo", cmd.Flags().Lookup("repo")) + + viper.BindPFlag("force-push", cmd.Flags().Lookup("force-push")) + viper.BindPFlag("push-images", cmd.Flags().Lookup("push-images")) + }, Run: func(cmd *cobra.Command, args []string) { var err error @@ -90,6 +94,8 @@ Create a repository from the metadata description defined in the luet.yaml confi treeFile := installer.NewDefaultTreeRepositoryFile() metaFile := installer.NewDefaultMetaRepositoryFile() compilerBackend := backend.NewBackend(backendType) + force := viper.GetBool("force-push") + imagePush := viper.GetBool("push-images") if source_repo != "" { // Search for system repository @@ -112,11 +118,11 @@ Create a repository from the metadata description defined in the luet.yaml confi lrepo.Priority, packages, treePaths, - pkg.NewInMemoryDatabase(false), compilerBackend, dst) + pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force) } else { repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages, - treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst) + treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force) } if err != nil { @@ -165,6 +171,9 @@ func init() { createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.") createrepoCmd.Flags().String("backend", "docker", "backend used (docker,img)") + createrepoCmd.Flags().Bool("force-push", false, "Force overwrite of docker images if already present") + createrepoCmd.Flags().Bool("push-images", false, "Enable/Disable docker image push for docker repositories") + createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd") createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename") createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd") diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go index 7a873065..46d9f0d4 100644 --- a/pkg/installer/repository.go +++ b/pkg/installer/repository.go @@ -65,6 +65,8 @@ type LuetSystemRepository struct { Tree tree.Builder `json:"-"` RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"` Backend compiler.CompilerBackend `json:"-"` + PushImages bool `json:"-"` + ForcePush bool `json:"-"` } type LuetSystemRepositorySerialized struct { @@ -184,7 +186,7 @@ func (f *LuetRepositoryFile) GetChecksums() compiler.Checksums { func GenerateRepository(name, descr, t string, urls []string, priority int, src string, treesDir []string, db pkg.PackageDatabase, - b compiler.CompilerBackend, imagePrefix string) (Repository, error) { + b compiler.CompilerBackend, imagePrefix string, pushImages, force bool) (Repository, error) { tr := tree.NewInstallerRecipe(db) @@ -205,7 +207,7 @@ func GenerateRepository(name, descr, t string, urls []string, } case DockerRepositoryType: - art, err = generatePackageImages(b, imagePrefix, src, tr.GetDatabase()) + art, err = generatePackageImages(b, imagePrefix, src, tr.GetDatabase(), pushImages, force) if err != nil { return nil, err } @@ -213,7 +215,7 @@ func GenerateRepository(name, descr, t string, urls []string, repo := NewLuetSystemRepository( config.NewLuetRepository(name, t, descr, urls, priority, true, false), - art, tr) + art, tr, pushImages, force) repo.SetBackend(b) return repo, nil } @@ -225,12 +227,14 @@ func NewSystemRepository(repo config.LuetRepository) Repository { } } -func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder) Repository { +func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder, pushImages, force bool) Repository { return &LuetSystemRepository{ LuetRepository: repo, Index: art, Tree: builder, RepositoryFiles: map[string]LuetRepositoryFile{}, + PushImages: pushImages, + ForcePush: force, } } @@ -264,7 +268,15 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos return r, err } -func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) { +func pushImage(b compiler.CompilerBackend, image string, force bool) error { + if b.ImageAvailable(image) && !force { + Debug("Image", image, "already present, skipping") + return nil + } + return b.Push(compiler.CompilerBackendOptions{ImageName: image}) +} + +func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]compiler.Artifact, error) { var art []compiler.Artifact var ff = func(currentpath string, info os.FileInfo, err error) error { @@ -291,12 +303,17 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, return nil } - Info("Generating final image", imagePrefix+artifact.GetCompileSpec().GetPackage().GetPackageImageName(), + packageImage := imagePrefix + artifact.GetCompileSpec().GetPackage().GetPackageImageName() + Info("Generating final image", packageImage, "for package ", artifact.GetCompileSpec().GetPackage().HumanReadableString()) - if opts, err := artifact.GenerateFinalImage(imagePrefix+artifact.GetCompileSpec().GetPackage().GetPackageImageName(), b, true); err != nil { + if opts, err := artifact.GenerateFinalImage(packageImage, b, true); err != nil { return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName) } - // TODO: Push image (check if exists first, and avoid to re-push the same images, unless --force is passed) + if imagePush { + if err := pushImage(b, packageImage, force); err != nil { + return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage) + } + } art = append(art, artifact) @@ -685,7 +702,11 @@ func (r *LuetSystemRepository) genDockerRepo(imagePrefix string, resetRevision, if opts, err := a.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil { return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName) } - // TODO: Push imageTree + if r.ForcePush { + if err := pushImage(r.GetBackend(), imageTree, true); err != nil { + return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree) + } + } // Create Metadata struct and serialized repository meta, serialized := r.Serialize() @@ -736,8 +757,11 @@ func (r *LuetSystemRepository) genDockerRepo(imagePrefix string, resetRevision, if opts, err := a.GenerateFinalImage(imageMetaTree, r.GetBackend(), false); err != nil { return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName) } - - // TODO: Push image meta tree + if r.ForcePush { + if err := pushImage(r.GetBackend(), imageMetaTree, true); err != nil { + return errors.Wrapf(err, "Failed while pushing image: '%s'", imageMetaTree) + } + } data, err := yaml.Marshal(serialized) if err != nil { return err @@ -757,7 +781,11 @@ func (r *LuetSystemRepository) genDockerRepo(imagePrefix string, resetRevision, if opts, err := a.GenerateFinalImage(imageRepo, r.GetBackend(), false); err != nil { return errors.Wrap(err, "Failed generating repository image"+opts.ImageName) } - // TODO: Push image meta tree + if r.ForcePush { + if err := pushImage(r.GetBackend(), imageRepo, true); err != nil { + return errors.Wrapf(err, "Failed while pushing image: '%s'", imageRepo) + } + } bus.Manager.Publish(bus.EventRepositoryPostBuild, struct { Repo LuetSystemRepository @@ -790,8 +818,14 @@ func (r *LuetSystemRepository) Client() Client { Urls: r.GetUrls(), Authentication: r.GetAuthentication(), }) - } + case DockerRepositoryType: + return client.NewDockerClient( + client.RepoData{ + Urls: r.GetUrls(), + Authentication: r.GetAuthentication(), + }) + } return nil }