diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go index f21e95d4..9bc6a1a7 100644 --- a/pkg/installer/repository.go +++ b/pkg/installer/repository.go @@ -16,18 +16,14 @@ package installer import ( - "fmt" "io/ioutil" "os" - "path" "path/filepath" "regexp" "sort" "strconv" - "strings" "time" - "github.com/mudler/luet/pkg/bus" "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" @@ -41,12 +37,14 @@ import ( ) const ( - REPOSITORY_METAFILE = "repository.meta.yaml" - REPOSITORY_SPECFILE = "repository.yaml" - TREE_TARBALL = "tree.tar" + REPOSITORY_METAFILE = "repository.meta.yaml" + REPOSITORY_SPECFILE = "repository.yaml" + TREE_TARBALL = "tree.tar" + COMPILERTREE_TARBALL = "compilertree.tar" - REPOFILE_TREE_KEY = "tree" - REPOFILE_META_KEY = "meta" + REPOFILE_TREE_KEY = "tree" + REPOFILE_COMPILER_TREE_KEY = "compilertree" + REPOFILE_META_KEY = "meta" DiskRepositoryType = "disk" HttpRepositoryType = "http" @@ -68,6 +66,8 @@ type LuetSystemRepository struct { Backend compiler.CompilerBackend `json:"-"` PushImages bool `json:"-"` ForcePush bool `json:"-"` + + imagePrefix string } type LuetSystemRepositorySerialized struct { @@ -159,6 +159,13 @@ func NewDefaultTreeRepositoryFile() LuetRepositoryFile { } } +func NewDefaultCompilerTreeRepositoryFile() LuetRepositoryFile { + return LuetRepositoryFile{ + FileName: COMPILERTREE_TARBALL, + CompressionType: compiler.GZip, + } +} + func NewDefaultMetaRepositoryFile() LuetRepositoryFile { return LuetRepositoryFile{ FileName: REPOSITORY_METAFILE + ".tar", @@ -224,30 +231,20 @@ func GenerateRepository(name, descr, t string, urls []string, } } - // if !strings.HasSuffix(imagePrefix, "/") { - // imagePrefix = imagePrefix + "/" - // } - - var art []compiler.Artifact - var err error - switch t { - case DiskRepositoryType, HttpRepositoryType: - art, err = buildPackageIndex(src, tr.GetDatabase()) - if err != nil { - return nil, err - } - - case DockerRepositoryType: - art, err = generatePackageImages(b, imagePrefix, src, tr.GetDatabase(), pushImages, force) - if err != nil { - return nil, err - } + repo := &LuetSystemRepository{ + LuetRepository: config.NewLuetRepository(name, t, descr, urls, priority, true, false), + Tree: tr, + RepositoryFiles: map[string]LuetRepositoryFile{}, + PushImages: pushImages, + ForcePush: force, + Backend: b, + imagePrefix: imagePrefix, + } + + if err := repo.initialize(src); err != nil { + return nil, errors.Wrap(err, "while building repository artifact index") } - repo := NewLuetSystemRepository( - config.NewLuetRepository(name, t, descr, urls, priority, true, false), - art, tr, pushImages, force) - repo.SetBackend(b) return repo, nil } @@ -258,17 +255,6 @@ func NewSystemRepository(repo config.LuetRepository) Repository { } } -func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder, pushImages, force bool) Repository { - return &LuetSystemRepository{ - LuetRepository: repo, - Index: art, - Tree: builder, - RepositoryFiles: map[string]LuetRepositoryFile{}, - PushImages: pushImages, - ForcePush: force, - } -} - func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, error) { var p *LuetSystemRepositorySerialized err := yaml.Unmarshal(data, &p) @@ -302,118 +288,24 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos return r, err } -func pushImage(b compiler.CompilerBackend, image string, force bool) error { - if b.ImageAvailable(image) && !force { - Debug("Image", image, "already present, skipping") - return nil - } - return b.Push(compiler.CompilerBackendOptions{ImageName: image}) -} - -func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]compiler.Artifact, error) { - Info("Generating docker images for packages in", imagePrefix) - var art []compiler.Artifact - var ff = func(currentpath string, info os.FileInfo, err error) error { - - if !strings.HasSuffix(info.Name(), ".metadata.yaml") { - return nil // Skip with no errors - } - - dat, err := ioutil.ReadFile(currentpath) - if err != nil { - return errors.Wrap(err, "Error reading file "+currentpath) - } - - artifact, err := compiler.NewPackageArtifactFromYaml(dat) - if err != nil { - return errors.Wrap(err, "Error reading yaml "+currentpath) - } - // Set the path relative to the file. - // The metadata contains the full path where the file was located during buildtime. - artifact.SetPath(filepath.Join(filepath.Dir(currentpath), filepath.Base(artifact.GetPath()))) - - // We want to include packages that are ONLY referenced in the tree. - // the ones which aren't should be deleted. (TODO: by another cli command?) - if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil { - Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.", - artifact.GetCompileSpec().GetPackage().HumanReadableString())) - return nil - } - - packageImage := fmt.Sprintf("%s:%s", imagePrefix, artifact.GetCompileSpec().GetPackage().ImageID()) - - if imagePush && b.ImageAvailable(packageImage) && !force { - Info("Image", packageImage, "already present, skipping. use --force-push to override") - } else { - Info("Generating final image", packageImage, - "for package ", artifact.GetCompileSpec().GetPackage().HumanReadableString()) - if opts, err := artifact.GenerateFinalImage(packageImage, b, true); err != nil { - return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName) - } - } - if imagePush { - if err := pushImage(b, packageImage, force); err != nil { - return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage) - } - } - - art = append(art, artifact) - - return nil - } - - err := filepath.Walk(path, ff) - if err != nil { - return nil, err - - } - return art, nil -} - -func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) { - - var art []compiler.Artifact - var ff = func(currentpath string, info os.FileInfo, err error) error { - - if !strings.HasSuffix(info.Name(), ".metadata.yaml") { - return nil // Skip with no errors - } - - dat, err := ioutil.ReadFile(currentpath) - if err != nil { - return errors.Wrap(err, "Error reading file "+currentpath) - } - - artifact, err := compiler.NewPackageArtifactFromYaml(dat) - if err != nil { - return errors.Wrap(err, "Error reading yaml "+currentpath) - } - - // We want to include packages that are ONLY referenced in the tree. - // the ones which aren't should be deleted. (TODO: by another cli command?) - if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil { - Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.", - artifact.GetCompileSpec().GetPackage().HumanReadableString())) - return nil - } - - art = append(art, artifact) - - return nil - } - - err := filepath.Walk(path, ff) - if err != nil { - return nil, err - - } - return art, nil -} - func (r *LuetSystemRepository) SetPriority(n int) { r.LuetRepository.Priority = n } +func (r *LuetSystemRepository) initialize(src string) error { + generator, err := r.getGenerator() + if err != nil { + return errors.Wrap(err, "while constructing repository generator") + } + art, err := generator.Initialize(src, r.Tree.GetDatabase()) + if err != nil { + return errors.Wrap(err, "while initializing repository generator") + } + // update the repository index + r.Index = art + return nil +} + // FileSearch search a pattern among the artifacts in a repository func (r *LuetSystemRepository) FileSearch(pattern string) (pkg.Packages, error) { var matches pkg.Packages @@ -561,339 +453,37 @@ func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repos return repo, err } -func (r *LuetSystemRepository) genLocalRepo(dst string, resetRevision bool) error { - err := os.MkdirAll(dst, os.ModePerm) - if err != nil { - return err - } - r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10) - - repospec := filepath.Join(dst, REPOSITORY_SPECFILE) - if resetRevision { - r.Revision = 0 - } else { - if _, err := os.Stat(repospec); !os.IsNotExist(err) { - // Read existing file for retrieve revision - spec, err := r.ReadSpecFile(repospec, false) - if err != nil { - return err - } - r.Revision = spec.GetRevision() - } - } - r.Revision++ - - Info(fmt.Sprintf( - "For repository %s creating revision %d and last update %s...", - r.Name, r.Revision, r.LastUpdate, - )) - - bus.Manager.Publish(bus.EventRepositoryPreBuild, struct { - Repo LuetSystemRepository - Path string - }{ - Repo: *r, - Path: dst, - }) - - // Create tree and repository file - archive, err := config.LuetCfg.GetSystem().TempDir("archive") - if err != nil { - return errors.Wrap(err, "Error met while creating tempdir for archive") - } - defer os.RemoveAll(archive) // clean up - err = r.GetTree().Save(archive) - if err != nil { - return errors.Wrap(err, "Error met while saving the tree") - } - - treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY) - if err != nil { - treeFile = NewDefaultTreeRepositoryFile() - r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile) - } - - a := compiler.NewPackageArtifact(filepath.Join(dst, treeFile.GetFileName())) - a.SetCompressionType(treeFile.GetCompressionType()) - err = a.Compress(archive, 1) - if err != nil { - return errors.Wrap(err, "Error met while creating package archive") - } - - // Update the tree name with the name created by compression selected. - treeFile.SetFileName(path.Base(a.GetPath())) - err = a.Hash() - if err != nil { - return errors.Wrap(err, "Failed generating checksums for tree") - } - treeFile.SetChecksums(a.GetChecksums()) - r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile) - - // Create Metadata struct and serialized repository - meta, serialized := r.Serialize() - - // Create metadata file and repository file - metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata") - defer os.RemoveAll(metaTmpDir) // clean up - if err != nil { - return errors.Wrap(err, "Error met while creating tempdir for metadata") - } - - metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY) - if err != nil { - metaFile = NewDefaultMetaRepositoryFile() - r.SetRepositoryFile(REPOFILE_META_KEY, metaFile) - } - - repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE) - // Create repository.meta.yaml file - err = meta.WriteFile(repoMetaSpec) - if err != nil { - return err - } - - a = compiler.NewPackageArtifact(filepath.Join(dst, metaFile.GetFileName())) - a.SetCompressionType(metaFile.GetCompressionType()) - err = a.Compress(metaTmpDir, 1) - if err != nil { - return errors.Wrap(err, "Error met while archiving repository metadata") - } - - metaFile.SetFileName(path.Base(a.GetPath())) - r.SetRepositoryFile(REPOFILE_META_KEY, metaFile) - err = a.Hash() - if err != nil { - return errors.Wrap(err, "Failed generating checksums for metadata") - } - metaFile.SetChecksums(a.GetChecksums()) - - data, err := yaml.Marshal(serialized) - if err != nil { - return err - } - err = ioutil.WriteFile(repospec, data, os.ModePerm) - if err != nil { - return err - } - - bus.Manager.Publish(bus.EventRepositoryPostBuild, struct { - Repo LuetSystemRepository - Path string - }{ - Repo: *r, - Path: dst, - }) - return nil +type RepositoryGenerator interface { + Generate(*LuetSystemRepository, string, bool) error + Initialize(string, pkg.PackageDatabase) ([]compiler.Artifact, error) } -func (r *LuetSystemRepository) genDockerRepo(imagePrefix string, resetRevision, force bool) error { - // - Iterate over meta, build final images, push them if necessary - // - while pushing, check if image already exists, and if exist push them only if --force is supplied - // - Generate final images for metadata and push - - imageRepository := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE) - - r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10) - - repoTemp, err := config.LuetCfg.GetSystem().TempDir("repo") - if err != nil { - return errors.Wrap(err, "Error met while creating tempdir for repository") - } - defer os.RemoveAll(repoTemp) // clean up - - if r.GetBackend().ImageAvailable(imageRepository) { - if err := r.GetBackend().DownloadImage(compiler.CompilerBackendOptions{ImageName: imageRepository}); err != nil { - return errors.Wrapf(err, "while downloading '%s'", imageRepository) - } - - if err := r.GetBackend().ExtractRootfs(compiler.CompilerBackendOptions{ImageName: imageRepository, Destination: repoTemp}, false); err != nil { - return errors.Wrapf(err, "while extracting '%s'", imageRepository) +func (r *LuetSystemRepository) getGenerator() (RepositoryGenerator, error) { + var rg RepositoryGenerator + switch r.GetType() { + case DiskRepositoryType, HttpRepositoryType: + rg = &localRepositoryGenerator{} + case DockerRepositoryType: + rg = &dockerRepositoryGenerator{ + b: r.Backend, + imagePrefix: r.imagePrefix, + imagePush: r.PushImages, + force: r.ForcePush, } + default: + return nil, errors.New("invalid repository type") } - - repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE) - if resetRevision { - r.Revision = 0 - } else { - if _, err := os.Stat(repospec); !os.IsNotExist(err) { - // Read existing file for retrieve revision - spec, err := r.ReadSpecFile(repospec, false) - if err != nil { - return err - } - r.Revision = spec.GetRevision() - } - } - r.Revision++ - - Info(fmt.Sprintf( - "For repository %s creating revision %d and last update %s...", - r.Name, r.Revision, r.LastUpdate, - )) - - bus.Manager.Publish(bus.EventRepositoryPreBuild, struct { - Repo LuetSystemRepository - Path string - }{ - Repo: *r, - Path: imageRepository, - }) - - // Create tree and repository file - archive, err := config.LuetCfg.GetSystem().TempDir("archive") - if err != nil { - return errors.Wrap(err, "Error met while creating tempdir for archive") - } - defer os.RemoveAll(archive) // clean up - err = r.GetTree().Save(archive) - if err != nil { - return errors.Wrap(err, "Error met while saving the tree") - } - - treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY) - if err != nil { - treeFile = NewDefaultTreeRepositoryFile() - r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile) - } - - a := compiler.NewPackageArtifact(filepath.Join(repoTemp, treeFile.GetFileName())) - a.SetCompressionType(treeFile.GetCompressionType()) - err = a.Compress(archive, 1) - if err != nil { - return errors.Wrap(err, "Error met while creating package archive") - } - - // Update the tree name with the name created by compression selected. - treeFile.SetFileName(a.GetFileName()) - err = a.Hash() - if err != nil { - return errors.Wrap(err, "Failed generating checksums for tree") - } - treeFile.SetChecksums(a.GetChecksums()) - r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile) - - // we generate a new archive containing the required compressed file. - // TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file - treeArchive, err := compiler.CreateArtifactForFile(a.GetPath()) - if err != nil { - return errors.Wrap(err, "Failed generating checksums for tree") - } - imageTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName()) - Debug("Generating image", imageTree) - if opts, err := treeArchive.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil { - return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName) - } - if r.PushImages { - if err := pushImage(r.GetBackend(), imageTree, true); err != nil { - return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree) - } - } - - // Create Metadata struct and serialized repository - meta, serialized := r.Serialize() - - // Create metadata file and repository file - metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata") - if err != nil { - return errors.Wrap(err, "Error met while creating tempdir for metadata") - } - defer os.RemoveAll(metaTmpDir) // clean up - - metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY) - if err != nil { - metaFile = NewDefaultMetaRepositoryFile() - r.SetRepositoryFile(REPOFILE_META_KEY, metaFile) - } - - repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE) - // Create repository.meta.yaml file - err = meta.WriteFile(repoMetaSpec) - if err != nil { - return err - } - - // create temp dir for metafile - metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata") - if err != nil { - return errors.Wrap(err, "Error met while creating tempdir for metadata") - } - defer os.RemoveAll(metaDir) // clean up - - a = compiler.NewPackageArtifact(filepath.Join(metaDir, metaFile.GetFileName())) - a.SetCompressionType(metaFile.GetCompressionType()) - err = a.Compress(metaTmpDir, 1) - if err != nil { - return errors.Wrap(err, "Error met while archiving repository metadata") - } - - metaFile.SetFileName(a.GetFileName()) - r.SetRepositoryFile(REPOFILE_META_KEY, metaFile) - err = a.Hash() - if err != nil { - return errors.Wrap(err, "Failed generating checksums for metadata") - } - metaFile.SetChecksums(a.GetChecksums()) - - // Files are downloaded as-is from docker images - // we generate a new archive containing the required compressed file. - // TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file - metaArchive, err := compiler.CreateArtifactForFile(a.GetPath()) - if err != nil { - return errors.Wrap(err, "Failed generating checksums for tree") - } - imageMetaTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName()) - if opts, err := metaArchive.GenerateFinalImage(imageMetaTree, r.GetBackend(), false); err != nil { - return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName) - } - if r.PushImages { - if err := pushImage(r.GetBackend(), imageMetaTree, true); err != nil { - return errors.Wrapf(err, "Failed while pushing image: '%s'", imageMetaTree) - } - } - data, err := yaml.Marshal(serialized) - if err != nil { - return err - } - err = ioutil.WriteFile(repospec, data, os.ModePerm) - if err != nil { - return err - } - - tempRepoFile := filepath.Join(metaDir, REPOSITORY_SPECFILE+".tar") - if err := helpers.Tar(repospec, tempRepoFile); err != nil { - return errors.Wrap(err, "Error met while archiving repository file") - } - - a = compiler.NewPackageArtifact(tempRepoFile) - imageRepo := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE) - if opts, err := a.GenerateFinalImage(imageRepo, r.GetBackend(), false); err != nil { - return errors.Wrap(err, "Failed generating repository image"+opts.ImageName) - } - if r.PushImages { - if err := pushImage(r.GetBackend(), imageRepo, true); err != nil { - return errors.Wrapf(err, "Failed while pushing image: '%s'", imageRepo) - } - } - - bus.Manager.Publish(bus.EventRepositoryPostBuild, struct { - Repo LuetSystemRepository - Path string - }{ - Repo: *r, - Path: imagePrefix, - }) - return nil + return rg, nil } // Write writes the repository metadata to the supplied destination func (r *LuetSystemRepository) Write(dst string, resetRevision, force bool) error { - switch r.GetType() { - case DiskRepositoryType, HttpRepositoryType: - return r.genLocalRepo(dst, resetRevision) - case DockerRepositoryType: - return r.genDockerRepo(dst, resetRevision, force) + rg, err := r.getGenerator() + if err != nil { + return err } - return errors.New("invalid repository type") + + return rg.Generate(r, dst, resetRevision) } func (r *LuetSystemRepository) Client() Client { diff --git a/pkg/installer/repository_docker.go b/pkg/installer/repository_docker.go new file mode 100644 index 00000000..a5e615e9 --- /dev/null +++ b/pkg/installer/repository_docker.go @@ -0,0 +1,313 @@ +// Copyright © 2019-2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package installer + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + . "github.com/mudler/luet/pkg/logger" + pkg "github.com/mudler/luet/pkg/package" + + "github.com/ghodss/yaml" + "github.com/mudler/luet/pkg/bus" + compiler "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/config" + "github.com/mudler/luet/pkg/helpers" + "github.com/pkg/errors" +) + +type dockerRepositoryGenerator struct { + b compiler.CompilerBackend + imagePrefix string + imagePush, force bool +} + +func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) { + return generatePackageImages(l.b, l.imagePrefix, path, db, l.imagePush, l.force) +} + +func pushImage(b compiler.CompilerBackend, image string, force bool) error { + if b.ImageAvailable(image) && !force { + Debug("Image", image, "already present, skipping") + return nil + } + return b.Push(compiler.CompilerBackendOptions{ImageName: image}) +} + +func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]compiler.Artifact, error) { + Info("Generating docker images for packages in", imagePrefix) + var art []compiler.Artifact + var ff = func(currentpath string, info os.FileInfo, err error) error { + + if !strings.HasSuffix(info.Name(), ".metadata.yaml") { + return nil // Skip with no errors + } + + dat, err := ioutil.ReadFile(currentpath) + if err != nil { + return errors.Wrap(err, "Error reading file "+currentpath) + } + + artifact, err := compiler.NewPackageArtifactFromYaml(dat) + if err != nil { + return errors.Wrap(err, "Error reading yaml "+currentpath) + } + // Set the path relative to the file. + // The metadata contains the full path where the file was located during buildtime. + artifact.SetPath(filepath.Join(filepath.Dir(currentpath), filepath.Base(artifact.GetPath()))) + + // We want to include packages that are ONLY referenced in the tree. + // the ones which aren't should be deleted. (TODO: by another cli command?) + if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil { + Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.", + artifact.GetCompileSpec().GetPackage().HumanReadableString())) + return nil + } + + packageImage := fmt.Sprintf("%s:%s", imagePrefix, artifact.GetCompileSpec().GetPackage().ImageID()) + + if imagePush && b.ImageAvailable(packageImage) && !force { + Info("Image", packageImage, "already present, skipping. use --force-push to override") + } else { + Info("Generating final image", packageImage, + "for package ", artifact.GetCompileSpec().GetPackage().HumanReadableString()) + if opts, err := artifact.GenerateFinalImage(packageImage, b, true); err != nil { + return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName) + } + } + if imagePush { + if err := pushImage(b, packageImage, force); err != nil { + return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage) + } + } + + art = append(art, artifact) + + return nil + } + + err := filepath.Walk(path, ff) + if err != nil { + return nil, err + + } + return art, nil +} + +func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefix string, resetRevision bool) error { + // - Iterate over meta, build final images, push them if necessary + // - while pushing, check if image already exists, and if exist push them only if --force is supplied + // - Generate final images for metadata and push + + imageRepository := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE) + + r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10) + + repoTemp, err := config.LuetCfg.GetSystem().TempDir("repo") + if err != nil { + return errors.Wrap(err, "Error met while creating tempdir for repository") + } + defer os.RemoveAll(repoTemp) // clean up + + if r.GetBackend().ImageAvailable(imageRepository) { + if err := r.GetBackend().DownloadImage(compiler.CompilerBackendOptions{ImageName: imageRepository}); err != nil { + return errors.Wrapf(err, "while downloading '%s'", imageRepository) + } + + if err := r.GetBackend().ExtractRootfs(compiler.CompilerBackendOptions{ImageName: imageRepository, Destination: repoTemp}, false); err != nil { + return errors.Wrapf(err, "while extracting '%s'", imageRepository) + } + } + + repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE) + if resetRevision { + r.Revision = 0 + } else { + if _, err := os.Stat(repospec); !os.IsNotExist(err) { + // Read existing file for retrieve revision + spec, err := r.ReadSpecFile(repospec, false) + if err != nil { + return err + } + r.Revision = spec.GetRevision() + } + } + r.Revision++ + + Info(fmt.Sprintf( + "For repository %s creating revision %d and last update %s...", + r.Name, r.Revision, r.LastUpdate, + )) + + bus.Manager.Publish(bus.EventRepositoryPreBuild, struct { + Repo LuetSystemRepository + Path string + }{ + Repo: *r, + Path: imageRepository, + }) + + // Create tree and repository file + archive, err := config.LuetCfg.GetSystem().TempDir("archive") + if err != nil { + return errors.Wrap(err, "Error met while creating tempdir for archive") + } + defer os.RemoveAll(archive) // clean up + err = r.GetTree().Save(archive) + if err != nil { + return errors.Wrap(err, "Error met while saving the tree") + } + + treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY) + if err != nil { + treeFile = NewDefaultTreeRepositoryFile() + r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile) + } + + a := compiler.NewPackageArtifact(filepath.Join(repoTemp, treeFile.GetFileName())) + a.SetCompressionType(treeFile.GetCompressionType()) + err = a.Compress(archive, 1) + if err != nil { + return errors.Wrap(err, "Error met while creating package archive") + } + + // Update the tree name with the name created by compression selected. + treeFile.SetFileName(a.GetFileName()) + err = a.Hash() + if err != nil { + return errors.Wrap(err, "Failed generating checksums for tree") + } + treeFile.SetChecksums(a.GetChecksums()) + r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile) + + // we generate a new archive containing the required compressed file. + // TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file + treeArchive, err := compiler.CreateArtifactForFile(a.GetPath()) + if err != nil { + return errors.Wrap(err, "Failed generating checksums for tree") + } + imageTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName()) + Debug("Generating image", imageTree) + if opts, err := treeArchive.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil { + return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName) + } + if r.PushImages { + if err := pushImage(r.GetBackend(), imageTree, true); err != nil { + return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree) + } + } + + // Create Metadata struct and serialized repository + meta, serialized := r.Serialize() + + // Create metadata file and repository file + metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata") + if err != nil { + return errors.Wrap(err, "Error met while creating tempdir for metadata") + } + defer os.RemoveAll(metaTmpDir) // clean up + + metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY) + if err != nil { + metaFile = NewDefaultMetaRepositoryFile() + r.SetRepositoryFile(REPOFILE_META_KEY, metaFile) + } + + repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE) + // Create repository.meta.yaml file + err = meta.WriteFile(repoMetaSpec) + if err != nil { + return err + } + + // create temp dir for metafile + metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata") + if err != nil { + return errors.Wrap(err, "Error met while creating tempdir for metadata") + } + defer os.RemoveAll(metaDir) // clean up + + a = compiler.NewPackageArtifact(filepath.Join(metaDir, metaFile.GetFileName())) + a.SetCompressionType(metaFile.GetCompressionType()) + err = a.Compress(metaTmpDir, 1) + if err != nil { + return errors.Wrap(err, "Error met while archiving repository metadata") + } + + metaFile.SetFileName(a.GetFileName()) + r.SetRepositoryFile(REPOFILE_META_KEY, metaFile) + err = a.Hash() + if err != nil { + return errors.Wrap(err, "Failed generating checksums for metadata") + } + metaFile.SetChecksums(a.GetChecksums()) + + // Files are downloaded as-is from docker images + // we generate a new archive containing the required compressed file. + // TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file + metaArchive, err := compiler.CreateArtifactForFile(a.GetPath()) + if err != nil { + return errors.Wrap(err, "Failed generating checksums for tree") + } + imageMetaTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName()) + if opts, err := metaArchive.GenerateFinalImage(imageMetaTree, r.GetBackend(), false); err != nil { + return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName) + } + if r.PushImages { + if err := pushImage(r.GetBackend(), imageMetaTree, true); err != nil { + return errors.Wrapf(err, "Failed while pushing image: '%s'", imageMetaTree) + } + } + data, err := yaml.Marshal(serialized) + if err != nil { + return err + } + err = ioutil.WriteFile(repospec, data, os.ModePerm) + if err != nil { + return err + } + + tempRepoFile := filepath.Join(metaDir, REPOSITORY_SPECFILE+".tar") + if err := helpers.Tar(repospec, tempRepoFile); err != nil { + return errors.Wrap(err, "Error met while archiving repository file") + } + + a = compiler.NewPackageArtifact(tempRepoFile) + imageRepo := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE) + if opts, err := a.GenerateFinalImage(imageRepo, r.GetBackend(), false); err != nil { + return errors.Wrap(err, "Failed generating repository image"+opts.ImageName) + } + if r.PushImages { + if err := pushImage(r.GetBackend(), imageRepo, true); err != nil { + return errors.Wrapf(err, "Failed while pushing image: '%s'", imageRepo) + } + } + + bus.Manager.Publish(bus.EventRepositoryPostBuild, struct { + Repo LuetSystemRepository + Path string + }{ + Repo: *r, + Path: imagePrefix, + }) + return nil +} diff --git a/pkg/installer/repository_local.go b/pkg/installer/repository_local.go new file mode 100644 index 00000000..78a16ec0 --- /dev/null +++ b/pkg/installer/repository_local.go @@ -0,0 +1,208 @@ +// Copyright © 2019-2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package installer + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + . "github.com/mudler/luet/pkg/logger" + pkg "github.com/mudler/luet/pkg/package" + + "github.com/ghodss/yaml" + "github.com/mudler/luet/pkg/bus" + compiler "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/config" + "github.com/pkg/errors" +) + +type localRepositoryGenerator struct{} + +func (l *localRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) { + return buildPackageIndex(path, db) +} + +func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) { + + var art []compiler.Artifact + var ff = func(currentpath string, info os.FileInfo, err error) error { + + if !strings.HasSuffix(info.Name(), ".metadata.yaml") { + return nil // Skip with no errors + } + + dat, err := ioutil.ReadFile(currentpath) + if err != nil { + return errors.Wrap(err, "Error reading file "+currentpath) + } + + artifact, err := compiler.NewPackageArtifactFromYaml(dat) + if err != nil { + return errors.Wrap(err, "Error reading yaml "+currentpath) + } + + // We want to include packages that are ONLY referenced in the tree. + // the ones which aren't should be deleted. (TODO: by another cli command?) + if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil { + Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.", + artifact.GetCompileSpec().GetPackage().HumanReadableString())) + return nil + } + + art = append(art, artifact) + + return nil + } + + err := filepath.Walk(path, ff) + if err != nil { + return nil, err + + } + return art, nil +} + +func (*localRepositoryGenerator) Generate(r *LuetSystemRepository, dst string, resetRevision bool) error { + err := os.MkdirAll(dst, os.ModePerm) + if err != nil { + return err + } + r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10) + + repospec := filepath.Join(dst, REPOSITORY_SPECFILE) + if resetRevision { + r.Revision = 0 + } else { + if _, err := os.Stat(repospec); !os.IsNotExist(err) { + // Read existing file for retrieve revision + spec, err := r.ReadSpecFile(repospec, false) + if err != nil { + return err + } + r.Revision = spec.GetRevision() + } + } + r.Revision++ + + Info(fmt.Sprintf( + "For repository %s creating revision %d and last update %s...", + r.Name, r.Revision, r.LastUpdate, + )) + + bus.Manager.Publish(bus.EventRepositoryPreBuild, struct { + Repo LuetSystemRepository + Path string + }{ + Repo: *r, + Path: dst, + }) + + // Create tree and repository file + archive, err := config.LuetCfg.GetSystem().TempDir("archive") + if err != nil { + return errors.Wrap(err, "Error met while creating tempdir for archive") + } + defer os.RemoveAll(archive) // clean up + + err = r.GetTree().Save(archive) + if err != nil { + return errors.Wrap(err, "Error met while saving the tree") + } + + treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY) + if err != nil { + treeFile = NewDefaultTreeRepositoryFile() + r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile) + } + + a := compiler.NewPackageArtifact(filepath.Join(dst, treeFile.GetFileName())) + a.SetCompressionType(treeFile.GetCompressionType()) + err = a.Compress(archive, 1) + if err != nil { + return errors.Wrap(err, "Error met while creating package archive") + } + + // Update the tree name with the name created by compression selected. + treeFile.SetFileName(path.Base(a.GetPath())) + err = a.Hash() + if err != nil { + return errors.Wrap(err, "Failed generating checksums for tree") + } + treeFile.SetChecksums(a.GetChecksums()) + r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile) + + // Create Metadata struct and serialized repository + meta, serialized := r.Serialize() + + // Create metadata file and repository file + metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata") + defer os.RemoveAll(metaTmpDir) // clean up + if err != nil { + return errors.Wrap(err, "Error met while creating tempdir for metadata") + } + + metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY) + if err != nil { + metaFile = NewDefaultMetaRepositoryFile() + r.SetRepositoryFile(REPOFILE_META_KEY, metaFile) + } + + repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE) + // Create repository.meta.yaml file + err = meta.WriteFile(repoMetaSpec) + if err != nil { + return err + } + + a = compiler.NewPackageArtifact(filepath.Join(dst, metaFile.GetFileName())) + a.SetCompressionType(metaFile.GetCompressionType()) + err = a.Compress(metaTmpDir, 1) + if err != nil { + return errors.Wrap(err, "Error met while archiving repository metadata") + } + + metaFile.SetFileName(path.Base(a.GetPath())) + r.SetRepositoryFile(REPOFILE_META_KEY, metaFile) + err = a.Hash() + if err != nil { + return errors.Wrap(err, "Failed generating checksums for metadata") + } + metaFile.SetChecksums(a.GetChecksums()) + + data, err := yaml.Marshal(serialized) + if err != nil { + return err + } + err = ioutil.WriteFile(repospec, data, os.ModePerm) + if err != nil { + return err + } + + bus.Manager.Publish(bus.EventRepositoryPostBuild, struct { + Repo LuetSystemRepository + Path string + }{ + Repo: *r, + Path: dst, + }) + return nil +} diff --git a/pkg/installer/repository_test.go b/pkg/installer/repository_test.go index 8b8abc3a..c5385371 100644 --- a/pkg/installer/repository_test.go +++ b/pkg/installer/repository_test.go @@ -1,4 +1,4 @@ -// Copyright © 2019 Ettore Di Giacinto +// Copyright © 2019 Ettore Di Giacinto // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by