kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-08-29 04:10:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Split up repository generators

Browse Source
This commit is contained in:
Ettore Di Giacinto 2021-04-08 15:22:18 +02:00
parent c7b79bf630
commit 036b5c08c6
4 changed files with 586 additions and 475 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

524
pkg/installer/repository.go
View File

@ -16,18 +16,14 @@
package installer
import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/mudler/luet/pkg/bus"
"github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
@ -44,8 +40,10 @@ const (
REPOSITORY_METAFILE = "repository.meta.yaml"
REPOSITORY_SPECFILE = "repository.yaml"
TREE_TARBALL = "tree.tar"
COMPILERTREE_TARBALL = "compilertree.tar"
REPOFILE_TREE_KEY = "tree"
REPOFILE_COMPILER_TREE_KEY = "compilertree"
REPOFILE_META_KEY = "meta"
DiskRepositoryType = "disk"
@ -68,6 +66,8 @@ type LuetSystemRepository struct {
Backend compiler.CompilerBackend `json:"-"`
PushImages bool `json:"-"`
ForcePush bool `json:"-"`
imagePrefix string
}
type LuetSystemRepositorySerialized struct {
@ -159,6 +159,13 @@ func NewDefaultTreeRepositoryFile() LuetRepositoryFile {
}
}
func NewDefaultCompilerTreeRepositoryFile() LuetRepositoryFile {
return LuetRepositoryFile{
FileName: COMPILERTREE_TARBALL,
CompressionType: compiler.GZip,
}
}
func NewDefaultMetaRepositoryFile() LuetRepositoryFile {
return LuetRepositoryFile{
FileName: REPOSITORY_METAFILE + ".tar",
@ -224,30 +231,20 @@ func GenerateRepository(name, descr, t string, urls []string,
}
}
// if !strings.HasSuffix(imagePrefix, "/") {
// imagePrefix = imagePrefix + "/"
// }
var art []compiler.Artifact
var err error
switch t {
case DiskRepositoryType, HttpRepositoryType:
art, err = buildPackageIndex(src, tr.GetDatabase())
if err != nil {
return nil, err
repo := &LuetSystemRepository{
LuetRepository: config.NewLuetRepository(name, t, descr, urls, priority, true, false),
Tree: tr,
RepositoryFiles: map[string]LuetRepositoryFile{},
PushImages: pushImages,
ForcePush: force,
Backend: b,
imagePrefix: imagePrefix,
}
case DockerRepositoryType:
art, err = generatePackageImages(b, imagePrefix, src, tr.GetDatabase(), pushImages, force)
if err != nil {
return nil, err
}
if err := repo.initialize(src); err != nil {
return nil, errors.Wrap(err, "while building repository artifact index")
}
repo := NewLuetSystemRepository(
config.NewLuetRepository(name, t, descr, urls, priority, true, false),
art, tr, pushImages, force)
repo.SetBackend(b)
return repo, nil
}
@ -258,17 +255,6 @@ func NewSystemRepository(repo config.LuetRepository) Repository {
}
}
func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder, pushImages, force bool) Repository {
return &LuetSystemRepository{
LuetRepository: repo,
Index: art,
Tree: builder,
RepositoryFiles: map[string]LuetRepositoryFile{},
PushImages: pushImages,
ForcePush: force,
}
}
func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, error) {
var p *LuetSystemRepositorySerialized
err := yaml.Unmarshal(data, &p)
@ -302,118 +288,24 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
return r, err
}
func pushImage(b compiler.CompilerBackend, image string, force bool) error {
if b.ImageAvailable(image) && !force {
Debug("Image", image, "already present, skipping")
return nil
}
return b.Push(compiler.CompilerBackendOptions{ImageName: image})
}
func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]compiler.Artifact, error) {
Info("Generating docker images for packages in", imagePrefix)
var art []compiler.Artifact
var ff = func(currentpath string, info os.FileInfo, err error) error {
if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
return nil // Skip with no errors
}
dat, err := ioutil.ReadFile(currentpath)
if err != nil {
return errors.Wrap(err, "Error reading file "+currentpath)
}
artifact, err := compiler.NewPackageArtifactFromYaml(dat)
if err != nil {
return errors.Wrap(err, "Error reading yaml "+currentpath)
}
// Set the path relative to the file.
// The metadata contains the full path where the file was located during buildtime.
artifact.SetPath(filepath.Join(filepath.Dir(currentpath), filepath.Base(artifact.GetPath())))
// We want to include packages that are ONLY referenced in the tree.
// the ones which aren't should be deleted. (TODO: by another cli command?)
if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
artifact.GetCompileSpec().GetPackage().HumanReadableString()))
return nil
}
packageImage := fmt.Sprintf("%s:%s", imagePrefix, artifact.GetCompileSpec().GetPackage().ImageID())
if imagePush && b.ImageAvailable(packageImage) && !force {
Info("Image", packageImage, "already present, skipping. use --force-push to override")
} else {
Info("Generating final image", packageImage,
"for package ", artifact.GetCompileSpec().GetPackage().HumanReadableString())
if opts, err := artifact.GenerateFinalImage(packageImage, b, true); err != nil {
return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
}
}
if imagePush {
if err := pushImage(b, packageImage, force); err != nil {
return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage)
}
}
art = append(art, artifact)
return nil
}
err := filepath.Walk(path, ff)
if err != nil {
return nil, err
}
return art, nil
}
func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) {
var art []compiler.Artifact
var ff = func(currentpath string, info os.FileInfo, err error) error {
if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
return nil // Skip with no errors
}
dat, err := ioutil.ReadFile(currentpath)
if err != nil {
return errors.Wrap(err, "Error reading file "+currentpath)
}
artifact, err := compiler.NewPackageArtifactFromYaml(dat)
if err != nil {
return errors.Wrap(err, "Error reading yaml "+currentpath)
}
// We want to include packages that are ONLY referenced in the tree.
// the ones which aren't should be deleted. (TODO: by another cli command?)
if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
artifact.GetCompileSpec().GetPackage().HumanReadableString()))
return nil
}
art = append(art, artifact)
return nil
}
err := filepath.Walk(path, ff)
if err != nil {
return nil, err
}
return art, nil
}
func (r *LuetSystemRepository) SetPriority(n int) {
r.LuetRepository.Priority = n
}
func (r *LuetSystemRepository) initialize(src string) error {
generator, err := r.getGenerator()
if err != nil {
return errors.Wrap(err, "while constructing repository generator")
}
art, err := generator.Initialize(src, r.Tree.GetDatabase())
if err != nil {
return errors.Wrap(err, "while initializing repository generator")
}
// update the repository index
r.Index = art
return nil
}
// FileSearch search a pattern among the artifacts in a repository
func (r *LuetSystemRepository) FileSearch(pattern string) (pkg.Packages, error) {
var matches pkg.Packages
@ -561,339 +453,37 @@ func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repos
return repo, err
}
func (r *LuetSystemRepository) genLocalRepo(dst string, resetRevision bool) error {
err := os.MkdirAll(dst, os.ModePerm)
if err != nil {
return err
}
r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
repospec := filepath.Join(dst, REPOSITORY_SPECFILE)
if resetRevision {
r.Revision = 0
} else {
if _, err := os.Stat(repospec); !os.IsNotExist(err) {
// Read existing file for retrieve revision
spec, err := r.ReadSpecFile(repospec, false)
if err != nil {
return err
}
r.Revision = spec.GetRevision()
}
}
r.Revision++
Info(fmt.Sprintf(
"For repository %s creating revision %d and last update %s...",
r.Name, r.Revision, r.LastUpdate,
))
bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
Repo LuetSystemRepository
Path string
}{
Repo: *r,
Path: dst,
})
// Create tree and repository file
archive, err := config.LuetCfg.GetSystem().TempDir("archive")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for archive")
}
defer os.RemoveAll(archive) // clean up
err = r.GetTree().Save(archive)
if err != nil {
return errors.Wrap(err, "Error met while saving the tree")
type RepositoryGenerator interface {
Generate(*LuetSystemRepository, string, bool) error
Initialize(string, pkg.PackageDatabase) ([]compiler.Artifact, error)
}
treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY)
if err != nil {
treeFile = NewDefaultTreeRepositoryFile()
r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
func (r *LuetSystemRepository) getGenerator() (RepositoryGenerator, error) {
var rg RepositoryGenerator
switch r.GetType() {
case DiskRepositoryType, HttpRepositoryType:
rg = &localRepositoryGenerator{}
case DockerRepositoryType:
rg = &dockerRepositoryGenerator{
b: r.Backend,
imagePrefix: r.imagePrefix,
imagePush: r.PushImages,
force: r.ForcePush,
}
a := compiler.NewPackageArtifact(filepath.Join(dst, treeFile.GetFileName()))
a.SetCompressionType(treeFile.GetCompressionType())
err = a.Compress(archive, 1)
if err != nil {
return errors.Wrap(err, "Error met while creating package archive")
default:
return nil, errors.New("invalid repository type")
}
// Update the tree name with the name created by compression selected.
treeFile.SetFileName(path.Base(a.GetPath()))
err = a.Hash()
if err != nil {
return errors.Wrap(err, "Failed generating checksums for tree")
}
treeFile.SetChecksums(a.GetChecksums())
r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
// Create Metadata struct and serialized repository
meta, serialized := r.Serialize()
// Create metadata file and repository file
metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
defer os.RemoveAll(metaTmpDir) // clean up
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for metadata")
}
metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY)
if err != nil {
metaFile = NewDefaultMetaRepositoryFile()
r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
}
repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE)
// Create repository.meta.yaml file
err = meta.WriteFile(repoMetaSpec)
if err != nil {
return err
}
a = compiler.NewPackageArtifact(filepath.Join(dst, metaFile.GetFileName()))
a.SetCompressionType(metaFile.GetCompressionType())
err = a.Compress(metaTmpDir, 1)
if err != nil {
return errors.Wrap(err, "Error met while archiving repository metadata")
}
metaFile.SetFileName(path.Base(a.GetPath()))
r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
err = a.Hash()
if err != nil {
return errors.Wrap(err, "Failed generating checksums for metadata")
}
metaFile.SetChecksums(a.GetChecksums())
data, err := yaml.Marshal(serialized)
if err != nil {
return err
}
err = ioutil.WriteFile(repospec, data, os.ModePerm)
if err != nil {
return err
}
bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
Repo LuetSystemRepository
Path string
}{
Repo: *r,
Path: dst,
})
return nil
}
func (r *LuetSystemRepository) genDockerRepo(imagePrefix string, resetRevision, force bool) error {
// - Iterate over meta, build final images, push them if necessary
// - while pushing, check if image already exists, and if exist push them only if --force is supplied
// - Generate final images for metadata and push
imageRepository := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
repoTemp, err := config.LuetCfg.GetSystem().TempDir("repo")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for repository")
}
defer os.RemoveAll(repoTemp) // clean up
if r.GetBackend().ImageAvailable(imageRepository) {
if err := r.GetBackend().DownloadImage(compiler.CompilerBackendOptions{ImageName: imageRepository}); err != nil {
return errors.Wrapf(err, "while downloading '%s'", imageRepository)
}
if err := r.GetBackend().ExtractRootfs(compiler.CompilerBackendOptions{ImageName: imageRepository, Destination: repoTemp}, false); err != nil {
return errors.Wrapf(err, "while extracting '%s'", imageRepository)
}
}
repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE)
if resetRevision {
r.Revision = 0
} else {
if _, err := os.Stat(repospec); !os.IsNotExist(err) {
// Read existing file for retrieve revision
spec, err := r.ReadSpecFile(repospec, false)
if err != nil {
return err
}
r.Revision = spec.GetRevision()
}
}
r.Revision++
Info(fmt.Sprintf(
"For repository %s creating revision %d and last update %s...",
r.Name, r.Revision, r.LastUpdate,
))
bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
Repo LuetSystemRepository
Path string
}{
Repo: *r,
Path: imageRepository,
})
// Create tree and repository file
archive, err := config.LuetCfg.GetSystem().TempDir("archive")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for archive")
}
defer os.RemoveAll(archive) // clean up
err = r.GetTree().Save(archive)
if err != nil {
return errors.Wrap(err, "Error met while saving the tree")
}
treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY)
if err != nil {
treeFile = NewDefaultTreeRepositoryFile()
r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
}
a := compiler.NewPackageArtifact(filepath.Join(repoTemp, treeFile.GetFileName()))
a.SetCompressionType(treeFile.GetCompressionType())
err = a.Compress(archive, 1)
if err != nil {
return errors.Wrap(err, "Error met while creating package archive")
}
// Update the tree name with the name created by compression selected.
treeFile.SetFileName(a.GetFileName())
err = a.Hash()
if err != nil {
return errors.Wrap(err, "Failed generating checksums for tree")
}
treeFile.SetChecksums(a.GetChecksums())
r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
// we generate a new archive containing the required compressed file.
// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
treeArchive, err := compiler.CreateArtifactForFile(a.GetPath())
if err != nil {
return errors.Wrap(err, "Failed generating checksums for tree")
}
imageTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName())
Debug("Generating image", imageTree)
if opts, err := treeArchive.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil {
return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
}
if r.PushImages {
if err := pushImage(r.GetBackend(), imageTree, true); err != nil {
return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree)
}
}
// Create Metadata struct and serialized repository
meta, serialized := r.Serialize()
// Create metadata file and repository file
metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for metadata")
}
defer os.RemoveAll(metaTmpDir) // clean up
metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY)
if err != nil {
metaFile = NewDefaultMetaRepositoryFile()
r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
}
repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE)
// Create repository.meta.yaml file
err = meta.WriteFile(repoMetaSpec)
if err != nil {
return err
}
// create temp dir for metafile
metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for metadata")
}
defer os.RemoveAll(metaDir) // clean up
a = compiler.NewPackageArtifact(filepath.Join(metaDir, metaFile.GetFileName()))
a.SetCompressionType(metaFile.GetCompressionType())
err = a.Compress(metaTmpDir, 1)
if err != nil {
return errors.Wrap(err, "Error met while archiving repository metadata")
}
metaFile.SetFileName(a.GetFileName())
r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
err = a.Hash()
if err != nil {
return errors.Wrap(err, "Failed generating checksums for metadata")
}
metaFile.SetChecksums(a.GetChecksums())
// Files are downloaded as-is from docker images
// we generate a new archive containing the required compressed file.
// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
metaArchive, err := compiler.CreateArtifactForFile(a.GetPath())
if err != nil {
return errors.Wrap(err, "Failed generating checksums for tree")
}
imageMetaTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName())
if opts, err := metaArchive.GenerateFinalImage(imageMetaTree, r.GetBackend(), false); err != nil {
return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
}
if r.PushImages {
if err := pushImage(r.GetBackend(), imageMetaTree, true); err != nil {
return errors.Wrapf(err, "Failed while pushing image: '%s'", imageMetaTree)
}
}
data, err := yaml.Marshal(serialized)
if err != nil {
return err
}
err = ioutil.WriteFile(repospec, data, os.ModePerm)
if err != nil {
return err
}
tempRepoFile := filepath.Join(metaDir, REPOSITORY_SPECFILE+".tar")
if err := helpers.Tar(repospec, tempRepoFile); err != nil {
return errors.Wrap(err, "Error met while archiving repository file")
}
a = compiler.NewPackageArtifact(tempRepoFile)
imageRepo := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
if opts, err := a.GenerateFinalImage(imageRepo, r.GetBackend(), false); err != nil {
return errors.Wrap(err, "Failed generating repository image"+opts.ImageName)
}
if r.PushImages {
if err := pushImage(r.GetBackend(), imageRepo, true); err != nil {
return errors.Wrapf(err, "Failed while pushing image: '%s'", imageRepo)
}
}
bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
Repo LuetSystemRepository
Path string
}{
Repo: *r,
Path: imagePrefix,
})
return nil
return rg, nil
}
// Write writes the repository metadata to the supplied destination
func (r *LuetSystemRepository) Write(dst string, resetRevision, force bool) error {
switch r.GetType() {
case DiskRepositoryType, HttpRepositoryType:
return r.genLocalRepo(dst, resetRevision)
case DockerRepositoryType:
return r.genDockerRepo(dst, resetRevision, force)
rg, err := r.getGenerator()
if err != nil {
return err
}
return errors.New("invalid repository type")
return rg.Generate(r, dst, resetRevision)
}
func (r *LuetSystemRepository) Client() Client {

313
pkg/installer/repository_docker.go Normal file
View File

@ -0,0 +1,313 @@
// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package installer
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"time"
. "github.com/mudler/luet/pkg/logger"
pkg "github.com/mudler/luet/pkg/package"
"github.com/ghodss/yaml"
"github.com/mudler/luet/pkg/bus"
compiler "github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
"github.com/pkg/errors"
)
type dockerRepositoryGenerator struct {
b compiler.CompilerBackend
imagePrefix string
imagePush, force bool
}
func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) {
return generatePackageImages(l.b, l.imagePrefix, path, db, l.imagePush, l.force)
}
func pushImage(b compiler.CompilerBackend, image string, force bool) error {
if b.ImageAvailable(image) && !force {
Debug("Image", image, "already present, skipping")
return nil
}
return b.Push(compiler.CompilerBackendOptions{ImageName: image})
}
func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]compiler.Artifact, error) {
Info("Generating docker images for packages in", imagePrefix)
var art []compiler.Artifact
var ff = func(currentpath string, info os.FileInfo, err error) error {
if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
return nil // Skip with no errors
}
dat, err := ioutil.ReadFile(currentpath)
if err != nil {
return errors.Wrap(err, "Error reading file "+currentpath)
}
artifact, err := compiler.NewPackageArtifactFromYaml(dat)
if err != nil {
return errors.Wrap(err, "Error reading yaml "+currentpath)
}
// Set the path relative to the file.
// The metadata contains the full path where the file was located during buildtime.
artifact.SetPath(filepath.Join(filepath.Dir(currentpath), filepath.Base(artifact.GetPath())))
// We want to include packages that are ONLY referenced in the tree.
// the ones which aren't should be deleted. (TODO: by another cli command?)
if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
artifact.GetCompileSpec().GetPackage().HumanReadableString()))
return nil
}
packageImage := fmt.Sprintf("%s:%s", imagePrefix, artifact.GetCompileSpec().GetPackage().ImageID())
if imagePush && b.ImageAvailable(packageImage) && !force {
Info("Image", packageImage, "already present, skipping. use --force-push to override")
} else {
Info("Generating final image", packageImage,
"for package ", artifact.GetCompileSpec().GetPackage().HumanReadableString())
if opts, err := artifact.GenerateFinalImage(packageImage, b, true); err != nil {
return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
}
}
if imagePush {
if err := pushImage(b, packageImage, force); err != nil {
return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage)
}
}
art = append(art, artifact)
return nil
}
err := filepath.Walk(path, ff)
if err != nil {
return nil, err
}
return art, nil
}
func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefix string, resetRevision bool) error {
// - Iterate over meta, build final images, push them if necessary
// - while pushing, check if image already exists, and if exist push them only if --force is supplied
// - Generate final images for metadata and push
imageRepository := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
repoTemp, err := config.LuetCfg.GetSystem().TempDir("repo")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for repository")
}
defer os.RemoveAll(repoTemp) // clean up
if r.GetBackend().ImageAvailable(imageRepository) {
if err := r.GetBackend().DownloadImage(compiler.CompilerBackendOptions{ImageName: imageRepository}); err != nil {
return errors.Wrapf(err, "while downloading '%s'", imageRepository)
}
if err := r.GetBackend().ExtractRootfs(compiler.CompilerBackendOptions{ImageName: imageRepository, Destination: repoTemp}, false); err != nil {
return errors.Wrapf(err, "while extracting '%s'", imageRepository)
}
}
repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE)
if resetRevision {
r.Revision = 0
} else {
if _, err := os.Stat(repospec); !os.IsNotExist(err) {
// Read existing file for retrieve revision
spec, err := r.ReadSpecFile(repospec, false)
if err != nil {
return err
}
r.Revision = spec.GetRevision()
}
}
r.Revision++
Info(fmt.Sprintf(
"For repository %s creating revision %d and last update %s...",
r.Name, r.Revision, r.LastUpdate,
))
bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
Repo LuetSystemRepository
Path string
}{
Repo: *r,
Path: imageRepository,
})
// Create tree and repository file
archive, err := config.LuetCfg.GetSystem().TempDir("archive")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for archive")
}
defer os.RemoveAll(archive) // clean up
err = r.GetTree().Save(archive)
if err != nil {
return errors.Wrap(err, "Error met while saving the tree")
}
treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY)
if err != nil {
treeFile = NewDefaultTreeRepositoryFile()
r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
}
a := compiler.NewPackageArtifact(filepath.Join(repoTemp, treeFile.GetFileName()))
a.SetCompressionType(treeFile.GetCompressionType())
err = a.Compress(archive, 1)
if err != nil {
return errors.Wrap(err, "Error met while creating package archive")
}
// Update the tree name with the name created by compression selected.
treeFile.SetFileName(a.GetFileName())
err = a.Hash()
if err != nil {
return errors.Wrap(err, "Failed generating checksums for tree")
}
treeFile.SetChecksums(a.GetChecksums())
r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
// we generate a new archive containing the required compressed file.
// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
treeArchive, err := compiler.CreateArtifactForFile(a.GetPath())
if err != nil {
return errors.Wrap(err, "Failed generating checksums for tree")
}
imageTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName())
Debug("Generating image", imageTree)
if opts, err := treeArchive.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil {
return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
}
if r.PushImages {
if err := pushImage(r.GetBackend(), imageTree, true); err != nil {
return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree)
}
}
// Create Metadata struct and serialized repository
meta, serialized := r.Serialize()
// Create metadata file and repository file
metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for metadata")
}
defer os.RemoveAll(metaTmpDir) // clean up
metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY)
if err != nil {
metaFile = NewDefaultMetaRepositoryFile()
r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
}
repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE)
// Create repository.meta.yaml file
err = meta.WriteFile(repoMetaSpec)
if err != nil {
return err
}
// create temp dir for metafile
metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for metadata")
}
defer os.RemoveAll(metaDir) // clean up
a = compiler.NewPackageArtifact(filepath.Join(metaDir, metaFile.GetFileName()))
a.SetCompressionType(metaFile.GetCompressionType())
err = a.Compress(metaTmpDir, 1)
if err != nil {
return errors.Wrap(err, "Error met while archiving repository metadata")
}
metaFile.SetFileName(a.GetFileName())
r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
err = a.Hash()
if err != nil {
return errors.Wrap(err, "Failed generating checksums for metadata")
}
metaFile.SetChecksums(a.GetChecksums())
// Files are downloaded as-is from docker images
// we generate a new archive containing the required compressed file.
// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
metaArchive, err := compiler.CreateArtifactForFile(a.GetPath())
if err != nil {
return errors.Wrap(err, "Failed generating checksums for tree")
}
imageMetaTree := fmt.Sprintf("%s:%s", imagePrefix, a.GetFileName())
if opts, err := metaArchive.GenerateFinalImage(imageMetaTree, r.GetBackend(), false); err != nil {
return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
}
if r.PushImages {
if err := pushImage(r.GetBackend(), imageMetaTree, true); err != nil {
return errors.Wrapf(err, "Failed while pushing image: '%s'", imageMetaTree)
}
}
data, err := yaml.Marshal(serialized)
if err != nil {
return err
}
err = ioutil.WriteFile(repospec, data, os.ModePerm)
if err != nil {
return err
}
tempRepoFile := filepath.Join(metaDir, REPOSITORY_SPECFILE+".tar")
if err := helpers.Tar(repospec, tempRepoFile); err != nil {
return errors.Wrap(err, "Error met while archiving repository file")
}
a = compiler.NewPackageArtifact(tempRepoFile)
imageRepo := fmt.Sprintf("%s:%s", imagePrefix, REPOSITORY_SPECFILE)
if opts, err := a.GenerateFinalImage(imageRepo, r.GetBackend(), false); err != nil {
return errors.Wrap(err, "Failed generating repository image"+opts.ImageName)
}
if r.PushImages {
if err := pushImage(r.GetBackend(), imageRepo, true); err != nil {
return errors.Wrapf(err, "Failed while pushing image: '%s'", imageRepo)
}
}
bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
Repo LuetSystemRepository
Path string
}{
Repo: *r,
Path: imagePrefix,
})
return nil
}

208
pkg/installer/repository_local.go Normal file
View File

@ -0,0 +1,208 @@
// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package installer
import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
. "github.com/mudler/luet/pkg/logger"
pkg "github.com/mudler/luet/pkg/package"
"github.com/ghodss/yaml"
"github.com/mudler/luet/pkg/bus"
compiler "github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/config"
"github.com/pkg/errors"
)
type localRepositoryGenerator struct{}
func (l *localRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) {
return buildPackageIndex(path, db)
}
func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) {
var art []compiler.Artifact
var ff = func(currentpath string, info os.FileInfo, err error) error {
if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
return nil // Skip with no errors
}
dat, err := ioutil.ReadFile(currentpath)
if err != nil {
return errors.Wrap(err, "Error reading file "+currentpath)
}
artifact, err := compiler.NewPackageArtifactFromYaml(dat)
if err != nil {
return errors.Wrap(err, "Error reading yaml "+currentpath)
}
// We want to include packages that are ONLY referenced in the tree.
// the ones which aren't should be deleted. (TODO: by another cli command?)
if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
artifact.GetCompileSpec().GetPackage().HumanReadableString()))
return nil
}
art = append(art, artifact)
return nil
}
err := filepath.Walk(path, ff)
if err != nil {
return nil, err
}
return art, nil
}
func (*localRepositoryGenerator) Generate(r *LuetSystemRepository, dst string, resetRevision bool) error {
err := os.MkdirAll(dst, os.ModePerm)
if err != nil {
return err
}
r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
repospec := filepath.Join(dst, REPOSITORY_SPECFILE)
if resetRevision {
r.Revision = 0
} else {
if _, err := os.Stat(repospec); !os.IsNotExist(err) {
// Read existing file for retrieve revision
spec, err := r.ReadSpecFile(repospec, false)
if err != nil {
return err
}
r.Revision = spec.GetRevision()
}
}
r.Revision++
Info(fmt.Sprintf(
"For repository %s creating revision %d and last update %s...",
r.Name, r.Revision, r.LastUpdate,
))
bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
Repo LuetSystemRepository
Path string
}{
Repo: *r,
Path: dst,
})
// Create tree and repository file
archive, err := config.LuetCfg.GetSystem().TempDir("archive")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for archive")
}
defer os.RemoveAll(archive) // clean up
err = r.GetTree().Save(archive)
if err != nil {
return errors.Wrap(err, "Error met while saving the tree")
}
treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY)
if err != nil {
treeFile = NewDefaultTreeRepositoryFile()
r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
}
a := compiler.NewPackageArtifact(filepath.Join(dst, treeFile.GetFileName()))
a.SetCompressionType(treeFile.GetCompressionType())
err = a.Compress(archive, 1)
if err != nil {
return errors.Wrap(err, "Error met while creating package archive")
}
// Update the tree name with the name created by compression selected.
treeFile.SetFileName(path.Base(a.GetPath()))
err = a.Hash()
if err != nil {
return errors.Wrap(err, "Failed generating checksums for tree")
}
treeFile.SetChecksums(a.GetChecksums())
r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
// Create Metadata struct and serialized repository
meta, serialized := r.Serialize()
// Create metadata file and repository file
metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
defer os.RemoveAll(metaTmpDir) // clean up
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for metadata")
}
metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY)
if err != nil {
metaFile = NewDefaultMetaRepositoryFile()
r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
}
repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE)
// Create repository.meta.yaml file
err = meta.WriteFile(repoMetaSpec)
if err != nil {
return err
}
a = compiler.NewPackageArtifact(filepath.Join(dst, metaFile.GetFileName()))
a.SetCompressionType(metaFile.GetCompressionType())
err = a.Compress(metaTmpDir, 1)
if err != nil {
return errors.Wrap(err, "Error met while archiving repository metadata")
}
metaFile.SetFileName(path.Base(a.GetPath()))
r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
err = a.Hash()
if err != nil {
return errors.Wrap(err, "Failed generating checksums for metadata")
}
metaFile.SetChecksums(a.GetChecksums())
data, err := yaml.Marshal(serialized)
if err != nil {
return err
}
err = ioutil.WriteFile(repospec, data, os.ModePerm)
if err != nil {
return err
}
bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
Repo LuetSystemRepository
Path string
}{
Repo: *r,
Path: dst,
})
return nil
}

2
pkg/installer/repository_test.go
View File

@ -1,4 +1,4 @@
// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
// Copyright © 2019 Ettore Di Giacinto <mudler@sabayon.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by