kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-09-02 07:45:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow to pull specfiles from published repositories

Browse Source 
- Interpolates values from the repositories compilespec if present
- Automatically merge cache images coming from specified repository when
  necessary

Fixes #194
This commit is contained in:
Ettore Di Giacinto
2021-04-14 16:49:43 +02:00
parent 57c769b4a5
commit 7ba7add2a8
11 changed files with 333 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
cmd/build.go
View File

@@ -16,7 +16,6 @@ package cmd
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
@@ -25,6 +24,7 @@ import (
"github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/compiler/types/artifact"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
"github.com/mudler/luet/pkg/installer"
"github.com/mudler/luet/pkg/compiler/types/compression"
"github.com/mudler/luet/pkg/compiler/types/options"
@@ -44,17 +44,17 @@ var buildCmd = &cobra.Command{
Long: `Builds one or more packages from a tree (current directory is implied):
$ luet build utils/busybox utils/yq ...
Builds all packages
$ luet build --all
Builds only the leaf packages:
$ luet build --full
Build package revdeps:
$ luet build --revdeps utils/yq
Build package without dependencies (needs the images already in the host, or either need to be available online):
@@ -69,7 +69,6 @@ Build packages specifying multiple definition trees:
viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged"))
viper.BindPFlag("database", cmd.Flags().Lookup("database"))
viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
viper.BindPFlag("all", cmd.Flags().Lookup("all"))
viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
@@ -101,7 +100,6 @@ Build packages specifying multiple definition trees:
privileged := viper.GetBool("privileged")
revdeps := viper.GetBool("revdeps")
all := viper.GetBool("all")
databaseType := viper.GetString("database")
compressionType := viper.GetString("compression")
imageRepository := viper.GetString("image-repository")
values := viper.GetStringSlice("values")
@@ -122,26 +120,25 @@ Build packages specifying multiple definition trees:
LuetCfg.GetLogging().SetLogLevel("error")
}
pretend, _ := cmd.Flags().GetBool("pretend")
fromRepo, _ := cmd.Flags().GetBool("from-repositories")
compilerSpecs := compilerspec.NewLuetCompilationspecs()
var db pkg.PackageDatabase
compilerBackend, err := compiler.NewBackend(backendType)
helpers.CheckErr(err)
switch databaseType {
case "memory":
db = pkg.NewInMemoryDatabase(false)
case "boltdb":
tmpdir, err := ioutil.TempDir("", "package")
helpers.CheckErr(err)
db = pkg.NewBoltDatabase(tmpdir)
}
db = pkg.NewInMemoryDatabase(false)
defer db.Clean()
generalRecipe := tree.NewCompilerRecipe(db)
if fromRepo {
if err := installer.LoadBuildTree(generalRecipe, db, LuetCfg); err != nil {
Warning("errors while loading trees from repositories", err.Error())
}
}
for _, src := range treePaths {
Info("Loading tree", src)
helpers.CheckErr(generalRecipe.Load(src))
@@ -202,7 +199,6 @@ Build packages specifying multiple definition trees:
}
} else if !all {
for _, a := range args {
pack, err := helpers.ParsePackageStr(a)
if err != nil {
Fatal("Invalid package string ", a, ": ", err.Error())
@@ -310,7 +306,6 @@ func init() {
buildCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the tree to use.")
buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)")
buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
@@ -333,6 +328,7 @@ func init() {
buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
buildCmd.Flags().Bool("live-output", LuetCfg.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
buildCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
buildCmd.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")

11
cmd/install.go
View File

@@ -70,16 +70,6 @@ To force install a package:
toInstall = append(toInstall, pack)
}
// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
repos := installer.Repositories{}
for _, repo := range LuetCfg.SystemRepositories {
if !repo.Enable {
continue
}
r := installer.NewSystemRepository(repo)
repos = append(repos, r)
}
stype := LuetCfg.Viper.GetString("solver.type")
discount := LuetCfg.Viper.GetFloat64("solver.discount")
rate := LuetCfg.Viper.GetFloat64("solver.rate")
@@ -111,6 +101,7 @@ To force install a package:
}
Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
repos := installer.SystemRepositories(LuetCfg)
// Load config protect configs
installer.LoadConfigProtectConfs(LuetCfg)

4
go.mod
View File

@@ -20,8 +20,10 @@ require (
github.com/genuinetools/img v0.5.11
github.com/ghodss/yaml v1.0.0
github.com/google/go-containerregistry v0.2.1
github.com/google/renameio v1.0.0
github.com/hashicorp/go-multierror v1.0.0
github.com/hashicorp/go-version v1.2.1
github.com/imdario/mergo v0.3.8
github.com/jedib0t/go-pretty v4.3.0+incompatible
github.com/jedib0t/go-pretty/v6 v6.0.5
github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
@@ -51,7 +53,7 @@ require (
github.com/theupdateframework/notary v0.7.0
go.etcd.io/bbolt v1.3.5
go.uber.org/atomic v1.5.1 // indirect
go.uber.org/multierr v1.4.0 // indirect
go.uber.org/multierr v1.4.0
go.uber.org/zap v1.13.0
google.golang.org/grpc v1.29.1
gopkg.in/yaml.v2 v2.3.0

2
go.sum
View File

@@ -493,6 +493,8 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/renameio v1.0.0 h1:xhp2CnJmgQmpJU4RY8chagahUq5mbPPAbiSQstKpVMA=
github.com/google/renameio v1.0.0/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk=
github.com/google/shlex v0.0.0-20150127133951-6f45313302b9 h1:JM174NTeGNJ2m/oLH3UOWOvWQQKd+BoL3hcSCUWFLt0=
github.com/google/shlex v0.0.0-20150127133951-6f45313302b9/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

99
pkg/compiler/compiler.go
View File

@@ -27,6 +27,7 @@ import (
"sync"
"time"
"github.com/imdario/mergo"
bus "github.com/mudler/luet/pkg/bus"
"github.com/mudler/luet/pkg/compiler/backend"
artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
@@ -37,6 +38,7 @@ import (
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/solver"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
const BuildFile = "build.yaml"
@@ -48,19 +50,11 @@ type ArtifactIndex []*artifact.PackageArtifact
func (i ArtifactIndex) CleanPath() ArtifactIndex {
newIndex := ArtifactIndex{}
for _, art := range i {
// FIXME: This is a dup and makes difficult to add attributes to artifacts
newIndex = append(newIndex, &artifact.PackageArtifact{
Path: path.Base(art.Path),
SourceAssertion: art.SourceAssertion,
CompileSpec: art.CompileSpec,
Dependencies: art.Dependencies,
CompressionType: art.CompressionType,
Checksums: art.Checksums,
Files: art.Files,
})
copy := art.ShallowCopy()
copy.Path = path.Base(art.Path)
newIndex = append(newIndex, copy)
}
return newIndex
//Update if exists, otherwise just create
}
type LuetCompiler struct {
@@ -544,7 +538,7 @@ func (cs *LuetCompiler) resolveExistingImageHash(imageHash string) string {
}
func LoadArtifactFromYaml(spec *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml"
metaFile := spec.GetPackage().GetMetadataFilePath()
dat, err := ioutil.ReadFile(spec.Rel(metaFile))
if err != nil {
return nil, errors.Wrap(err, "Error reading file "+metaFile)
@@ -731,6 +725,12 @@ func genImageList(refs []string, hash string) []string {
}
func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
if len(p.BuildOptions.PullImageRepository) != 0 {
orig := cs.Options.PullImageRepository
cs.Options.PullImageRepository = append(orig, p.BuildOptions.PullImageRepository...)
defer func() { cs.Options.PullImageRepository = orig }()
}
Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
@@ -866,7 +866,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
type templatedata map[string]interface{}
func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) {
func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.Package) ([]byte, error) {
var dataresult []byte
val := pack.Rel(DefinitionFile)
@@ -876,7 +876,7 @@ func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) {
if err != nil {
return nil, errors.Wrap(err, "unmarshalling values")
}
cs.Options.BuildValues = []map[string]interface{}{(map[string]interface{})(dst)}
cs.Options.BuildValues = append(vals, (map[string]interface{})(dst))
if _, err := os.Stat(pack.Rel(CollectionFile)); err == nil {
val = pack.Rel(CollectionFile)
@@ -897,14 +897,46 @@ func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) {
}
raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
td := templatedata{}
if len(vals) > 0 {
for _, bv := range vals {
current := templatedata(bv)
if err := mergo.Merge(&td, current); err != nil {
return nil, errors.Wrap(err, "merging values maps")
}
}
}
dat, err := helpers.RenderHelm(string(dataBuild), raw, dst)
if err := mergo.Merge(&td, raw); err != nil {
return nil, errors.Wrap(err, "merging values maps")
}
dat, err := helpers.RenderHelm(string(dataBuild), td, dst)
if err != nil {
return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
}
dataresult = []byte(dat)
} else {
out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, cs.Options.BuildValuesFile...)
bv := cs.Options.BuildValuesFile
if len(vals) > 0 {
valuesdir, err := ioutil.TempDir("", "genvalues")
if err != nil {
return nil, errors.Wrap(err, "Could not create tempdir")
}
defer os.RemoveAll(valuesdir) // clean up
for _, b := range vals {
out, err := yaml.Marshal(b)
if err != nil {
return nil, errors.Wrap(err, "while marshalling values file")
}
f := filepath.Join(valuesdir, helpers.RandStringRunes(20))
if err := ioutil.WriteFile(f, out, os.ModePerm); err != nil {
return nil, errors.Wrap(err, "while writing temporary values file")
}
bv = append([]string{f}, bv...)
}
}
out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, bv...)
if err != nil {
return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
}
@@ -922,12 +954,43 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio
return nil, err
}
bytes, err := cs.templatePackage(pack)
opts := options.Compiler{}
artifactMetadataFile := filepath.Join(p.GetTreeDir(), p.GetMetadataFilePath())
if fi, err := os.Stat(artifactMetadataFile); err == nil {
f, err := os.Open(fi.Name())
if err != nil {
return nil, errors.Wrapf(err, "could not open %s", fi.Name())
}
dat, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
art, err := artifact.NewPackageArtifactFromYaml(dat)
if err != nil {
return nil, errors.Wrap(err, "could not decode package from yaml")
}
opts = art.CompileSpec.BuildOptions
opts.PushImageRepository = ""
} else if !os.IsNotExist(err) {
Debug("error reading already existing artifact metadata file: ", err.Error())
}
bytes, err := cs.templatePackage(opts.BuildValues, pack)
if err != nil {
return nil, errors.Wrap(err, "while rendering package template")
}
return compilerspec.NewLuetCompilationSpec(bytes, pack)
newSpec, err := compilerspec.NewLuetCompilationSpec(bytes, pack)
if err != nil {
return nil, err
}
newSpec.BuildOptions = opts
return newSpec, err
}
// GetBackend returns the current compilation backend

7
pkg/compiler/types/artifact/artifact.go
View File

@@ -62,6 +62,11 @@ type PackageArtifact struct {
Files []string `json:"files"`
}
func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
copy := *p
return &copy
}
func NewPackageArtifact(path string) *PackageArtifact {
return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: compression.None}
}
@@ -129,7 +134,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
}
err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm)
err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetMetadataFilePath()), data, os.ModePerm)
if err != nil {
return errors.Wrap(err, "While writing PackageArtifact YAML")
}

32
pkg/helpers/file.go
View File

@@ -19,6 +19,7 @@ import (
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"sort"
@@ -26,10 +27,41 @@ import (
"syscall"
"time"
"github.com/google/renameio"
copy "github.com/otiai10/copy"
"github.com/pkg/errors"
)
var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
func RandStringRunes(n int) string {
b := make([]rune, n)
for i := range b {
b[i] = letterRunes[rand.Intn(len(letterRunes))]
}
return string(b)
}
func Move(src, dst string) error {
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
t, err := renameio.TempFile("", dst)
if err != nil {
return err
}
defer t.Cleanup()
_, err = io.Copy(t, f)
if err != nil {
return err
}
return t.CloseAtomicallyReplace()
}
func OrderFiles(target string, files []string) ([]string, []string) {
var newFiles []string

185
pkg/installer/repository.go
View File

@@ -16,6 +16,7 @@
package installer
import (
"fmt"
"io/ioutil"
"os"
"path"
@@ -27,6 +28,7 @@ import (
artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
compression "github.com/mudler/luet/pkg/compiler/types/compression"
"go.uber.org/multierr"
"github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/config"
@@ -100,6 +102,40 @@ func NewLuetSystemRepositoryMetadata(file string, removeFile bool) (*LuetSystemR
return ans, nil
}
// SystemRepositories returns the repositories from the local configuration file
func SystemRepositories(c *config.LuetConfig) Repositories {
repos := Repositories{}
for _, repo := range c.SystemRepositories {
if !repo.Enable {
continue
}
r := NewSystemRepository(repo)
repos = append(repos, r)
}
return repos
}
// LoadBuildTree loads to the tree the compilation specs from the system repositories
func LoadBuildTree(t tree.Builder, db pkg.PackageDatabase, c *config.LuetConfig) error {
var reserr error
repos := SystemRepositories(c)
for _, r := range repos {
repodir, err := config.LuetCfg.GetSystem().TempDir(r.Name)
if err != nil {
reserr = multierr.Append(reserr, err)
}
if err := r.SyncBuildMetadata(repodir); err != nil {
reserr = multierr.Append(reserr, err)
}
if err := t.Load(filepath.Join(repodir, "tree")); err != nil {
reserr = multierr.Append(reserr, err)
}
}
repos.SyncDatabase(db)
return reserr
}
func (m *LuetSystemRepositoryMetadata) WriteFile(path string) error {
data, err := yaml.Marshal(m)
if err != nil {
@@ -471,7 +507,7 @@ func (r *LuetSystemRepository) AddRepositoryFile(src, fileKey, repositoryRoot st
treeFile, err := r.GetRepositoryFile(fileKey)
if err != nil {
treeFile = defaults
r.SetRepositoryFile(fileKey, treeFile)
// r.SetRepositoryFile(fileKey, treeFile)
}
a := artifact.NewPackageArtifact(filepath.Join(repositoryRoot, treeFile.GetFileName()))
@@ -594,6 +630,70 @@ func (r *LuetSystemRepository) SearchArtefact(p pkg.Package) (*artifact.PackageA
return nil, errors.New("Not found")
}
func (r *LuetSystemRepository) getRepoFile(c Client, key string) (*artifact.PackageArtifact, error) {
treeFile, err := r.GetRepositoryFile(key)
if err != nil {
return nil, errors.Wrapf(err, "key %s not present in the repository", key)
}
// Get Tree
downloadedTreeFile, err := c.DownloadFile(treeFile.GetFileName())
if err != nil {
return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName())
}
//defer os.Remove(downloadedTreeFile)
treeFileArtifact := artifact.NewPackageArtifact(downloadedTreeFile)
treeFileArtifact.Checksums = treeFile.GetChecksums()
treeFileArtifact.CompressionType = treeFile.GetCompressionType()
err = treeFileArtifact.Verify()
if err != nil {
return nil, errors.Wrap(err, "file integrity check failure")
}
return treeFileArtifact, nil
}
func (r *LuetSystemRepository) SyncBuildMetadata(path string) error {
repo, err := r.Sync(false)
if err != nil {
return errors.Wrap(err, "while syncronizing repository")
}
c := repo.Client()
if c == nil {
return errors.New("no client could be generated from repository")
}
a, err := repo.getRepoFile(c, REPOFILE_COMPILER_TREE_KEY)
if err != nil {
return fmt.Errorf("failed while getting: %s", REPOFILE_COMPILER_TREE_KEY)
}
defer os.RemoveAll(a.Path)
if err := a.Unpack(filepath.Join(path, "tree"), false); err != nil {
return errors.Wrapf(err, "while unpacking: %s", REPOFILE_COMPILER_TREE_KEY)
}
for _, ai := range repo.GetTree().GetDatabase().World() {
// Retrieve remote repository.yaml for retrieve revision and date
file, err := c.DownloadFile(ai.GetMetadataFilePath())
if err != nil {
return errors.Wrapf(err, "while downloading metadata for %s", ai.HumanReadableString())
}
if err := helpers.Move(file, filepath.Join(path, ai.GetMetadataFilePath())); err != nil {
return err
}
}
return nil
}
func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
var repoUpdated bool = false
var treefs, metafs string
@@ -612,7 +712,7 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
}
repobasedir := config.LuetCfg.GetSystem().GetRepoDatabaseDirPath(r.GetName())
repo, err := r.ReadSpecFile(file)
downloadedRepoMeta, err := r.ReadSpecFile(file)
if err != nil {
return nil, err
}
@@ -624,8 +724,8 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
if !force {
localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir, REPOSITORY_SPECFILE))
if localRepo != nil {
if localRepo.GetRevision() == repo.GetRevision() &&
localRepo.GetLastUpdate() == repo.GetLastUpdate() {
if localRepo.GetRevision() == downloadedRepoMeta.GetRevision() &&
localRepo.GetLastUpdate() == downloadedRepoMeta.GetLastUpdate() {
repoUpdated = true
}
}
@@ -652,47 +752,22 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
}
}
// POST: treeFile and metaFile are present. I check this inside
// ReadSpecFile and NewLuetSystemRepositoryFromYaml
treeFile, _ := repo.GetRepositoryFile(REPOFILE_TREE_KEY)
metaFile, _ := repo.GetRepositoryFile(REPOFILE_META_KEY)
// treeFile and metaFile must be present, they aren't optional
if !repoUpdated {
// Get Tree
downloadedTreeFile, err := c.DownloadFile(treeFile.GetFileName())
treeFileArtifact, err := downloadedRepoMeta.getRepoFile(c, REPOFILE_TREE_KEY)
if err != nil {
return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName())
}
defer os.Remove(downloadedTreeFile)
// Treat the file as artifact, in order to verify it
treeFileArtifact := artifact.NewPackageArtifact(downloadedTreeFile)
treeFileArtifact.Checksums = treeFile.GetChecksums()
treeFileArtifact.CompressionType = treeFile.GetCompressionType()
err = treeFileArtifact.Verify()
if err != nil {
return nil, errors.Wrap(err, "Tree integrity check failure")
return nil, errors.Wrapf(err, "while fetching '%s'", REPOFILE_TREE_KEY)
}
defer os.Remove(treeFileArtifact.Path)
Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
// Get Repository Metadata
downloadedMeta, err := c.DownloadFile(metaFile.GetFileName())
metaFileArtifact, err := downloadedRepoMeta.getRepoFile(c, REPOFILE_META_KEY)
if err != nil {
return nil, errors.Wrap(err, "While downloading "+metaFile.GetFileName())
}
defer os.Remove(downloadedMeta)
metaFileArtifact := artifact.NewPackageArtifact(downloadedMeta)
metaFileArtifact.Checksums = metaFile.GetChecksums()
metaFileArtifact.CompressionType = metaFile.GetCompressionType()
err = metaFileArtifact.Verify()
if err != nil {
return nil, errors.Wrap(err, "Metadata integrity check failure")
return nil, errors.Wrapf(err, "while fetching '%s'", REPOFILE_META_KEY)
}
defer os.Remove(metaFileArtifact.Path)
Debug("Metadata tarball for the repository " + r.GetName() + " downloaded correctly.")
@@ -722,17 +797,17 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
return nil, errors.Wrap(err, "Error met while unpacking metadata")
}
tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64)
tsec, _ := strconv.ParseInt(downloadedRepoMeta.GetLastUpdate(), 10, 64)
InfoC(
aurora.Bold(
aurora.Red(":house: Repository "+repo.GetName()+" revision: ")).String() +
aurora.Bold(aurora.Green(repo.GetRevision())).String() + " - " +
aurora.Red(":house: Repository "+downloadedRepoMeta.GetName()+" revision: ")).String() +
aurora.Bold(aurora.Green(downloadedRepoMeta.GetRevision())).String() + " - " +
aurora.Bold(aurora.Green(time.Unix(tsec, 0).String())).String(),
)
} else {
Info("Repository", repo.GetName(), "is already up to date.")
Info("Repository", downloadedRepoMeta.GetName(), "is already up to date.")
}
meta, err := NewLuetSystemRepositoryMetadata(
@@ -741,7 +816,7 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
if err != nil {
return nil, errors.Wrap(err, "While processing "+REPOSITORY_METAFILE)
}
repo.SetIndex(meta.ToArtifactIndex())
downloadedRepoMeta.SetIndex(meta.ToArtifactIndex())
reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
err = reciper.Load(treefs)
@@ -749,29 +824,33 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) {
return nil, errors.Wrap(err, "Error met while unpacking rootfs")
}
repo.SetTree(reciper)
repo.SetTreePath(treefs)
downloadedRepoMeta.SetTree(reciper)
downloadedRepoMeta.SetTreePath(treefs)
// Copy the local available data to the one which was synced
// e.g. locally we can override the type (disk), or priority
// while remotely it could be advertized differently
repo.SetUrls(r.GetUrls())
repo.SetAuthentication(r.GetAuthentication())
repo.SetType(r.GetType())
repo.SetPriority(r.GetPriority())
repo.SetName(r.GetName())
repo.SetVerify(r.GetVerify())
r.fill(downloadedRepoMeta)
InfoC(
aurora.Yellow(":information_source:").String() +
aurora.Magenta("Repository: ").String() +
aurora.Green(aurora.Bold(repo.GetName()).String()).String() +
aurora.Green(aurora.Bold(downloadedRepoMeta.GetName()).String()).String() +
aurora.Magenta(" Priority: ").String() +
aurora.Bold(aurora.Green(repo.GetPriority())).String() +
aurora.Bold(aurora.Green(downloadedRepoMeta.GetPriority())).String() +
aurora.Magenta(" Type: ").String() +
aurora.Bold(aurora.Green(repo.GetType())).String(),
aurora.Bold(aurora.Green(downloadedRepoMeta.GetType())).String(),
)
return repo, nil
return downloadedRepoMeta, nil
}
func (r *LuetSystemRepository) fill(r2 *LuetSystemRepository) {
r2.SetUrls(r.GetUrls())
r2.SetAuthentication(r.GetAuthentication())
r2.SetType(r.GetType())
r2.SetPriority(r.GetPriority())
r2.SetName(r.GetName())
r2.SetVerify(r.GetVerify())
}
func (r *LuetSystemRepository) Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepository) {

63
pkg/installer/repository_docker.go
View File

@@ -44,23 +44,22 @@ type dockerRepositoryGenerator struct {
}
func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
return generatePackageImages(l.b, l.imagePrefix, path, db, l.imagePush, l.force)
}
func pushImage(b compiler.CompilerBackend, image string, force bool) error {
if b.ImageAvailable(image) && !force {
Debug("Image", image, "already present, skipping")
return nil
}
return b.Push(backend.Options{ImageName: image})
}
func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]*artifact.PackageArtifact, error) {
Info("Generating docker images for packages in", imagePrefix)
Info("Generating docker images for packages in", l.imagePrefix)
var art []*artifact.PackageArtifact
var ff = func(currentpath string, info os.FileInfo, err error) error {
if err != nil {
Debug("Skipping", info.Name(), err.Error())
return nil
}
if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
if strings.HasSuffix(info.Name(), ".metadata.yaml") {
a := artifact.NewPackageArtifact(info.Name())
imageRepo := fmt.Sprintf("%s:%s", l.imagePrefix, filepath.Base(info.Name()))
if err := l.pushFileFromArtifact(a, imageRepo); err != nil {
return errors.Wrap(err, "while pushing file from artifact")
}
return nil // Skip with no errors
}
@@ -85,19 +84,19 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string,
return nil
}
packageImage := fmt.Sprintf("%s:%s", imagePrefix, a.CompileSpec.GetPackage().ImageID())
packageImage := fmt.Sprintf("%s:%s", l.imagePrefix, a.CompileSpec.GetPackage().ImageID())
if imagePush && b.ImageAvailable(packageImage) && !force {
if l.imagePush && l.b.ImageAvailable(packageImage) && !l.force {
Info("Image", packageImage, "already present, skipping. use --force-push to override")
} else {
Info("Generating final image", packageImage,
"for package ", a.CompileSpec.GetPackage().HumanReadableString())
if opts, err := a.GenerateFinalImage(packageImage, b, true); err != nil {
if opts, err := a.GenerateFinalImage(packageImage, l.b, true); err != nil {
return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
}
}
if imagePush {
if err := pushImage(b, packageImage, force); err != nil {
if l.imagePush {
if err := pushImage(l.b, packageImage, l.force); err != nil {
return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage)
}
}
@@ -115,13 +114,21 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string,
return art, nil
}
func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string, r *LuetSystemRepository) error {
func pushImage(b compiler.CompilerBackend, image string, force bool) error {
if b.ImageAvailable(image) && !force {
Debug("Image", image, "already present, skipping")
return nil
}
return b.Push(backend.Options{ImageName: image})
}
func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string) error {
Debug("Generating image", imageTree)
if opts, err := a.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil {
if opts, err := a.GenerateFinalImage(imageTree, d.b, false); err != nil {
return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
}
if r.PushImages {
if err := pushImage(r.GetBackend(), imageTree, true); err != nil {
if d.imagePush {
if err := pushImage(d.b, imageTree, true); err != nil {
return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree)
}
}
@@ -144,13 +151,13 @@ func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec string, r *LuetSys
a := artifact.NewPackageArtifact(tempRepoFile)
imageRepo := fmt.Sprintf("%s:%s", d.imagePrefix, REPOSITORY_SPECFILE)
if err := d.pushFileFromArtifact(a, imageRepo, r); err != nil {
if err := d.pushFileFromArtifact(a, imageRepo); err != nil {
return errors.Wrap(err, "while pushing file from artifact")
}
return nil
}
func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, r *LuetSystemRepository) error {
func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, b compiler.CompilerBackend) error {
// we generate a new archive containing the required compressed file.
// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
treeArchive, err := artifact.CreateArtifactForFile(a.Path)
@@ -159,7 +166,7 @@ func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArt
}
imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, a.GetFileName())
return d.pushFileFromArtifact(treeArchive, imageTree, r)
return d.pushFileFromArtifact(treeArchive, imageTree)
}
// Generate creates a Docker luet repository
@@ -216,7 +223,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
// we generate a new archive containing the required compressed file.
// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
if err := d.pushImageFromArtifact(a, r); err != nil {
if err := d.pushImageFromArtifact(a, d.b); err != nil {
return errors.Wrap(err, "error met while pushing runtime tree")
}
@@ -226,7 +233,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
}
// we generate a new archive containing the required compressed file.
// TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file
if err := d.pushImageFromArtifact(a, r); err != nil {
if err := d.pushImageFromArtifact(a, d.b); err != nil {
return errors.Wrap(err, "error met while pushing compiler tree")
}
@@ -242,7 +249,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
return errors.Wrap(err, "failed adding Metadata file to repository")
}
if err := d.pushImageFromArtifact(a, r); err != nil {
if err := d.pushImageFromArtifact(a, d.b); err != nil {
return errors.Wrap(err, "error met while pushing docker image from artifact")
}

17
pkg/package/package.go
View File

@@ -113,6 +113,10 @@ type Package interface {
GetBuildTimestamp() string
Clone() Package
GetMetadataFilePath() string
SetTreeDir(s string)
GetTreeDir() string
}
type Tree interface {
@@ -210,6 +214,11 @@ func (t *DefaultPackage) JSON() ([]byte, error) {
return buffer.Bytes(), err
}
// GetMetadataFilePath returns the canonical name of an artifact metadata file
func (d *DefaultPackage) GetMetadataFilePath() string {
return d.GetFingerPrint() + ".metadata.yaml"
}
// DefaultPackage represent a standard package definition
type DefaultPackage struct {
ID int `storm:"id,increment" json:"id"` // primary key with auto increment
@@ -235,6 +244,8 @@ type DefaultPackage struct {
BuildTimestamp string `json:"buildtimestamp,omitempty"`
Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
treeDir string
}
// State represent the package state
@@ -251,6 +262,12 @@ func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*D
}
}
func (p *DefaultPackage) SetTreeDir(s string) {
p.treeDir = s
}
func (p *DefaultPackage) GetTreeDir() string {
return p.treeDir
}
func (p *DefaultPackage) String() string {
b, err := p.JSON()
if err != nil {

10
pkg/tree/compiler_recipe.go
View File

@@ -98,6 +98,7 @@ func (r *CompilerRecipe) Load(path string) error {
}
// Path is set only internally when tree is loaded from disk
pack.SetPath(filepath.Dir(currentpath))
pack.SetTreeDir(path)
// Instead of rdeps, have a different tree for build deps.
compileDefPath := pack.Rel(CompilerDefinitionFile)
@@ -126,18 +127,25 @@ func (r *CompilerRecipe) Load(path string) error {
}
case CollectionFile:
dat, err := ioutil.ReadFile(currentpath)
if err != nil {
return errors.Wrap(err, "Error reading file "+currentpath)
}
packs, err := pkg.DefaultPackagesFromYaml(dat)
if err != nil {
return errors.Wrap(err, "Error reading yaml "+currentpath)
}
packsRaw, err := pkg.GetRawPackages(dat)
if err != nil {
return errors.Wrap(err, "Error reading raw packages from "+currentpath)
}
for _, pack := range packs {
pack.SetPath(filepath.Dir(currentpath))
pack.SetTreeDir(path)
// Instead of rdeps, have a different tree for build deps.
compileDefPath := pack.Rel(CompilerDefinitionFile)
@@ -170,9 +178,7 @@ func (r *CompilerRecipe) Load(path string) error {
return errors.Wrap(err, "Error creating package "+pack.GetName())
}
}
}
return nil
}