Tag bugfix release 0.17.11

Also switches GenerateRepository to functional interface to allow
more parametrization

.github/workflows/push.yml

@@ -1,7 +1,7 @@
 on: push
 concurrency:
-  group: ci-${{ github.head_ref || github.ref }}-${{ github.repository }}
-  cancel-in-progress: true
+  group: registries-tests
+
 name: Build on push
 jobs:
   tests-integration:

.github/workflows/release.yml

@@ -2,6 +2,9 @@ on:
   push:
     tags:
       - '*'  # only test and release when a tag is pushed
+concurrency:
+  group: registries-tests
+
 name: Test and Release on tag
 jobs:
   tests-integration:

cmd/create-repo.go

@@ -69,7 +69,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename"))
 		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
 		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
-
+		viper.BindPFlag("from-metadata", cmd.Flags().Lookup("from-metadata"))
 		viper.BindPFlag("force-push", cmd.Flags().Lookup("force-push"))
 		viper.BindPFlag("push-images", cmd.Flags().Lookup("push-images"))
 
@@ -80,7 +80,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 
 		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("output")
-		packages := viper.GetString("packages")
+
 		name := viper.GetString("name")
 		descr := viper.GetString("descr")
 		urls := viper.GetStringSlice("urls")
@@ -101,6 +101,18 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		force := viper.GetBool("force-push")
 		imagePush := viper.GetBool("push-images")
 
+		opts := []installer.RepositoryOption{
+			installer.WithSource(viper.GetString("packages")),
+			installer.WithPushImages(imagePush),
+			installer.WithForce(force),
+			installer.FromRepository(fromRepo),
+			installer.WithConfig(LuetCfg),
+			installer.WithImagePrefix(dst),
+			installer.WithDatabase(pkg.NewInMemoryDatabase(false)),
+			installer.WithCompilerBackend(compilerBackend),
+			installer.FromMetadata(viper.GetBool("from-metadata")),
+		}
+
 		if source_repo != "" {
 			// Search for system repository
 			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
@@ -114,27 +126,28 @@ Create a repository from the metadata description defined in the luet.yaml confi
 				t = lrepo.Type
 			}
 
-			repo, err = installer.GenerateRepository(lrepo.Name,
-				lrepo.Description, t,
-				lrepo.Urls,
-				lrepo.Priority,
-				packages,
-				treePaths,
-				pkg.NewInMemoryDatabase(false),
-				compilerBackend,
-				dst,
-				imagePush,
-				force,
-				fromRepo,
-				LuetCfg)
-			helpers.CheckErr(err)
+			opts = append(opts,
+				installer.WithName(lrepo.Name),
+				installer.WithDescription(lrepo.Description),
+				installer.WithType(t),
+				installer.WithUrls(lrepo.Urls...),
+				installer.WithPriority(lrepo.Priority),
+				installer.WithTree(treePaths...),
+			)
 
 		} else {
-			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
-				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force, fromRepo, LuetCfg)
-			helpers.CheckErr(err)
+			opts = append(opts,
+				installer.WithName(name),
+				installer.WithDescription(descr),
+				installer.WithType(t),
+				installer.WithUrls(urls...),
+				installer.WithTree(treePaths...),
+			)
 		}
 
+		repo, err = installer.GenerateRepository(opts...)
+		helpers.CheckErr(err)
+
 		if treetype != "" {
 			treeFile.SetCompressionType(compression.Implementation(treetype))
 		}
@@ -177,6 +190,7 @@ func init() {
 
 	createrepoCmd.Flags().Bool("force-push", false, "Force overwrite of docker images if already present online")
 	createrepoCmd.Flags().Bool("push-images", false, "Enable/Disable docker image push for docker repositories")
+	createrepoCmd.Flags().Bool("from-metadata", false, "Consider metadata files from the packages folder while indexing the new tree")
 
 	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")

cmd/pack.go

@@ -80,7 +80,7 @@ Afterwards, you can use the content generated and associate it with a tree and a
 		}
 		a.Files = filelist
 		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
-		err = a.WriteYaml(dst)
+		err = a.WriteYAML(dst)
 		if err != nil {
 			Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
 		}

cmd/root.go

@@ -41,7 +41,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.17.9"
+	LuetCLIVersion = "0.17.11"
 	LuetEnvPrefix  = "LUET"
 	license        = `
 	Luet Copyright (C) 2019-2021 Ettore Di Giacinto

pkg/compiler/compiler.go

@@ -442,7 +442,7 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 		a.CompileSpec = p
 		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
 
-		err = a.WriteYaml(p.GetOutputPath())
+		err = a.WriteYAML(p.GetOutputPath())
 		if err != nil {
 			return a, errors.Wrap(err, "Failed while writing metadata file")
 		}
@@ -472,7 +472,7 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	a.Files = filelist
 	a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
 
-	err = a.WriteYaml(p.GetOutputPath())
+	err = a.WriteYAML(p.GetOutputPath())
 	if err != nil {
 		return a, errors.Wrap(err, "Failed while writing metadata file")
 	}

pkg/compiler/types/artifact/artifact.go

@@ -63,6 +63,7 @@ type PackageArtifact struct {
 	CompressionType   compression.Implementation        `json:"compressiontype"`
 	Files             []string                          `json:"files"`
 	PackageCacheImage string                            `json:"package_cacheimage"`
+	Runtime           *pkg.DefaultPackage               `json:"runtime,omitempty"`
 }
 
 func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
@@ -101,19 +102,23 @@ func (a *PackageArtifact) Verify() error {
 	return nil
 }
 
-func (a *PackageArtifact) WriteYaml(dst string) error {
+func (a *PackageArtifact) WriteYAML(dst string) error {
 	// First compute checksum of artifact. When we write the yaml we want to write up-to-date informations.
 	err := a.Hash()
 	if err != nil {
 		return errors.Wrap(err, "Failed generating checksums for artifact")
 	}
 
-	//p := a.CompileSpec.GetPackage().GetPath()
+	// Update runtime package information
+	if a.CompileSpec != nil && a.CompileSpec.Package != nil {
+		runtime, err := a.CompileSpec.Package.GetRuntimePackage()
+		if err != nil {
+			return errors.Wrapf(err, "getting runtime package for '%s'", a.CompileSpec.Package.HumanReadableString())
+		}
+		Debug(fmt.Sprintf("embedding runtime package (%s) definition to artifact metadata", a.CompileSpec.Package.HumanReadableString()))
+		a.Runtime = runtime
+	}
 
-	//a.CompileSpec.GetPackage().SetPath("")
-	//	for _, ass := range a.CompileSpec.GetSourceAssertion() {
-	//		ass.Package.SetPath("")
-	//	}
 	data, err := yaml.Marshal(a)
 	if err != nil {
 		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")

pkg/installer/installer_test.go

@@ -38,14 +38,15 @@ import (
 
 func stubRepo(tmpdir, tree string) (*LuetSystemRepository, error) {
 	return GenerateRepository(
-		"test",
-		"description",
-		"disk",
-		[]string{tmpdir},
-		1,
-		tmpdir,
-		[]string{tree},
-		pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
+		WithName("test"),
+		WithDescription("description"),
+		WithType("disk"),
+		WithUrls(tmpdir),
+		WithPriority(1),
+		WithSource(tmpdir),
+		WithTree(tree),
+		WithDatabase(pkg.NewInMemoryDatabase(false)),
+	)
 }
 
 var _ = Describe("Installer", func() {
@@ -337,13 +338,15 @@ urls:
 			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := GenerateRepository(
-				"test",
-				"description",
-				"disk",
-				[]string{tmpdir}, 1,
-				tmpdir,
-				[]string{"../../tests/fixtures/buildable"},
-				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
+				WithName("test"),
+				WithDescription("description"),
+				WithType("disk"),
+				WithUrls(tmpdir),
+				WithPriority(1),
+				WithSource(tmpdir),
+				WithTree("../../tests/fixtures/buildable"),
+				WithDatabase(pkg.NewInMemoryDatabase(false)),
+			)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
@@ -462,14 +465,15 @@ urls:
 			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := GenerateRepository(
-				"test",
-				"description",
-				"disk",
-				[]string{tmpdir},
-				1,
-				tmpdir,
-				[]string{"../../tests/fixtures/buildable"},
-				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
+				WithName("test"),
+				WithDescription("description"),
+				WithType("disk"),
+				WithUrls(tmpdir),
+				WithPriority(1),
+				WithSource(tmpdir),
+				WithTree("../../tests/fixtures/buildable"),
+				WithDatabase(pkg.NewInMemoryDatabase(false)),
+			)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())

pkg/installer/repository.go

@@ -24,6 +24,7 @@ import (
 	"regexp"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
@@ -252,11 +253,9 @@ func (f *LuetRepositoryFile) GetChecksums() artifact.Checksums {
 // GenerateRepository generates a new repository from the given argument.
 // If the repository is of the docker type, it will also push the package images.
 // In case the repository is local, it will build the package Index
-func GenerateRepository(name, descr, t string, urls []string,
-	priority int, src string, treesDir []string, db pkg.PackageDatabase,
-	b compiler.CompilerBackend, imagePrefix string, pushImages, force, fromRepo bool, c *config.LuetConfig) (*LuetSystemRepository, error) {
-
-	// 1: First filter the runtime db to only the metadata we actually have
+func GenerateRepository(p ...RepositoryOption) (*LuetSystemRepository, error) {
+	c := RepositoryConfig{}
+	c.Apply(p...)
 
 	btr := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 	runtimeTree := pkg.NewInMemoryDatabase(false)
@@ -264,7 +263,7 @@ func GenerateRepository(name, descr, t string, urls []string,
 	tempTree := pkg.NewInMemoryDatabase(false)
 	temptr := tree.NewInstallerRecipe(tempTree)
 
-	for _, treeDir := range treesDir {
+	for _, treeDir := range c.Tree {
 		if err := temptr.Load(treeDir); err != nil {
 			return nil, err
 		}
@@ -279,8 +278,8 @@ func GenerateRepository(name, descr, t string, urls []string,
 	repodb := pkg.NewInMemoryDatabase(false)
 	generalRecipe := tree.NewCompilerRecipe(repodb)
 
-	if fromRepo {
-		if err := LoadBuildTree(generalRecipe, repodb, c); err != nil {
+	if c.FromRepository {
+		if err := LoadBuildTree(generalRecipe, repodb, c.config); err != nil {
 			Warning("errors while loading trees from repositories", err.Error())
 		}
 
@@ -292,23 +291,66 @@ func GenerateRepository(name, descr, t string, urls []string,
 
 	// Pick only atoms in db which have a real metadata for runtime db (tr)
 	for _, p := range tempTree.World() {
-		if _, err := os.Stat(filepath.Join(src, p.GetMetadataFilePath())); err == nil {
+		if _, err := os.Stat(filepath.Join(c.Src, p.GetMetadataFilePath())); err == nil {
 			runtimeTree.CreatePackage(p)
 		}
 	}
 
+	// Load packages from metadata files if not present already.
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+
+		// Only those which are metadata
+		if !strings.HasSuffix(info.Name(), pkg.PackageMetaSuffix) {
+			return nil
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return nil
+		}
+
+		art, err := artifact.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return nil
+		}
+		if _, err := runtimeTree.FindPackage(art.CompileSpec.Package); err != nil && art.CompileSpec.Package.Name != "" {
+			Debug("Adding", art.CompileSpec.Package.HumanReadableString(), "from metadata file", currentpath)
+			if art.Runtime != nil && art.Runtime.Name != "" {
+				runtimeTree.CreatePackage(art.Runtime)
+			} else {
+				// We don't have runtime at this point. So we import the package as is
+				r := []*pkg.DefaultPackage{}
+				p := art.CompileSpec.Package.Clone()
+				p.Requires(r)
+				p.SetProvides(r)
+				p.Conflicts(r)
+				runtimeTree.CreatePackage(p)
+			}
+		}
+
+		return nil
+	}
+
+	if c.FromMetadata {
+		// Best effort
+		filepath.Walk(c.Src, ff)
+	}
+
 	repo := &LuetSystemRepository{
-		LuetRepository:  config.NewLuetRepository(name, t, descr, urls, priority, true, false),
+		LuetRepository:  config.NewLuetRepository(c.Name, c.Type, c.Description, c.Urls, c.Priority, true, false),
 		Tree:            tree.NewInstallerRecipe(runtimeTree),
 		BuildTree:       btr,
 		RepositoryFiles: map[string]LuetRepositoryFile{},
-		PushImages:      pushImages,
-		ForcePush:       force,
-		Backend:         b,
-		imagePrefix:     imagePrefix,
+		PushImages:      c.PushImages,
+		ForcePush:       c.Force,
+		Backend:         c.CompilerBackend,
+		imagePrefix:     c.ImagePrefix,
 	}
 
-	if err := repo.initialize(src); err != nil {
+	if err := repo.initialize(c.Src); err != nil {
 		return nil, errors.Wrap(err, "while building repository artifact index")
 	}
 

pkg/installer/repository_docker.go

@@ -44,7 +44,6 @@ type dockerRepositoryGenerator struct {
 }
 
 func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
-
 	Info("Generating docker images for packages in", l.imagePrefix)
 	var art []*artifact.PackageArtifact
 	var ff = func(currentpath string, info os.FileInfo, err error) error {

pkg/installer/repository_options.go

@@ -0,0 +1,164 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	pkg "github.com/mudler/luet/pkg/package"
+)
+
+type RepositoryOption func(cfg *RepositoryConfig) error
+
+type RepositoryConfig struct {
+	Name, Description, Type string
+	Urls                    []string
+	Priority                int
+	Src                     string
+	Tree                    []string
+	DB                      pkg.PackageDatabase
+	CompilerBackend         compiler.CompilerBackend
+	ImagePrefix             string
+
+	config                                          *config.LuetConfig
+	PushImages, Force, FromRepository, FromMetadata bool
+}
+
+// Apply applies the given options to the config, returning the first error
+// encountered (if any).
+func (cfg *RepositoryConfig) Apply(opts ...RepositoryOption) error {
+	for _, opt := range opts {
+		if opt == nil {
+			continue
+		}
+		if err := opt(cfg); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func WithConfig(c *config.LuetConfig) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.config = c
+		return nil
+	}
+}
+
+func WithDatabase(b pkg.PackageDatabase) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.DB = b
+		return nil
+	}
+}
+
+func WithCompilerBackend(b compiler.CompilerBackend) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.CompilerBackend = b
+		return nil
+	}
+}
+
+func WithTree(s ...string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Tree = append(cfg.Tree, s...)
+		return nil
+	}
+}
+
+func WithUrls(s ...string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Urls = append(cfg.Urls, s...)
+		return nil
+	}
+}
+
+func WithSource(s string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Src = s
+		return nil
+	}
+}
+
+func WithName(s string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Name = s
+		return nil
+	}
+}
+
+func WithDescription(s string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Description = s
+		return nil
+	}
+}
+
+func WithType(s string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Type = s
+		return nil
+	}
+}
+
+func WithImagePrefix(s string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.ImagePrefix = s
+		return nil
+	}
+}
+
+func WithPushImages(b bool) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.PushImages = b
+		return nil
+	}
+}
+
+func WithForce(b bool) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Force = b
+		return nil
+	}
+}
+
+// FromRepository when enabled
+// considers packages metadata
+// from remote repositories when building
+// the new repository index
+func FromRepository(b bool) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.FromRepository = b
+		return nil
+	}
+}
+
+// FromMetadata when enabled
+// considers packages metadata
+// when building repository indexes
+func FromMetadata(b bool) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.FromMetadata = b
+		return nil
+	}
+}
+
+func WithPriority(b int) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Priority = b
+		return nil
+	}
+}

pkg/installer/repository_test.go

@@ -41,14 +41,18 @@ import (
 
 func dockerStubRepo(tmpdir, tree, image string, push, force bool) (*LuetSystemRepository, error) {
 	return GenerateRepository(
-		"test",
-		"description",
-		"docker",
-		[]string{image},
-		1,
-		tmpdir,
-		[]string{tree},
-		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force, false, nil)
+		WithName("test"),
+		WithDescription("description"),
+		WithType("docker"),
+		WithUrls(image),
+		WithPriority(1),
+		WithSource(tmpdir),
+		WithTree(tree),
+		WithDatabase(pkg.NewInMemoryDatabase(false)),
+		WithCompilerBackend(backend.NewSimpleDockerBackend()),
+		WithImagePrefix(image),
+		WithPushImages(push),
+		WithForce(force))
 }
 
 var _ = Describe("Repository", func() {
@@ -210,6 +214,114 @@ urls:
 			_, err = repos.GetTree().GetDatabase().FindPackage(spec2.GetPackage())
 			Expect(err).To(HaveOccurred()) // should throw error
 		})
+
+		It("Generate repository metadata of files referenced in a tree and from packages", func() {
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			generalRecipe2 := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe2.Load("../../tests/fixtures/finalizers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase())
+			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+			Expect(spec2.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			artifact2, err := compiler2.Compile(false, spec2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact2.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact2.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			content1, err := fileHelper.Read(spec.Rel("test5"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := fileHelper.Read(spec.Rel("test6"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("artifact5\n"))
+			Expect(content2).To(Equal("artifact6\n"))
+
+			// will contain both
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
+
+			repo, err := GenerateRepository(
+				WithName("test"),
+				WithDescription("description"),
+				WithType("disk"),
+				WithUrls(tmpdir),
+				WithPriority(1),
+				WithSource(tmpdir),
+				FromMetadata(true), // Enabling from metadata makes the package visible
+				WithTree("../../tests/fixtures/buildable"),
+				WithDatabase(pkg.NewInMemoryDatabase(false)),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+
+			// We check now that the artifact not referenced in the tree
+			// (spec2) is not indexed in the repository
+			repository, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			repos, err := repository.Sync(true)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = repos.GetTree().GetDatabase().FindPackage(spec.GetPackage())
+			Expect(err).ToNot(HaveOccurred())
+			_, err = repos.GetTree().GetDatabase().FindPackage(spec2.GetPackage())
+			Expect(err).ToNot(HaveOccurred()) // should NOT throw error
+		})
 	})
 	Context("Matching packages", func() {
 		It("Matches packages in different repositories by priority", func() {

pkg/installer/system.go

@@ -29,7 +29,7 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
 	executedFinalizer := map[string]bool{}
 	for _, p := range packs {
 		if fileHelper.Exists(p.Rel(tree.FinalizerFile)) {
-			out, err := helpers.RenderFiles(helpers.ChartFile(p.Rel(tree.FinalizerFile)), p.Rel(tree.DefinitionFile))
+			out, err := helpers.RenderFiles(helpers.ChartFile(p.Rel(tree.FinalizerFile)), p.Rel(pkg.PackageDefinitionFile))
 			if err != nil {
 				Warning("Failed rendering finalizer for ", p.HumanReadableString(), err.Error())
 				errs = multierror.Append(errs, err)

pkg/package/package.go

@@ -21,11 +21,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
 
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	"github.com/mudler/luet/pkg/helpers/docker"
 	"github.com/mudler/luet/pkg/helpers/match"
 	version "github.com/mudler/luet/pkg/versioner"
@@ -124,6 +127,12 @@ type Package interface {
 	JSON() ([]byte, error)
 }
 
+const (
+	PackageMetaSuffix     = "metadata.yaml"
+	PackageCollectionFile = "collection.yaml"
+	PackageDefinitionFile = "definition.yaml"
+)
+
 type Tree interface {
 	GetPackageSet() PackageDatabase
 	Prelude() string // A tree might have a prelude to be able to consume a tree
@@ -213,11 +222,14 @@ func GetRawPackages(yml []byte) (rawPackages, error) {
 	return rawPackages.Packages, nil
 
 }
-func DefaultPackagesFromYaml(yml []byte) ([]DefaultPackage, error) {
 
-	var unescaped struct {
-		Packages []DefaultPackage `json:"packages"`
-	}
+type Collection struct {
+	Packages []DefaultPackage `json:"packages"`
+}
+
+func DefaultPackagesFromYAML(yml []byte) ([]DefaultPackage, error) {
+
+	var unescaped Collection
 	source, err := yaml.YAMLToJSON(yml)
 	if err != nil {
 		return []DefaultPackage{}, err
@@ -246,7 +258,7 @@ func (t *DefaultPackage) JSON() ([]byte, error) {
 
 // GetMetadataFilePath returns the canonical name of an artifact metadata file
 func (d *DefaultPackage) GetMetadataFilePath() string {
-	return d.GetFingerPrint() + ".metadata.yaml"
+	return fmt.Sprintf("%s.%s", d.GetFingerPrint(), PackageMetaSuffix)
 }
 
 // DefaultPackage represent a standard package definition
@@ -379,6 +391,10 @@ func (p *DefaultPackage) MatchLabel(r *regexp.Regexp) bool {
 	return match.MapMatchRegex(&p.Labels, r)
 }
 
+func (p DefaultPackage) IsCollection() bool {
+	return fileHelper.Exists(filepath.Join(p.Path, PackageCollectionFile))
+}
+
 func (p *DefaultPackage) HasAnnotation(label string) bool {
 	return match.MapHasKey(&p.Annotations, label)
 }
@@ -701,6 +717,39 @@ func (set Packages) Unique() Packages {
 	return result
 }
 
+func (p *DefaultPackage) GetRuntimePackage() (*DefaultPackage, error) {
+	var r *DefaultPackage
+	if p.IsCollection() {
+		collectionFile := filepath.Join(p.Path, PackageCollectionFile)
+		dat, err := ioutil.ReadFile(collectionFile)
+		if err != nil {
+			return r, errors.Wrapf(err, "failed while reading '%s'", collectionFile)
+		}
+		coll, err := DefaultPackagesFromYAML(dat)
+		if err != nil {
+			return r, errors.Wrapf(err, "failed while parsing YAML '%s'", collectionFile)
+		}
+		for _, c := range coll {
+			if c.Matches(p) {
+				r = &c
+				break
+			}
+		}
+	} else {
+		definitionFile := filepath.Join(p.Path, PackageDefinitionFile)
+		dat, err := ioutil.ReadFile(definitionFile)
+		if err != nil {
+			return r, errors.Wrapf(err, "failed while reading '%s'", definitionFile)
+		}
+		d, err := DefaultPackageFromYaml(dat)
+		if err != nil {
+			return r, errors.Wrapf(err, "failed while parsing YAML '%s'", definitionFile)
+		}
+		r = &d
+	}
+	return r, nil
+}
+
 func (pack *DefaultPackage) buildFormula(definitiondb PackageDatabase, db PackageDatabase, visited map[string]interface{}) ([]bf.Formula, error) {
 	if _, ok := visited[pack.HumanReadableString()]; ok {
 		return nil, nil

pkg/tree/compiler_recipe.go

@@ -90,12 +90,12 @@ func (r *CompilerRecipe) Load(path string) error {
 			return errors.Wrap(err, "Error on walk path "+currentpath)
 		}
 
-		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
+		if info.Name() != pkg.PackageDefinitionFile && info.Name() != pkg.PackageCollectionFile {
 			return nil // Skip with no errors
 		}
 
 		switch info.Name() {
-		case DefinitionFile:
+		case pkg.PackageDefinitionFile:
 
 			pack, err := ReadDefinitionFile(currentpath)
 			if err != nil {
@@ -130,14 +130,14 @@ func (r *CompilerRecipe) Load(path string) error {
 				return errors.Wrap(err, "Error creating package "+pack.GetName())
 			}
 
-		case CollectionFile:
+		case pkg.PackageCollectionFile:
 
 			dat, err := ioutil.ReadFile(currentpath)
 			if err != nil {
 				return errors.Wrap(err, "Error reading file "+currentpath)
 			}
 
-			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			packs, err := pkg.DefaultPackagesFromYAML(dat)
 			if err != nil {
 				return errors.Wrap(err, "Error reading yaml "+currentpath)
 			}

pkg/tree/installer_recipe.go

@@ -56,7 +56,7 @@ func (r *InstallerRecipe) Save(path string) error {
 		if err != nil {
 			return err
 		}
-		err = ioutil.WriteFile(filepath.Join(dir, DefinitionFile), data, 0644)
+		err = ioutil.WriteFile(filepath.Join(dir, pkg.PackageDefinitionFile), data, 0644)
 		if err != nil {
 			return err
 		}
@@ -86,7 +86,7 @@ func (r *InstallerRecipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
-		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
+		if info.Name() != pkg.PackageDefinitionFile && info.Name() != pkg.PackageCollectionFile {
 			return nil // Skip with no errors
 		}
 
@@ -96,7 +96,7 @@ func (r *InstallerRecipe) Load(path string) error {
 		}
 
 		switch info.Name() {
-		case DefinitionFile:
+		case pkg.PackageDefinitionFile:
 			pack, err := pkg.DefaultPackageFromYaml(dat)
 			if err != nil {
 				return errors.Wrap(err, "Error reading yaml "+currentpath)
@@ -109,8 +109,8 @@ func (r *InstallerRecipe) Load(path string) error {
 				return errors.Wrap(err, "Error creating package "+pack.GetName())
 			}
 
-		case CollectionFile:
-			packs, err := pkg.DefaultPackagesFromYaml(dat)
+		case pkg.PackageCollectionFile:
+			packs, err := pkg.DefaultPackagesFromYAML(dat)
 			if err != nil {
 				return errors.Wrap(err, "Error reading yaml "+currentpath)
 			}

pkg/tree/recipes.go

@@ -33,11 +33,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-const (
-	DefinitionFile = "definition.yaml"
-	CollectionFile = "collection.yaml"
-)
-
 func NewGeneralRecipe(db pkg.PackageDatabase) Builder { return &Recipe{Database: db} }
 
 // Recipe is the "general" reciper for Trees
@@ -64,7 +59,7 @@ func (r *Recipe) Save(path string) error {
 		dir := filepath.Join(path, p.GetCategory(), p.GetName(), p.GetVersion())
 		os.MkdirAll(dir, os.ModePerm)
 
-		err := WriteDefinitionFile(p, filepath.Join(dir, DefinitionFile))
+		err := WriteDefinitionFile(p, filepath.Join(dir, pkg.PackageDefinitionFile))
 		if err != nil {
 			return err
 		}
@@ -96,7 +91,7 @@ func (r *Recipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
-		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
+		if info.Name() != pkg.PackageDefinitionFile && info.Name() != pkg.PackageCollectionFile {
 			return nil // Skip with no errors
 		}
 
@@ -106,7 +101,7 @@ func (r *Recipe) Load(path string) error {
 		}
 
 		switch info.Name() {
-		case DefinitionFile:
+		case pkg.PackageDefinitionFile:
 			pack, err := pkg.DefaultPackageFromYaml(dat)
 			if err != nil {
 				return errors.Wrap(err, "Error reading yaml "+currentpath)
@@ -118,8 +113,8 @@ func (r *Recipe) Load(path string) error {
 			if err != nil {
 				return errors.Wrap(err, "Error creating package "+pack.GetName())
 			}
-		case CollectionFile:
-			packs, err := pkg.DefaultPackagesFromYaml(dat)
+		case pkg.PackageCollectionFile:
+			packs, err := pkg.DefaultPackagesFromYAML(dat)
 			if err != nil {
 				return errors.Wrap(err, "Error reading yaml "+currentpath)
 			}