Update tests for including packages in tree from metadata

Also switches GenerateRepository to functional interface to allow
more parametrization

cmd/create-repo.go

@@ -69,7 +69,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename"))
 		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
 		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
-
+		viper.BindPFlag("from-metadata", cmd.Flags().Lookup("from-metadata"))
 		viper.BindPFlag("force-push", cmd.Flags().Lookup("force-push"))
 		viper.BindPFlag("push-images", cmd.Flags().Lookup("push-images"))
 
@@ -80,7 +80,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 
 		treePaths := viper.GetStringSlice("tree")
 		dst := viper.GetString("output")
-		packages := viper.GetString("packages")
+
 		name := viper.GetString("name")
 		descr := viper.GetString("descr")
 		urls := viper.GetStringSlice("urls")
@@ -101,6 +101,18 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		force := viper.GetBool("force-push")
 		imagePush := viper.GetBool("push-images")
 
+		opts := []installer.RepositoryOption{
+			installer.WithSource(viper.GetString("packages")),
+			installer.WithPushImages(imagePush),
+			installer.WithForce(force),
+			installer.FromRepository(fromRepo),
+			installer.WithConfig(LuetCfg),
+			installer.WithImagePrefix(dst),
+			installer.WithDatabase(pkg.NewInMemoryDatabase(false)),
+			installer.WithCompilerBackend(compilerBackend),
+			installer.FromMetadata(viper.GetBool("from-metadata")),
+		}
+
 		if source_repo != "" {
 			// Search for system repository
 			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
@@ -114,27 +126,28 @@ Create a repository from the metadata description defined in the luet.yaml confi
 				t = lrepo.Type
 			}
 
-			repo, err = installer.GenerateRepository(lrepo.Name,
+			opts = append(opts,
-				lrepo.Description, t,
+				installer.WithName(lrepo.Name),
-				lrepo.Urls,
+				installer.WithDescription(lrepo.Description),
-				lrepo.Priority,
+				installer.WithType(t),
-				packages,
+				installer.WithUrls(lrepo.Urls...),
-				treePaths,
+				installer.WithPriority(lrepo.Priority),
-				pkg.NewInMemoryDatabase(false),
+				installer.WithTree(treePaths...),
-				compilerBackend,
+			)
-				dst,
-				imagePush,
-				force,
-				fromRepo,
-				LuetCfg)
-			helpers.CheckErr(err)
 
 		} else {
-			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
+			opts = append(opts,
-				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force, fromRepo, LuetCfg)
+				installer.WithName(name),
-			helpers.CheckErr(err)
+				installer.WithDescription(descr),
+				installer.WithType(t),
+				installer.WithUrls(urls...),
+				installer.WithTree(treePaths...),
+			)
 		}
 
+		repo, err = installer.GenerateRepository(opts...)
+		helpers.CheckErr(err)
+
 		if treetype != "" {
 			treeFile.SetCompressionType(compression.Implementation(treetype))
 		}
@@ -177,6 +190,7 @@ func init() {
 
 	createrepoCmd.Flags().Bool("force-push", false, "Force overwrite of docker images if already present online")
 	createrepoCmd.Flags().Bool("push-images", false, "Enable/Disable docker image push for docker repositories")
+	createrepoCmd.Flags().Bool("from-metadata", false, "Consider metadata files from the packages folder while indexing the new tree")
 
 	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")

pkg/installer/installer_test.go

@@ -38,14 +38,15 @@ import (
 
 func stubRepo(tmpdir, tree string) (*LuetSystemRepository, error) {
 	return GenerateRepository(
-		"test",
+		WithName("test"),
-		"description",
+		WithDescription("description"),
-		"disk",
+		WithType("disk"),
-		[]string{tmpdir},
+		WithUrls(tmpdir),
-		1,
+		WithPriority(1),
-		tmpdir,
+		WithSource(tmpdir),
-		[]string{tree},
+		WithTree(tree),
-		pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
+		WithDatabase(pkg.NewInMemoryDatabase(false)),
+	)
 }
 
 var _ = Describe("Installer", func() {
@@ -337,13 +338,15 @@ urls:
 			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := GenerateRepository(
-				"test",
+				WithName("test"),
-				"description",
+				WithDescription("description"),
-				"disk",
+				WithType("disk"),
-				[]string{tmpdir}, 1,
+				WithUrls(tmpdir),
-				tmpdir,
+				WithPriority(1),
-				[]string{"../../tests/fixtures/buildable"},
+				WithSource(tmpdir),
-				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
+				WithTree("../../tests/fixtures/buildable"),
+				WithDatabase(pkg.NewInMemoryDatabase(false)),
+			)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
@@ -462,14 +465,15 @@ urls:
 			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
 
 			repo, err := GenerateRepository(
-				"test",
+				WithName("test"),
-				"description",
+				WithDescription("description"),
-				"disk",
+				WithType("disk"),
-				[]string{tmpdir},
+				WithUrls(tmpdir),
-				1,
+				WithPriority(1),
-				tmpdir,
+				WithSource(tmpdir),
-				[]string{"../../tests/fixtures/buildable"},
+				WithTree("../../tests/fixtures/buildable"),
-				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
+				WithDatabase(pkg.NewInMemoryDatabase(false)),
+			)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(fileHelper.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())

pkg/installer/repository.go

@@ -253,11 +253,9 @@ func (f *LuetRepositoryFile) GetChecksums() artifact.Checksums {
 // GenerateRepository generates a new repository from the given argument.
 // If the repository is of the docker type, it will also push the package images.
 // In case the repository is local, it will build the package Index
-func GenerateRepository(name, descr, t string, urls []string,
+func GenerateRepository(p ...RepositoryOption) (*LuetSystemRepository, error) {
-	priority int, src string, treesDir []string, db pkg.PackageDatabase,
+	c := RepositoryConfig{}
-	b compiler.CompilerBackend, imagePrefix string, pushImages, force, fromRepo bool, c *config.LuetConfig) (*LuetSystemRepository, error) {
+	c.Apply(p...)
-
-	// 1: First filter the runtime db to only the metadata we actually have
 
 	btr := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 	runtimeTree := pkg.NewInMemoryDatabase(false)
@@ -265,7 +263,7 @@ func GenerateRepository(name, descr, t string, urls []string,
 	tempTree := pkg.NewInMemoryDatabase(false)
 	temptr := tree.NewInstallerRecipe(tempTree)
 
-	for _, treeDir := range treesDir {
+	for _, treeDir := range c.Tree {
 		if err := temptr.Load(treeDir); err != nil {
 			return nil, err
 		}
@@ -280,8 +278,8 @@ func GenerateRepository(name, descr, t string, urls []string,
 	repodb := pkg.NewInMemoryDatabase(false)
 	generalRecipe := tree.NewCompilerRecipe(repodb)
 
-	if fromRepo {
+	if c.FromRepository {
-		if err := LoadBuildTree(generalRecipe, repodb, c); err != nil {
+		if err := LoadBuildTree(generalRecipe, repodb, c.config); err != nil {
 			Warning("errors while loading trees from repositories", err.Error())
 		}
 
@@ -293,7 +291,7 @@ func GenerateRepository(name, descr, t string, urls []string,
 
 	// Pick only atoms in db which have a real metadata for runtime db (tr)
 	for _, p := range tempTree.World() {
-		if _, err := os.Stat(filepath.Join(src, p.GetMetadataFilePath())); err == nil {
+		if _, err := os.Stat(filepath.Join(c.Src, p.GetMetadataFilePath())); err == nil {
 			runtimeTree.CreatePackage(p)
 		}
 	}
@@ -319,28 +317,30 @@ func GenerateRepository(name, descr, t string, urls []string,
 			return nil
 		}
 		if _, err := runtimeTree.FindPackage(art.CompileSpec.Package); err != nil && art.CompileSpec.Package.Name != "" {
-			Debug("Added", art.CompileSpec.Package.HumanReadableString(), "from metadata files")
+			Debug("Adding", art.CompileSpec.Package.HumanReadableString(), "from metadata file", currentpath)
 			runtimeTree.CreatePackage(art.CompileSpec.Package)
 		}
 
 		return nil
 	}
 
+	if c.FromMetadata {
 		// Best effort
-	filepath.Walk(src, ff)
+		filepath.Walk(c.Src, ff)
+	}
 
 	repo := &LuetSystemRepository{
-		LuetRepository:  config.NewLuetRepository(name, t, descr, urls, priority, true, false),
+		LuetRepository:  config.NewLuetRepository(c.Name, c.Type, c.Description, c.Urls, c.Priority, true, false),
 		Tree:            tree.NewInstallerRecipe(runtimeTree),
 		BuildTree:       btr,
 		RepositoryFiles: map[string]LuetRepositoryFile{},
-		PushImages:      pushImages,
+		PushImages:      c.PushImages,
-		ForcePush:       force,
+		ForcePush:       c.Force,
-		Backend:         b,
+		Backend:         c.CompilerBackend,
-		imagePrefix:     imagePrefix,
+		imagePrefix:     c.ImagePrefix,
 	}
 
-	if err := repo.initialize(src); err != nil {
+	if err := repo.initialize(c.Src); err != nil {
 		return nil, errors.Wrap(err, "while building repository artifact index")
 	}
 

pkg/installer/repository_docker.go

@@ -44,7 +44,6 @@ type dockerRepositoryGenerator struct {
 }
 
 func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
-
 	Info("Generating docker images for packages in", l.imagePrefix)
 	var art []*artifact.PackageArtifact
 	var ff = func(currentpath string, info os.FileInfo, err error) error {

pkg/installer/repository_options.go

@@ -0,0 +1,164 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	pkg "github.com/mudler/luet/pkg/package"
+)
+
+type RepositoryOption func(cfg *RepositoryConfig) error
+
+type RepositoryConfig struct {
+	Name, Description, Type string
+	Urls                    []string
+	Priority                int
+	Src                     string
+	Tree                    []string
+	DB                      pkg.PackageDatabase
+	CompilerBackend         compiler.CompilerBackend
+	ImagePrefix             string
+
+	config                                          *config.LuetConfig
+	PushImages, Force, FromRepository, FromMetadata bool
+}
+
+// Apply applies the given options to the config, returning the first error
+// encountered (if any).
+func (cfg *RepositoryConfig) Apply(opts ...RepositoryOption) error {
+	for _, opt := range opts {
+		if opt == nil {
+			continue
+		}
+		if err := opt(cfg); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func WithConfig(c *config.LuetConfig) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.config = c
+		return nil
+	}
+}
+
+func WithDatabase(b pkg.PackageDatabase) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.DB = b
+		return nil
+	}
+}
+
+func WithCompilerBackend(b compiler.CompilerBackend) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.CompilerBackend = b
+		return nil
+	}
+}
+
+func WithTree(s ...string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Tree = append(cfg.Tree, s...)
+		return nil
+	}
+}
+
+func WithUrls(s ...string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Urls = append(cfg.Urls, s...)
+		return nil
+	}
+}
+
+func WithSource(s string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Src = s
+		return nil
+	}
+}
+
+func WithName(s string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Name = s
+		return nil
+	}
+}
+
+func WithDescription(s string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Description = s
+		return nil
+	}
+}
+
+func WithType(s string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Type = s
+		return nil
+	}
+}
+
+func WithImagePrefix(s string) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.ImagePrefix = s
+		return nil
+	}
+}
+
+func WithPushImages(b bool) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.PushImages = b
+		return nil
+	}
+}
+
+func WithForce(b bool) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Force = b
+		return nil
+	}
+}
+
+// FromRepository when enabled
+// considers packages metadata
+// from remote repositories when building
+// the new repository index
+func FromRepository(b bool) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.FromRepository = b
+		return nil
+	}
+}
+
+// FromMetadata when enabled
+// considers packages metadata
+// when building repository indexes
+func FromMetadata(b bool) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.FromMetadata = b
+		return nil
+	}
+}
+
+func WithPriority(b int) func(cfg *RepositoryConfig) error {
+	return func(cfg *RepositoryConfig) error {
+		cfg.Priority = b
+		return nil
+	}
+}

pkg/installer/repository_test.go

@@ -41,14 +41,18 @@ import (
 
 func dockerStubRepo(tmpdir, tree, image string, push, force bool) (*LuetSystemRepository, error) {
 	return GenerateRepository(
-		"test",
+		WithName("test"),
-		"description",
+		WithDescription("description"),
-		"docker",
+		WithType("docker"),
-		[]string{image},
+		WithUrls(image),
-		1,
+		WithPriority(1),
-		tmpdir,
+		WithSource(tmpdir),
-		[]string{tree},
+		WithTree(tree),
-		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force, false, nil)
+		WithDatabase(pkg.NewInMemoryDatabase(false)),
+		WithCompilerBackend(backend.NewSimpleDockerBackend()),
+		WithImagePrefix(image),
+		WithPushImages(push),
+		WithForce(force))
 }
 
 var _ = Describe("Repository", func() {
@@ -210,6 +214,114 @@ urls:
 			_, err = repos.GetTree().GetDatabase().FindPackage(spec2.GetPackage())
 			Expect(err).To(HaveOccurred()) // should throw error
 		})
+
+		It("Generate repository metadata of files referenced in a tree and from packages", func() {
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			generalRecipe2 := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe2.Load("../../tests/fixtures/finalizers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase())
+			spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+			Expect(spec2.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
+
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+
+			artifact, err := compiler.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			artifact2, err := compiler2.Compile(false, spec2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(artifact2.Path)).To(BeTrue())
+			Expect(helpers.Untar(artifact2.Path, tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			content1, err := fileHelper.Read(spec.Rel("test5"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := fileHelper.Read(spec.Rel("test6"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("artifact5\n"))
+			Expect(content2).To(Equal("artifact6\n"))
+
+			// will contain both
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("alpine-seed-1.0.package.tar"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec2.Rel("alpine-seed-1.0.metadata.yaml"))).To(BeTrue())
+
+			repo, err := GenerateRepository(
+				WithName("test"),
+				WithDescription("description"),
+				WithType("disk"),
+				WithUrls(tmpdir),
+				WithPriority(1),
+				WithSource(tmpdir),
+				FromMetadata(true), // Enabling from metadata makes the package visible
+				WithTree("../../tests/fixtures/buildable"),
+				WithDatabase(pkg.NewInMemoryDatabase(false)),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(fileHelper.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+
+			// We check now that the artifact not referenced in the tree
+			// (spec2) is not indexed in the repository
+			repository, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+			repos, err := repository.Sync(true)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = repos.GetTree().GetDatabase().FindPackage(spec.GetPackage())
+			Expect(err).ToNot(HaveOccurred())
+			_, err = repos.GetTree().GetDatabase().FindPackage(spec2.GetPackage())
+			Expect(err).ToNot(HaveOccurred()) // should NOT throw error
+		})
 	})
 	Context("Matching packages", func() {
 		It("Matches packages in different repositories by priority", func() {