🆕 Tag 0.22.7

This fixes #284

cmd/build.go

@@ -131,16 +131,21 @@ Build packages specifying multiple definition trees:
 
 		generalRecipe := tree.NewCompilerRecipe(db)
 
-		if fromRepo {
-			if err := installer.LoadBuildTree(generalRecipe, db, util.DefaultContext); err != nil {
-				util.DefaultContext.Warning("errors while loading trees from repositories", err.Error())
-			}
-		}
-
 		for _, src := range treePaths {
 			util.DefaultContext.Info("Loading tree", src)
 			helpers.CheckErr(generalRecipe.Load(src))
 		}
+		templateFolders := []string{}
+
+		if fromRepo {
+			bt, err := installer.LoadBuildTree(generalRecipe, db, util.DefaultContext)
+			if err != nil {
+				util.DefaultContext.Warning("errors while loading trees from repositories", err.Error())
+			}
+			templateFolders = util.TemplateFolders(util.DefaultContext, bt, treePaths)
+		} else {
+			templateFolders = util.TemplateFolders(util.DefaultContext, installer.BuildTreeResult{}, treePaths)
+		}
 
 		util.DefaultContext.Info("Building in", dst)
 
@@ -161,7 +166,7 @@ Build packages specifying multiple definition trees:
 			options.WithPullRepositories(pullRepo),
 			options.WithPushRepository(imageRepository),
 			options.Rebuild(rebuild),
-			options.WithTemplateFolder(util.TemplateFolders(util.DefaultContext, fromRepo, treePaths)),
+			options.WithTemplateFolder(templateFolders),
 			options.WithSolverOptions(opts),
 			options.Wait(wait),
 			options.OnlyTarget(onlyTarget),

cmd/root.go

@@ -30,7 +30,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.22.5"
+	LuetCLIVersion = "0.22.7"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/tree/images.go

@@ -26,6 +26,7 @@ import (
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/mudler/luet/pkg/compiler/types/options"
+	"github.com/mudler/luet/pkg/installer"
 
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
@@ -84,7 +85,7 @@ func NewTreeImageCommand() *cobra.Command {
 				options.WithContext(util.DefaultContext),
 				options.WithPushRepository(imageRepository),
 				options.WithPullRepositories(pullRepo),
-				options.WithTemplateFolder(util.TemplateFolders(util.DefaultContext, false, treePath)),
+				options.WithTemplateFolder(util.TemplateFolders(util.DefaultContext, installer.BuildTreeResult{}, treePath)),
 				options.WithSolverOptions(opts),
 			)
 

cmd/util/cli.go

@@ -17,7 +17,6 @@ package util
 
 import (
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/marcsauter/single"
@@ -26,6 +25,7 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/mudler/luet/pkg/api/core/context"
+	"github.com/mudler/luet/pkg/api/core/template"
 	"github.com/mudler/luet/pkg/api/core/types"
 	"github.com/mudler/luet/pkg/installer"
 )
@@ -42,16 +42,15 @@ func ValuesFlags() []string {
 }
 
 // TemplateFolders returns the default folders which holds shared template between packages in a given tree path
-func TemplateFolders(ctx *context.Context, fromRepo bool, treePaths []string) []string {
+func TemplateFolders(ctx *context.Context, i installer.BuildTreeResult, treePaths []string) []string {
 	templateFolders := []string{}
 	for _, t := range treePaths {
-		templateFolders = append(templateFolders, filepath.Join(t, "templates"))
+		templateFolders = append(templateFolders, template.FindPossibleTemplatesDir(t)...)
 	}
-	if fromRepo {
+	for _, r := range i.TemplatesDir {
-		for _, s := range installer.SystemRepositories(ctx.GetConfig().SystemRepositories) {
+		templateFolders = append(templateFolders, r...)
-			templateFolders = append(templateFolders, filepath.Join(s.TreePath, "templates"))
-		}
 	}
+
 	return templateFolders
 }
 

pkg/api/core/template/directory.go

@@ -0,0 +1,39 @@
+// Copyright © 2022 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package template
+
+import (
+	"os"
+	"path/filepath"
+)
+
+const Directory = "templates"
+
+// FindPossibleTemplatesDir returns templates dir located at root
+func FindPossibleTemplatesDir(root string) (res []string) {
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if info.IsDir() && info.Name() == Directory {
+			res = append(res, currentpath)
+		}
+		return nil
+	}
+
+	filepath.Walk(root, ff)
+	return
+}

pkg/compiler/compiler.go

@@ -502,12 +502,14 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 		}
 	}
 
-	filelist, err := a.FileList()
+	if !p.Package.Hidden {
-	if err != nil {
+		filelist, err := a.FileList()
-		return a, errors.Wrapf(err, "Failed getting package list for '%s' '%s'", a.Path, a.CompileSpec.Package.HumanReadableString())
+		if err != nil {
+			return a, errors.Wrapf(err, "Failed getting package list for '%s' '%s'", a.Path, a.CompileSpec.Package.HumanReadableString())
+		}
+		a.Files = filelist
 	}
 
-	a.Files = filelist
 	a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
 
 	err = a.WriteYAML(p.GetOutputPath())

pkg/compiler/compiler_test.go

@@ -1064,5 +1064,41 @@ var _ = Describe("Compiler", func() {
 			files := art.Files
 			Expect(files).To(ContainElement("bin/busybox"))
 		})
+
+		It("is not generated after the compilation process and annotated in the metadata if a package is hidden", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/packagelayers_hidden")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.WithContext(context.NewContext()))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			compiler.Options.CompressionType = compression.GZip
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+			Expect(len(artifacts[0].Dependencies)).To(Equal(1))
+			Expect(artifacts[0].Files).ToNot(ContainElement("bin/busybox"))
+
+			Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
+
+			art, err := LoadArtifactFromYaml(spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			files := art.Files
+			Expect(files).ToNot(ContainElement("bin/busybox"))
+		})
 	})
 })

pkg/installer/repository.go

@@ -27,6 +27,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mudler/luet/pkg/api/core/template"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
@@ -114,9 +115,18 @@ func SystemRepositories(t types.LuetRepositories) Repositories {
 	return repos
 }
 
+type BuildTreeResult struct {
+	Repositories Repositories
+	TemplatesDir map[*LuetSystemRepository][]string
+}
+
 // LoadBuildTree loads to the tree the compilation specs from the system repositories
-func LoadBuildTree(t tree.Builder, db pkg.PackageDatabase, ctx types.Context) error {
+func LoadBuildTree(t tree.Builder, db pkg.PackageDatabase, ctx types.Context) (BuildTreeResult, error) {
 	var reserr error
+	res := &BuildTreeResult{
+		TemplatesDir: make(map[*LuetSystemRepository][]string),
+	}
+
 	repos := SystemRepositories(ctx.GetConfig().SystemRepositories)
 	for _, r := range repos {
 		repodir, err := ctx.TempDir(r.Name)
@@ -136,11 +146,14 @@ func LoadBuildTree(t tree.Builder, db pkg.PackageDatabase, ctx types.Context) er
 		}
 
 		r.SetTree(generalRecipe)
+		res.TemplatesDir[r] = template.FindPossibleTemplatesDir(repodir)
 	}
 
+	res.Repositories = repos
+
 	repos.SyncDatabase(db)
 
-	return reserr
+	return *res, reserr
 }
 
 func (m *LuetSystemRepositoryMetadata) WriteFile(path string) error {
@@ -277,14 +290,12 @@ func GenerateRepository(p ...RepositoryOption) (*LuetSystemRepository, error) {
 	generalRecipe := tree.NewCompilerRecipe(repodb)
 
 	if c.FromRepository {
-		if err := LoadBuildTree(generalRecipe, repodb, c.context); err != nil {
+		if _, err := LoadBuildTree(generalRecipe, repodb, c.context); err != nil {
 			c.context.Warning("errors while loading trees from repositories", err.Error())
 		}
-
 		if err := repodb.Clone(tempTree); err != nil {
 			c.context.Warning("errors while cloning trees from repositories", err.Error())
 		}
-
 	}
 
 	// Pick only atoms in db which have a real metadata for runtime db (tr)

pkg/tree/compiler_recipe.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mudler/luet/pkg/api/core/template"
 	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -72,18 +73,12 @@ func (r *CompilerRecipe) Save(path string) error {
 func (r *CompilerRecipe) Load(path string) error {
 
 	r.SourcePath = append(r.SourcePath, path)
-	//tmpfile, err := ioutil.TempFile("", "luet")
+
-	//if err != nil {
+	c, err := helpers.ChartFiles(template.FindPossibleTemplatesDir(path))
-	//	return err
-	//}
-	c, err := helpers.ChartFiles([]string{filepath.Join(path, "templates")})
 	if err != nil {
 		return err
 	}
 
-	//r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))
-	// TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean()
-	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
 		if err != nil {

tests/fixtures/packagelayers_hidden/build/build.yaml

@@ -0,0 +1,3 @@
+
+unpack: true
+image: "alpine"

tests/fixtures/packagelayers_hidden/build/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "build"
+version: "0.1"

tests/fixtures/packagelayers_hidden/runtime/build.yaml

@@ -0,0 +1,8 @@
+
+unpack: true
+requires:
+- category: "layer"
+  name: "build"
+  version: ">=0.1"
+steps: 
+- rm -rfv /var

tests/fixtures/packagelayers_hidden/runtime/definition.yaml

@@ -0,0 +1,4 @@
+category: "layer"
+name: "runtime"
+version: "0.1"
+hidden: true