Tag 0.17.5

This changeset allows to have shared templates in a static folder
"templates" present in each luet tree. If the directory is present, it
gets scanned and templated accordingly on top of each package. This
allows to use such folder to store custom blocks to share between
packages.

This is still experimental and subject to change, this is just a first
pass version to provide the feature. It needs to be refined still as it
would be more elegant to use the helm engine properly and map our
structure to the engine instead of adapting it roughly.

Fixes #224

cmd/build.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
@@ -83,10 +84,8 @@ Build packages specifying multiple definition trees:
 		viper.BindPFlag("wait", cmd.Flags().Lookup("wait"))
 		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
 
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSolverFlags(cmd)
+
 		LuetCfg.Viper.BindPFlag("general.show_build_output", cmd.Flags().Lookup("live-output"))
 		LuetCfg.Viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
 
@@ -148,21 +147,11 @@ Build packages specifying multiple definition trees:
 
 		Info("Building in", dst)
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		opts := util.SetSolverConfig()
 		pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
 
 		LuetCfg.GetGeneral().ShowBuildOutput = LuetCfg.Viper.GetBool("general.show_build_output")
 
-		opts := &LuetSolverOptions{
-			Type:        stype,
-			LearnRate:   float32(rate),
-			Discount:    float32(discount),
-			MaxAttempts: attempts,
-		}
-
 		Debug("Solver", opts.CompactString())
 
 		if concurrent {
@@ -171,6 +160,17 @@ Build packages specifying multiple definition trees:
 			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
 		}
 
+		templateFolders := []string{}
+		if !fromRepo {
+			for _, t := range treePaths {
+				templateFolders = append(templateFolders, filepath.Join(t, "templates"))
+			}
+		} else {
+			for _, s := range installer.SystemRepositories(LuetCfg) {
+				templateFolders = append(templateFolders, filepath.Join(s.TreePath, "templates"))
+			}
+		}
+
 		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
 			options.NoDeps(nodeps),
 			options.WithBackendType(backendType),
@@ -179,6 +179,7 @@ Build packages specifying multiple definition trees:
 			options.WithPullRepositories(pullRepo),
 			options.WithPushRepository(imageRepository),
 			options.Rebuild(rebuild),
+			options.WithTemplateFolder(templateFolders),
 			options.WithSolverOptions(*opts),
 			options.Wait(wait),
 			options.OnlyTarget(onlyTarget),

cmd/cleanup.go

@@ -20,8 +20,8 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
-	config "github.com/mudler/luet/pkg/config"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	. "github.com/mudler/luet/pkg/logger"
 
@@ -33,19 +33,11 @@ var cleanupCmd = &cobra.Command{
 	Short: "Clean packages cache.",
 	Long:  `remove downloaded packages tarballs and clean cache directory`,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+		util.BindSystemFlags(cmd)
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var cleaned int = 0
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := config.LuetCfg.Viper.GetString("system.rootfs")
-		engine := config.LuetCfg.Viper.GetString("system.database_engine")
-
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig()
 		// Check if cache dir exists
 		if fileHelper.Exists(LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
 

cmd/database/create.go

@@ -18,6 +18,7 @@ package cmd_database
 import (
 	"io/ioutil"
 
+	"github.com/mudler/luet/cmd/util"
 	artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
 
 	. "github.com/mudler/luet/pkg/logger"
@@ -45,21 +46,11 @@ The yaml must contain the package definition, and the file list at least.
 For reference, inspect a "metadata.yaml" file generated while running "luet build"`,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 
-			dbpath := LuetCfg.Viper.GetString("system.database_path")
-			rootfs := LuetCfg.Viper.GetString("system.rootfs")
-			engine := LuetCfg.Viper.GetString("system.database_engine")
-
-			LuetCfg.System.DatabaseEngine = engine
-			LuetCfg.System.DatabasePath = dbpath
-			LuetCfg.System.Rootfs = rootfs
-
+			util.SetSystemConfig()
 			systemDB := LuetCfg.GetSystemDB()
 
 			for _, a := range args {

cmd/database/get.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	"gopkg.in/yaml.v2"
 
 	. "github.com/mudler/luet/pkg/config"
@@ -38,20 +39,11 @@ To return also files:
 		$ luet database get --files system/foo`,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			showFiles, _ := cmd.Flags().GetBool("files")
-			dbpath := LuetCfg.Viper.GetString("system.database_path")
-			rootfs := LuetCfg.Viper.GetString("system.rootfs")
-			engine := LuetCfg.Viper.GetString("system.database_engine")
-
-			LuetCfg.System.DatabaseEngine = engine
-			LuetCfg.System.DatabasePath = dbpath
-			LuetCfg.System.Rootfs = rootfs
+			util.SetSystemConfig()
 
 			systemDB := LuetCfg.GetSystemDB()
 

cmd/database/remove.go

@@ -19,6 +19,7 @@ import (
 	. "github.com/mudler/luet/pkg/logger"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 
 	"github.com/spf13/cobra"
@@ -36,19 +37,10 @@ This commands takes multiple packages as arguments and prunes their entries from
 `,
 		Args: cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-			LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-
+			util.BindSystemFlags(cmd)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			dbpath := LuetCfg.Viper.GetString("system.database_path")
-			rootfs := LuetCfg.Viper.GetString("system.rootfs")
-			engine := LuetCfg.Viper.GetString("system.database_engine")
-
-			LuetCfg.System.DatabaseEngine = engine
-			LuetCfg.System.DatabasePath = dbpath
-			LuetCfg.System.Rootfs = rootfs
+			util.SetSystemConfig()
 
 			systemDB := LuetCfg.GetSystemDB()
 

cmd/install.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -47,13 +48,8 @@ To force install a package:
 `,
 	Aliases: []string{"i"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
@@ -70,10 +66,6 @@ To force install a package:
 			toInstall = append(toInstall, pack)
 		}
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps := LuetCfg.Viper.GetBool("nodeps")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
@@ -81,18 +73,8 @@ To force install a package:
 		yes := LuetCfg.Viper.GetBool("yes")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
-
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
-
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		util.SetSystemConfig()
+		util.SetSolverConfig()
 
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple

cmd/reclaim.go

@@ -15,6 +15,7 @@
 package cmd
 
 import (
+	"github.com/mudler/luet/cmd/util"
 	installer "github.com/mudler/luet/pkg/installer"
 
 	. "github.com/mudler/luet/pkg/config"
@@ -27,9 +28,7 @@ var reclaimCmd = &cobra.Command{
 	Use:   "reclaim",
 	Short: "Reclaim packages to Luet database from available repositories",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+		util.BindSystemFlags(cmd)
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Long: `Reclaim tries to find association between packages in the online repositories and the system one.
@@ -39,13 +38,7 @@ var reclaimCmd = &cobra.Command{
 It scans the target file system, and if finds a match with a package available in the repositories, it marks as installed in the system database.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
-
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig()
 
 		// This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type
 		repos := installer.Repositories{}

cmd/reinstall.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -34,13 +35,8 @@ var reinstallCmd = &cobra.Command{
 	$ luet reinstall -y system/busybox shells/bash system/coreutils ...
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		LuetCfg.Viper.BindPFlag("for", cmd.Flags().Lookup("for"))
@@ -51,22 +47,14 @@ var reinstallCmd = &cobra.Command{
 		var toUninstall pkg.Packages
 		var toAdd pkg.Packages
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
+
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig()
 
 		for _, a := range args {
 			pack, err := helpers.ParsePackageStr(a)
@@ -87,10 +75,7 @@ var reinstallCmd = &cobra.Command{
 			repos = append(repos, r)
 		}
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		util.SetSolverConfig()
 
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple

cmd/replace.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mudler/luet/pkg/solver"
 
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -35,13 +36,8 @@ var replaceCmd = &cobra.Command{
 	$ luet replace -y system/busybox ... --for shells/bash --for system/coreutils ...
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
@@ -54,24 +50,15 @@ var replaceCmd = &cobra.Command{
 		var toAdd pkg.Packages
 
 		f := LuetCfg.Viper.GetStringSlice("for")
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps := LuetCfg.Viper.GetBool("nodeps")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
-
+		util.SetSystemConfig()
+		util.SetSolverConfig()
 		for _, a := range args {
 			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
@@ -98,11 +85,6 @@ var replaceCmd = &cobra.Command{
 			repos = append(repos, r)
 		}
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
 		} else {

cmd/root.go

@@ -41,7 +41,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.17.1"
+	LuetCLIVersion = "0.17.5"
 	LuetEnvPrefix  = "LUET"
 	license        = `
 	Luet Copyright (C) 2019-2021 Ettore Di Giacinto

cmd/search.go

@@ -21,6 +21,7 @@ import (
 	"github.com/ghodss/yaml"
 	"github.com/jedib0t/go-pretty/table"
 	"github.com/jedib0t/go-pretty/v6/list"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -306,14 +307,9 @@ Search can also return results in the terminal in different ways: as terminal ou
 `,
 	Aliases: []string{"s"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var results Results
@@ -325,32 +321,20 @@ Search can also return results in the terminal in different ways: as terminal ou
 		hidden, _ := cmd.Flags().GetBool("hidden")
 
 		installed := LuetCfg.Viper.GetBool("installed")
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		searchWithLabel, _ := cmd.Flags().GetBool("by-label")
 		searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex")
 		revdeps, _ := cmd.Flags().GetBool("revdeps")
 		tableMode, _ := cmd.Flags().GetBool("table")
 		files, _ := cmd.Flags().GetBool("files")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
+		util.SetSystemConfig()
+		util.SetSolverConfig()
+
 		out, _ := cmd.Flags().GetString("output")
 		if out != "terminal" {
 			LuetCfg.GetLogging().SetLogLevel("error")
 		}
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
-
 		l := list.NewWriter()
 		t := table.NewWriter()
 		t.AppendHeader(rows)

cmd/uninstall.go

@@ -16,6 +16,7 @@ package cmd
 
 import (
 	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -31,16 +32,11 @@ var uninstallCmd = &cobra.Command{
 	Long:    `Uninstall packages`,
 	Aliases: []string{"rm", "un"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		toRemove := []pkg.Package{}
@@ -53,10 +49,6 @@ var uninstallCmd = &cobra.Command{
 			toRemove = append(toRemove, pack)
 		}
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps, _ := cmd.Flags().GetBool("nodeps")
 		full, _ := cmd.Flags().GetBool("full")
@@ -64,20 +56,12 @@ var uninstallCmd = &cobra.Command{
 		fullClean, _ := cmd.Flags().GetBool("full-clean")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
 		keepProtected, _ := cmd.Flags().GetBool("keep-protected-files")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
-		LuetCfg.ConfigProtectSkip = !keepProtected
+		util.SetSystemConfig()
+		util.SetSolverConfig()
 
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		LuetCfg.ConfigProtectSkip = !keepProtected
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
 		} else {

cmd/upgrade.go

@@ -15,6 +15,7 @@
 package cmd
 
 import (
+	"github.com/mudler/luet/cmd/util"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -28,15 +29,10 @@ var upgradeCmd = &cobra.Command{
 	Short:   "Upgrades the system",
 	Aliases: []string{"u"},
 	PreRun: func(cmd *cobra.Command, args []string) {
-		LuetCfg.Viper.BindPFlag("system.database_path", installCmd.Flags().Lookup("system-dbpath"))
-		LuetCfg.Viper.BindPFlag("system.rootfs", installCmd.Flags().Lookup("system-target"))
-		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
-		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
-		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
-		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		LuetCfg.Viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
-		LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
 	},
 	Long: `Upgrades packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
@@ -51,10 +47,6 @@ var upgradeCmd = &cobra.Command{
 			repos = append(repos, r)
 		}
 
-		stype := LuetCfg.Viper.GetString("solver.type")
-		discount := LuetCfg.Viper.GetFloat64("solver.discount")
-		rate := LuetCfg.Viper.GetFloat64("solver.rate")
-		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps, _ := cmd.Flags().GetBool("nodeps")
 		full, _ := cmd.Flags().GetBool("full")
@@ -63,18 +55,11 @@ var upgradeCmd = &cobra.Command{
 		sync, _ := cmd.Flags().GetBool("sync")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
-		dbpath := LuetCfg.Viper.GetString("system.database_path")
-		rootfs := LuetCfg.Viper.GetString("system.rootfs")
-		engine := LuetCfg.Viper.GetString("system.database_engine")
 		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
-		LuetCfg.System.DatabaseEngine = engine
-		LuetCfg.System.DatabasePath = dbpath
-		LuetCfg.System.Rootfs = rootfs
-		LuetCfg.GetSolverOptions().Type = stype
-		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
-		LuetCfg.GetSolverOptions().Discount = float32(discount)
-		LuetCfg.GetSolverOptions().MaxAttempts = attempts
+		util.SetSystemConfig()
+		util.SetSolverConfig()
+
 		if concurrent {
 			LuetCfg.GetSolverOptions().Implementation = solver.ParallelSimple
 		} else {

cmd/util/cli.go

@@ -0,0 +1,65 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/config"
+)
+
+func BindSystemFlags(cmd *cobra.Command) {
+	LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+	LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+	LuetCfg.Viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+}
+
+func BindSolverFlags(cmd *cobra.Command) {
+	LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+	LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+	LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+	LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+}
+
+func SetSystemConfig() {
+	dbpath := LuetCfg.Viper.GetString("system.database_path")
+	rootfs := LuetCfg.Viper.GetString("system.rootfs")
+	engine := LuetCfg.Viper.GetString("system.database_engine")
+
+	LuetCfg.System.DatabaseEngine = engine
+	LuetCfg.System.DatabasePath = dbpath
+	LuetCfg.System.SetRootFS(rootfs)
+}
+
+func SetSolverConfig() (c *config.LuetSolverOptions) {
+	stype := LuetCfg.Viper.GetString("solver.type")
+	discount := LuetCfg.Viper.GetFloat64("solver.discount")
+	rate := LuetCfg.Viper.GetFloat64("solver.rate")
+	attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+
+	LuetCfg.GetSolverOptions().Type = stype
+	LuetCfg.GetSolverOptions().LearnRate = float32(rate)
+	LuetCfg.GetSolverOptions().Discount = float32(discount)
+	LuetCfg.GetSolverOptions().MaxAttempts = attempts
+
+	return &config.LuetSolverOptions{
+		Type:        stype,
+		LearnRate:   float32(rate),
+		Discount:    float32(discount),
+		MaxAttempts: attempts,
+	}
+}

pkg/compiler/compiler.go

@@ -42,6 +42,7 @@ import (
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
+	"helm.sh/helm/v3/pkg/chart"
 )
 
 const BuildFile = "build.yaml"
@@ -750,6 +751,7 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS
 		p.BuildOptions.PullImageRepository = append(p.BuildOptions.PullImageRepository, cs.Options.PullImageRepository...)
 		Debug("Inheriting pull repository from PullImageRepository buildoptions", p.BuildOptions.PullImageRepository)
 	}
+
 	Debug(p.GetPackage().HumanReadableString(), "Build options after inherit", p.BuildOptions)
 }
 
@@ -1128,6 +1130,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 type templatedata map[string]interface{}
 
 func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.Package, dst templatedata) ([]byte, error) {
+	// Grab shared templates first
+	var chartFiles []*chart.File
+	if len(cs.Options.TemplatesFolder) != 0 {
+		c, err := helpers.ChartFiles(cs.Options.TemplatesFolder)
+		if err == nil {
+			chartFiles = c
+		}
+	}
 
 	var dataresult []byte
 	val := pack.Rel(DefinitionFile)
@@ -1165,7 +1175,7 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
 			return nil, errors.Wrap(err, "merging values maps")
 		}
 
-		dat, err := helpers.RenderHelm(string(dataBuild), td, dst)
+		dat, err := helpers.RenderHelm(append(chartFiles, helpers.ChartFileB(dataBuild)...), td, dst)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}
@@ -1190,7 +1200,13 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
 				bv = append([]string{f}, bv...)
 			}
 		}
-		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, bv...)
+
+		raw, err := ioutil.ReadFile(pack.Rel(BuildFile))
+		if err != nil {
+			return nil, err
+		}
+
+		out, err := helpers.RenderFiles(append(chartFiles, helpers.ChartFileB(raw)...), val, bv...)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}
@@ -1256,6 +1272,11 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio
 
 	cs.inheritSpecBuildOptions(newSpec)
 
+	// Update the package in the compiler database to catch updates from NewLuetCompilationSpec
+	if err := cs.Database.UpdatePackage(newSpec.Package); err != nil {
+		return nil, errors.Wrap(err, "failed updating new package entry in compiler database")
+	}
+
 	return newSpec, err
 }
 

pkg/compiler/types/artifact/artifact.go

@@ -453,6 +453,9 @@ func (a *PackageArtifact) GetProtectFiles() []string {
 
 // Unpack Untar and decompress (TODO) to the given path
 func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+	if !strings.HasPrefix(dst, "/") {
+		return errors.New("destination must be an absolute path")
+	}
 
 	// Create
 	protectedFiles := a.GetProtectFiles()

pkg/compiler/types/options/compiler_options.go

@@ -43,6 +43,9 @@ type Compiler struct {
 	BackendArgs []string
 
 	BackendType string
+
+	// TemplatesFolder. should default to tree/templates
+	TemplatesFolder []string
 }
 
 func NewDefaultCompiler() *Compiler {
@@ -87,6 +90,13 @@ func WithBackendType(r string) func(cfg *Compiler) error {
 	}
 }
 
+func WithTemplateFolder(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.TemplatesFolder = r
+		return nil
+	}
+}
+
 func WithBuildValues(r []string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		cfg.BuildValuesFile = r

pkg/compiler/types/spec/spec.go

@@ -23,11 +23,11 @@ import (
 	"github.com/mitchellh/hashstructure/v2"
 	options "github.com/mudler/luet/pkg/compiler/types/options"
 
+	"github.com/ghodss/yaml"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/otiai10/copy"
 	dirhash "golang.org/x/mod/sumdb/dirhash"
-	yaml "gopkg.in/yaml.v2"
 )
 
 type LuetCompilationspecs []LuetCompilationSpec
@@ -157,11 +157,31 @@ func (cs *LuetCompilationSpec) signature() Signature {
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
 	var spec LuetCompilationSpec
+	var packageDefinition pkg.DefaultPackage
 	err := yaml.Unmarshal(b, &spec)
 	if err != nil {
 		return &spec, err
 	}
-	spec.Package = p.(*pkg.DefaultPackage)
+	err = yaml.Unmarshal(b, &packageDefinition)
+	if err != nil {
+		return &spec, err
+	}
+
+	// Update requires/conflict/provides
+	// When we have been passed a bytes slice, parse it as a package
+	// and updates requires/conflicts/provides.
+	// This is required in order to allow manipulation of such fields with templating
+	copy := *p.(*pkg.DefaultPackage)
+	spec.Package = &copy
+	if len(packageDefinition.GetRequires()) != 0 {
+		spec.Package.Requires(packageDefinition.GetRequires())
+	}
+	if len(packageDefinition.GetConflicts()) != 0 {
+		spec.Package.Conflicts(packageDefinition.GetConflicts())
+	}
+	if len(packageDefinition.GetProvides()) != 0 {
+		spec.Package.SetProvides(packageDefinition.GetProvides())
+	}
 	return &spec, nil
 }
 func (cs *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
@@ -270,10 +290,13 @@ func (cs *LuetCompilationSpec) Hash() (string, error) {
 	// build a signature, we want to be part of the hash only the fields that are relevant for build purposes
 	signature := cs.signature()
 	h, err := hashstructure.Hash(signature, hashstructure.FormatV2, nil)
-	sum, err := dirhash.HashDir(cs.Package.Path, "", dirhash.DefaultHash)
 	if err != nil {
 		return "", err
 	}
+	sum, err := dirhash.HashDir(cs.Package.Path, "", dirhash.DefaultHash)
+	if err != nil {
+		return fmt.Sprint(h), err
+	}
 	return fmt.Sprint(h, sum), err
 }
 

pkg/config/config.go

@@ -27,6 +27,8 @@ import (
 	"strings"
 	"time"
 
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	pkg "github.com/mudler/luet/pkg/package"
 	solver "github.com/mudler/luet/pkg/solver"
 
@@ -107,6 +109,16 @@ type LuetSystemConfig struct {
 	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
 }
 
+func (s *LuetSystemConfig) SetRootFS(path string) error {
+	p, err := fileHelper.Rel2Abs(path)
+	if err != nil {
+		return err
+	}
+
+	s.Rootfs = p
+	return nil
+}
+
 func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
 	dbpath = filepath.Join(dbpath, "repos/"+name)

pkg/helpers/file/file.go

@@ -287,3 +287,15 @@ func CopyDir(src string, dst string) (err error) {
 		Sync:      true,
 		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }
+
+func Rel2Abs(s string) (string, error) {
+	pathToSet := s
+	if !filepath.IsAbs(s) {
+		abs, err := filepath.Abs(s)
+		if err != nil {
+			return "", err
+		}
+		pathToSet = abs
+	}
+	return pathToSet, nil
+}

pkg/helpers/helm.go

@@ -2,6 +2,8 @@ package helpers
 
 import (
 	"io/ioutil"
+	"path/filepath"
+	"strings"
 
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
@@ -13,17 +15,86 @@ import (
 	"helm.sh/helm/v3/pkg/engine"
 )
 
+// ChartFileB is an helper that takes a slice of bytes and construct a chart.File slice from it
+func ChartFileB(s []byte) []*chart.File {
+	return []*chart.File{
+		{Name: "templates", Data: s},
+	}
+}
+
+// ChartFileS is an helper that takes a string and construct a chart.File slice from it
+func ChartFileS(s string) []*chart.File {
+	return []*chart.File{
+		{Name: "templates", Data: []byte(s)},
+	}
+}
+
+// ChartFile reads all the given files and returns a slice of []*chart.File
+// containing the raw content and the file name for each file
+func ChartFile(s ...string) []*chart.File {
+	files := []*chart.File{}
+	for _, c := range s {
+		raw, err := ioutil.ReadFile(c)
+		if err != nil {
+			return files
+		}
+		files = append(files, &chart.File{Name: c, Data: raw})
+	}
+
+	return files
+}
+
+// ChartFiles reads a list of paths and reads all yaml file inside. It returns a
+// slice of pointers of chart.File(s) with the raw content of the yaml
+func ChartFiles(path []string) ([]*chart.File, error) {
+	var chartFiles []*chart.File
+	for _, t := range path {
+		rel, err := fileHelper.Rel2Abs(t)
+		if err != nil {
+			return nil, err
+		}
+
+		if !fileHelper.Exists(rel) {
+			continue
+		}
+		files, err := fileHelper.ListDir(rel)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, f := range files {
+			if strings.ToLower(filepath.Ext(f)) == ".yaml" {
+				raw, err := ioutil.ReadFile(f)
+				if err != nil {
+					return nil, err
+				}
+				chartFiles = append(chartFiles, &chart.File{Name: f, Data: raw})
+			}
+		}
+	}
+	return chartFiles, nil
+}
+
 // RenderHelm renders the template string with helm
-func RenderHelm(template string, values, d map[string]interface{}) (string, error) {
+func RenderHelm(files []*chart.File, values, d map[string]interface{}) (string, error) {
+
+	// We slurp all the files into one here. This is not elegant, but still works.
+	// As a reminder, the files passed here have on the head the templates in the 'templates/' folder
+	// of each luet tree, and it have at the bottom the package buildpsec to be templated.
+	// TODO: Replace by correctly populating the files so that the helm render engine templates it
+	// correctly
+	toTemplate := ""
+	for _, f := range files {
+		toTemplate += string(f.Data)
+	}
+
 	c := &chart.Chart{
 		Metadata: &chart.Metadata{
 			Name:    "",
 			Version: "",
 		},
-		Templates: []*chart.File{
-			{Name: "templates", Data: []byte(template)},
-		},
-		Values: map[string]interface{}{"Values": values},
+		Templates: ChartFileS(toTemplate),
+		Values:    map[string]interface{}{"Values": values},
 	}
 
 	v, err := chartutil.CoalesceValues(c, map[string]interface{}{"Values": d})
@@ -71,23 +142,18 @@ func reverse(s []string) []string {
 	return s
 }
 
-func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string, error) {
-	raw, err := ioutil.ReadFile(toTemplate)
-	if err != nil {
-		return "", errors.Wrap(err, "reading file "+toTemplate)
-	}
-
+func RenderFiles(files []*chart.File, valuesFile string, defaultFile ...string) (string, error) {
 	if !fileHelper.Exists(valuesFile) {
-		return "", errors.Wrap(err, "file not existing "+valuesFile)
+		return "", errors.New("file does not exist: " + valuesFile)
 	}
 	val, err := ioutil.ReadFile(valuesFile)
 	if err != nil {
-		return "", errors.Wrap(err, "reading file "+valuesFile)
+		return "", errors.Wrap(err, "reading file: "+valuesFile)
 	}
 
 	var values templatedata
 	if err = yaml.Unmarshal(val, &values); err != nil {
-		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
+		return "", errors.Wrap(err, "unmarshalling values")
 	}
 
 	dst, err := UnMarshalValues(defaultFile)
@@ -95,5 +161,5 @@ func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string,
 		return "", errors.Wrap(err, "unmarshalling values")
 	}
 
-	return RenderHelm(string(raw), values, dst)
+	return RenderHelm(files, values, dst)
 }

pkg/helpers/helm_test.go

@@ -30,21 +30,21 @@ func writeFile(path string, content string) {
 	Expect(err).ToNot(HaveOccurred())
 }
 
-var _ = Describe("Helpers", func() {
+var _ = Describe("Helm", func() {
 	Context("RenderHelm", func() {
 		It("Renders templates", func() {
-			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
+			out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).To(Equal("foobar"))
 		})
 		It("Renders templates with overrides", func() {
-			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
+			out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).To(Equal("foobar"))
 		})
 
 		It("Renders templates", func() {
-			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
+			out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).To(Equal("foobar"))
 		})
@@ -68,7 +68,7 @@ foo: "baz"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("baz"))
 
@@ -93,7 +93,7 @@ faa: "baz"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bar"))
 
@@ -114,7 +114,7 @@ foo: "bar"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values)
+			res, err := RenderFiles(ChartFile(toTemplate), values)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bar"))
 		})
@@ -145,11 +145,11 @@ bar: "nei"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d2, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d2, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("bazneif"))
 
-			res, err = RenderFiles(toTemplate, values, d, d2)
+			res, err = RenderFiles(ChartFile(toTemplate), values, d, d2)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal("doneif"))
 		})
@@ -173,7 +173,7 @@ faa: "baz"
 
 			Expect(err).ToNot(HaveOccurred())
 
-			res, err := RenderFiles(toTemplate, values, d)
+			res, err := RenderFiles(ChartFile(toTemplate), values, d)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res).To(Equal(""))
 

pkg/installer/client/http.go

@@ -18,10 +18,12 @@ package client
 import (
 	"fmt"
 	"math"
+	"net/http"
 	"net/url"
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"time"
 
 	"github.com/mudler/luet/pkg/compiler/types/artifact"
@@ -42,6 +44,27 @@ func NewHttpClient(r RepoData) *HttpClient {
 	return &HttpClient{RepoData: r}
 }
 
+func NewGrabClient() *grab.Client {
+	httpTimeout := 30
+	timeout := os.Getenv("HTTP_TIMEOUT")
+	if timeout != "" {
+		timeoutI, err := strconv.Atoi(timeout)
+		if err == nil {
+			httpTimeout = timeoutI
+		}
+	}
+
+	return &grab.Client{
+		UserAgent: "grab",
+		HTTPClient: &http.Client{
+			Timeout: time.Duration(httpTimeout) * time.Second,
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+			},
+		},
+	}
+}
+
 func (c *HttpClient) PrepareReq(dst, url string) (*grab.Request, error) {
 
 	req, err := grab.NewRequest(dst, url)
@@ -86,7 +109,7 @@ func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.Pa
 		}
 		defer os.RemoveAll(temp)
 
-		client := grab.NewClient()
+		client := NewGrabClient()
 
 		for _, uri := range c.RepoData.Urls {
 			Debug("Downloading artifact", artifactName, "from", uri)
@@ -186,7 +209,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 		return "", err
 	}
 
-	client := grab.NewClient()
+	client := NewGrabClient()
 
 	for _, uri := range c.RepoData.Urls {
 

pkg/installer/installer.go

@@ -272,6 +272,15 @@ func (l *LuetInstaller) swap(o Option, syncedRepos Repositories, toRemove pkg.Pa
 	if err := l.download(syncedRepos, match); err != nil {
 		return errors.Wrap(err, "Pre-downloading packages")
 	}
+
+	if err := l.checkFileconflicts(match, false, s); err != nil {
+		if !l.Options.Force {
+			return errors.Wrap(err, "file conflict found")
+		} else {
+			Warning("file conflict found", err.Error())
+		}
+	}
+
 	if l.Options.DownloadOnly {
 		return nil
 	}
@@ -756,8 +765,10 @@ func (l *LuetInstaller) getFinalizers(allRepos pkg.PackageDatabase, solution sol
 	return toFinalize, nil
 }
 
-func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, s *System) error {
+func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, checkSystem bool, s *System) error {
 	Info("Checking for file conflicts..")
+	defer s.Clean() // Release memory
+
 	filesToInstall := []string{}
 	for _, m := range toInstall {
 		a, err := l.downloadPackage(m)
@@ -775,18 +786,19 @@ func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, s
 					"file conflict between packages to be installed",
 				)
 			}
-
-			exists, p, err := s.ExistsPackageFile(f)
-			if err != nil {
-				return errors.Wrap(err, "failed checking into system db")
-			}
-			if exists {
-				return fmt.Errorf(
-					"file conflict between '%s' and '%s' ( file: %s )",
-					p.HumanReadableString(),
-					m.Package.HumanReadableString(),
-					f,
-				)
+			if checkSystem {
+				exists, p, err := s.ExistsPackageFile(f)
+				if err != nil {
+					return errors.Wrap(err, "failed checking into system db")
+				}
+				if exists {
+					return fmt.Errorf(
+						"file conflict between '%s' and '%s' ( file: %s )",
+						p.HumanReadableString(),
+						m.Package.HumanReadableString(),
+						f,
+					)
+				}
 			}
 		}
 		filesToInstall = append(filesToInstall, files...)
@@ -803,7 +815,7 @@ func (l *LuetInstaller) install(o Option, syncedRepos Repositories, toInstall ma
 	}
 
 	// Check file conflicts
-	if err := l.checkFileconflicts(toInstall, s); err != nil {
+	if err := l.checkFileconflicts(toInstall, true, s); err != nil {
 		if !l.Options.Force {
 			return errors.Wrap(err, "file conflict found")
 		} else {

pkg/installer/system.go

@@ -1,6 +1,8 @@
 package installer
 
 import (
+	"sync"
+
 	"github.com/hashicorp/go-multierror"
 	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
@@ -10,8 +12,10 @@ import (
 )
 
 type System struct {
-	Database pkg.PackageDatabase
-	Target   string
+	Database  pkg.PackageDatabase
+	Target    string
+	fileIndex map[string]pkg.Package
+	sync.Mutex
 }
 
 func (s *System) World() (pkg.Packages, error) {
@@ -25,7 +29,7 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
 	executedFinalizer := map[string]bool{}
 	for _, p := range packs {
 		if fileHelper.Exists(p.Rel(tree.FinalizerFile)) {
-			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile))
+			out, err := helpers.RenderFiles(helpers.ChartFile(p.Rel(tree.FinalizerFile)), p.Rel(tree.DefinitionFile))
 			if err != nil {
 				Warning("Failed rendering finalizer for ", p.HumanReadableString(), err.Error())
 				errs = multierror.Append(errs, err)
@@ -53,17 +57,37 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error {
 	return errs
 }
 
-func (s *System) ExistsPackageFile(file string) (bool, pkg.Package, error) {
-	for _, p := range s.Database.World() {
-		files, err := s.Database.GetPackageFiles(p)
-		if err != nil {
-			return false, nil, err
-		}
-		for _, f := range files {
-			if f == file {
-				return true, p, nil
+func (s *System) buildFileIndex() {
+	s.Lock()
+	defer s.Unlock()
+	// Check if cache is empty or if it got modified
+	if s.fileIndex == nil { //|| len(s.Database.GetPackages()) != len(s.fileIndex) {
+		s.fileIndex = make(map[string]pkg.Package)
+		for _, p := range s.Database.World() {
+			files, _ := s.Database.GetPackageFiles(p)
+			for _, f := range files {
+				s.fileIndex[f] = p
 			}
 		}
 	}
+}
+
+func (s *System) Clean() {
+	s.Lock()
+	defer s.Unlock()
+	s.fileIndex = nil
+}
+
+func (s *System) ExistsPackageFile(file string) (bool, pkg.Package, error) {
+	Debug("Checking if file ", file, "belongs to any package")
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	if p, exists := s.fileIndex[file]; exists {
+		Debug(file, "belongs already to", p.HumanReadableString())
+
+		return exists, p, nil
+	}
+	Debug(file, "doesn't belong to any package")
 	return false, nil, nil
 }

pkg/installer/system_test.go

@@ -60,7 +60,7 @@ var _ = Describe("System", func() {
 			r, p, err = s.ExistsPackageFile("f")
 			Expect(r).To(BeTrue())
 			Expect(err).ToNot(HaveOccurred())
-			Expect(p).To(Equal(a))
+			Expect(p).To(Equal(b))
 			r, p, err = s.ExistsPackageFile("barz")
 			Expect(r).To(BeTrue())
 			Expect(err).ToNot(HaveOccurred())

pkg/tree/compiler_recipe.go

@@ -76,6 +76,10 @@ func (r *CompilerRecipe) Load(path string) error {
 	//if err != nil {
 	//	return err
 	//}
+	c, err := helpers.ChartFiles([]string{filepath.Join(path, "templates")})
+	if err != nil {
+		return err
+	}
 
 	//r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))
 	// TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean()
@@ -104,8 +108,7 @@ func (r *CompilerRecipe) Load(path string) error {
 			// Instead of rdeps, have a different tree for build deps.
 			compileDefPath := pack.Rel(CompilerDefinitionFile)
 			if fileHelper.Exists(compileDefPath) {
-
-				dat, err := helpers.RenderFiles(compileDefPath, currentpath)
+				dat, err := helpers.RenderFiles(append(c, helpers.ChartFile(compileDefPath)...), currentpath)
 				if err != nil {
 					return errors.Wrap(err,
 						"Error templating file "+CompilerDefinitionFile+" from "+
@@ -157,7 +160,7 @@ func (r *CompilerRecipe) Load(path string) error {
 					if err != nil {
 						return errors.Wrap(err, "Error reading file "+currentpath)
 					}
-					dat, err := helpers.RenderHelm(string(buildyaml), raw, map[string]interface{}{})
+					dat, err := helpers.RenderHelm(append(c, helpers.ChartFileB(buildyaml)...), raw, map[string]interface{}{})
 					if err != nil {
 						return errors.Wrap(err,
 							"Error templating file "+CompilerDefinitionFile+" from "+
@@ -183,7 +186,7 @@ func (r *CompilerRecipe) Load(path string) error {
 		return nil
 	}
 
-	err := filepath.Walk(path, ff)
+	err = filepath.Walk(path, ff)
 	if err != nil {
 		return err
 	}

tests/fixtures/fileconflicts_upgrade/conflict1/build.yaml

@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+  - mkdir /foo
+steps:
+  - echo conflict > /foo/test1
+package_dir: /foo

tests/fixtures/fileconflicts_upgrade/conflict1/collection.yaml

@@ -0,0 +1,13 @@
+packages:
+- category: "test1"
+  name: "conflict"
+  version: "1.0"
+- category: "test2"
+  name: "conflict"
+  version: "1.0"
+- category: "test1"
+  name: "conflict"
+  version: "1.1"
+- category: "test2"
+  name: "conflict"
+  version: "1.1"

tests/fixtures/shared_templates/templates/writefile.yaml

@@ -0,0 +1,3 @@
+{{ define "writefile" }}
+- echo conflict > /foo/{{.}}
+{{end}}

tests/fixtures/shared_templates/test/build.yaml

@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+  - mkdir /foo
+steps:
+{{ template "writefile" .Values.name }}
+package_dir: /foo

tests/fixtures/shared_templates/test/collection.yaml

@@ -0,0 +1,7 @@
+packages:
+- category: "test"
+  name: "foo"
+  version: "1.0"
+- category: "test"
+  name: "bar"
+  version: "1.0"

tests/integration/32_fileconflicts_upgrade.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/fileconflicts_upgrade" --destination $tmpdir/testbuild --compression gzip --all
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test1-1.0.package.tar.gz' ]"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/conflict-test2-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/fileconflicts_upgrade" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install -y --force --config $tmpdir/luet.yaml test1/conflict@1.0 test2/conflict@1.0
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test succeded' "$installst" "0"
+    #assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testUpgrade() {
+    out=$(luet upgrade -y --config $tmpdir/luet.yaml)
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "1"
+    assertContains 'does find conflicts' "$out" "Error: file conflict found: file conflict between packages to be installed"
+
+    luet upgrade -y --config $tmpdir/luet.yaml --force
+    #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test succeeded' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/34_templates.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/shared_templates" --destination $tmpdir/testbuild --compression gzip --all
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/foo-test-1.0.package.tar.gz' ]"
+    assertTrue 'create packages' "[ -e '$tmpdir/testbuild/bar-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/shared_templates" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install -y --config $tmpdir/luet.yaml test/foo test/bar
+    installst=$?
+    assertEquals 'install test failed' "$installst" "0"
+    assertTrue 'package bar installed' "[ -e '$tmpdir/testrootfs/bar' ]"
+    assertTrue 'package foo installed' "[ -e '$tmpdir/testrootfs/foo' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+