Allow to define multiple templated packages with collections

Collections, similarly to packages, have a `build.yaml` and
a `finalize.yaml` that are templated for each package.
They have a `collection.yaml` containing a list of
packages that are part of the tree.

pkg/compiler/compiler.go

@@ -38,6 +38,7 @@ import (
 
 const BuildFile = "build.yaml"
 const DefinitionFile = "definition.yaml"
+const CollectionFile = "collection.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
@@ -723,12 +724,39 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 		return nil, err
 	}
 
-	out, err := helpers.RenderFiles(pack.Rel(BuildFile), pack.Rel(DefinitionFile))
+	var dataresult []byte
+
+	val := pack.Rel(DefinitionFile)
+	if _, err := os.Stat(pack.Rel(CollectionFile)); err == nil {
+		val = pack.Rel(CollectionFile)
+
+		data, err := ioutil.ReadFile(val)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+val)
+		}
+
+		dataBuild, err := ioutil.ReadFile(pack.Rel(BuildFile))
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+val)
+		}
+		packsRaw, err := pkg.GetRawPackages(data)
+
+		raw := packsRaw.Find(p.GetName(), p.GetCategory(), p.GetVersion())
+
+		dat, err := helpers.RenderHelm(string(dataBuild), raw)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}
+		dataresult = []byte(dat)
+	} else {
+		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
+		}
+		dataresult = []byte(out)
+	}
 
-	return NewLuetCompilationSpec([]byte(out), pack)
+	return NewLuetCompilationSpec(dataresult, pack)
 }
 
 func (cs *LuetCompiler) GetBackend() CompilerBackend {

pkg/package/package.go

@@ -147,6 +147,60 @@ func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) {
 	return unescaped, nil
 }
 
+type rawPackages []map[string]interface{}
+
+func (r rawPackages) Find(name, category, version string) map[string]interface{} {
+	for _, v := range r {
+		if v["name"] == name && v["category"] == category && v["version"] == version {
+			return v
+		}
+	}
+	return map[string]interface{}{}
+}
+
+func GetRawPackages(yml []byte) (rawPackages, error) {
+	var rawPackages struct {
+		Packages []map[string]interface{} `yaml:"packages"`
+	}
+	source, err := yaml.YAMLToJSON(yml)
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+	err = json.Unmarshal(bytes, &rawPackages)
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+	return rawPackages.Packages, nil
+
+}
+func DefaultPackagesFromYaml(yml []byte) ([]DefaultPackage, error) {
+
+	var unescaped struct {
+		Packages []DefaultPackage `json:"packages"`
+	}
+	source, err := yaml.YAMLToJSON(yml)
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+	err = json.Unmarshal(bytes, &unescaped)
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+	return unescaped.Packages, nil
+}
+
 // Major and minor gets escaped when marshalling in JSON, making compiler fails recognizing selectors for expansion
 func (t *DefaultPackage) JSON() ([]byte, error) {
 	buffer := &bytes.Buffer{}

pkg/tree/compiler_recipe.go

@@ -74,10 +74,13 @@ func (r *CompilerRecipe) Load(path string) error {
 			return errors.Wrap(err, "Error on walk path "+currentpath)
 		}
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
+		switch info.Name() {
+		case DefinitionFile:
+
 			pack, err := ReadDefinitionFile(currentpath)
 			if err != nil {
 				return err
@@ -88,13 +91,15 @@ func (r *CompilerRecipe) Load(path string) error {
 			// Instead of rdeps, have a different tree for build deps.
 			compileDefPath := pack.Rel(CompilerDefinitionFile)
 			if helpers.Exists(compileDefPath) {
-			dat, err := ioutil.ReadFile(compileDefPath)
+
+				dat, err := helpers.RenderFiles(compileDefPath, currentpath)
 				if err != nil {
 					return errors.Wrap(err,
-					"Error reading file "+CompilerDefinitionFile+" from "+
+						"Error templating file "+CompilerDefinitionFile+" from "+
 							filepath.Dir(currentpath))
 				}
-			packbuild, err := pkg.DefaultPackageFromYaml(dat)
+
+				packbuild, err := pkg.DefaultPackageFromYaml([]byte(dat))
 				if err != nil {
 					return errors.Wrap(err,
 						"Error reading yaml "+CompilerDefinitionFile+" from "+
@@ -109,6 +114,54 @@ func (r *CompilerRecipe) Load(path string) error {
 				return errors.Wrap(err, "Error creating package "+pack.GetName())
 			}
 
+		case CollectionFile:
+			dat, err := ioutil.ReadFile(currentpath)
+			if err != nil {
+				return errors.Wrap(err, "Error reading file "+currentpath)
+			}
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			packsRaw, err := pkg.GetRawPackages(dat)
+
+			for _, pack := range packs {
+				pack.SetPath(filepath.Dir(currentpath))
+
+				// Instead of rdeps, have a different tree for build deps.
+				compileDefPath := pack.Rel(CompilerDefinitionFile)
+				if helpers.Exists(compileDefPath) {
+
+					raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
+					buildyaml, err := ioutil.ReadFile(compileDefPath)
+					if err != nil {
+						return errors.Wrap(err, "Error reading file "+currentpath)
+					}
+					dat, err := helpers.RenderHelm(string(buildyaml), raw)
+					if err != nil {
+						return errors.Wrap(err,
+							"Error templating file "+CompilerDefinitionFile+" from "+
+								filepath.Dir(currentpath))
+					}
+
+					packbuild, err := pkg.DefaultPackageFromYaml([]byte(dat))
+					if err != nil {
+						return errors.Wrap(err,
+							"Error reading yaml "+CompilerDefinitionFile+" from "+
+								filepath.Dir(currentpath))
+					}
+					pack.Requires(packbuild.GetRequires())
+					pack.Conflicts(packbuild.GetConflicts())
+				}
+
+				_, err = r.Database.CreatePackage(&pack)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+pack.GetName())
+				}
+			}
+
+		}
+
 		return nil
 	}
 

pkg/tree/installer_recipe.go

@@ -85,7 +85,7 @@ func (r *InstallerRecipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
@@ -93,6 +93,9 @@ func (r *InstallerRecipe) Load(path string) error {
 		if err != nil {
 			return errors.Wrap(err, "Error reading file "+currentpath)
 		}
+
+		switch info.Name() {
+		case DefinitionFile:
 			pack, err := pkg.DefaultPackageFromYaml(dat)
 			if err != nil {
 				return errors.Wrap(err, "Error reading yaml "+currentpath)
@@ -105,6 +108,22 @@ func (r *InstallerRecipe) Load(path string) error {
 				return errors.Wrap(err, "Error creating package "+pack.GetName())
 			}
 
+		case CollectionFile:
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			for _, p := range packs {
+				// Path is set only internally when tree is loaded from disk
+				p.SetPath(filepath.Dir(currentpath))
+				_, err = r.Database.CreatePackage(&p)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+p.GetName())
+				}
+			}
+
+		}
+
 		return nil
 	}
 

pkg/tree/recipes.go

@@ -34,6 +34,7 @@ import (
 
 const (
 	DefinitionFile = "definition.yaml"
+	CollectionFile = "collection.yaml"
 )
 
 func NewGeneralRecipe(db pkg.PackageDatabase) Builder { return &Recipe{Database: db} }
@@ -94,7 +95,7 @@ func (r *Recipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
@@ -102,6 +103,9 @@ func (r *Recipe) Load(path string) error {
 		if err != nil {
 			return errors.Wrap(err, "Error reading file "+currentpath)
 		}
+
+		switch info.Name() {
+		case DefinitionFile:
 			pack, err := pkg.DefaultPackageFromYaml(dat)
 			if err != nil {
 				return errors.Wrap(err, "Error reading yaml "+currentpath)
@@ -113,6 +117,21 @@ func (r *Recipe) Load(path string) error {
 			if err != nil {
 				return errors.Wrap(err, "Error creating package "+pack.GetName())
 			}
+		case CollectionFile:
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			for _, p := range packs {
+				// Path is set only internally when tree is loaded from disk
+				p.SetPath(filepath.Dir(currentpath))
+				_, err = r.Database.CreatePackage(&p)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+p.GetName())
+				}
+			}
+
+		}
 
 		return nil
 	}