Allow to define multiple templated packages with collections

Collections, similarly to packages, have a `build.yaml` and
a `finalize.yaml` that are templated for each package.
They have a `collection.yaml` containing a list of
packages that are part of the tree.

pkg/compiler/compiler.go

@@ -38,6 +38,7 @@ import (
 
 const BuildFile = "build.yaml"
 const DefinitionFile = "definition.yaml"
+const CollectionFile = "collection.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
@@ -723,12 +724,39 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 		return nil, err
 	}
 
-	out, err := helpers.RenderFiles(pack.Rel(BuildFile), pack.Rel(DefinitionFile))
+	var dataresult []byte
-	if err != nil {
+
-		return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
+	val := pack.Rel(DefinitionFile)
+	if _, err := os.Stat(pack.Rel(CollectionFile)); err == nil {
+		val = pack.Rel(CollectionFile)
+
+		data, err := ioutil.ReadFile(val)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+val)
+		}
+
+		dataBuild, err := ioutil.ReadFile(pack.Rel(BuildFile))
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+val)
+		}
+		packsRaw, err := pkg.GetRawPackages(data)
+
+		raw := packsRaw.Find(p.GetName(), p.GetCategory(), p.GetVersion())
+
+		dat, err := helpers.RenderHelm(string(dataBuild), raw)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
+		}
+		dataresult = []byte(dat)
+	} else {
+		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val)
+		if err != nil {
+			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
+		}
+		dataresult = []byte(out)
 	}
 
-	return NewLuetCompilationSpec([]byte(out), pack)
+	return NewLuetCompilationSpec(dataresult, pack)
 }
 
 func (cs *LuetCompiler) GetBackend() CompilerBackend {

pkg/package/package.go

@@ -147,6 +147,60 @@ func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) {
 	return unescaped, nil
 }
 
+type rawPackages []map[string]interface{}
+
+func (r rawPackages) Find(name, category, version string) map[string]interface{} {
+	for _, v := range r {
+		if v["name"] == name && v["category"] == category && v["version"] == version {
+			return v
+		}
+	}
+	return map[string]interface{}{}
+}
+
+func GetRawPackages(yml []byte) (rawPackages, error) {
+	var rawPackages struct {
+		Packages []map[string]interface{} `yaml:"packages"`
+	}
+	source, err := yaml.YAMLToJSON(yml)
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+	err = json.Unmarshal(bytes, &rawPackages)
+	if err != nil {
+		return []map[string]interface{}{}, err
+	}
+	return rawPackages.Packages, nil
+
+}
+func DefaultPackagesFromYaml(yml []byte) ([]DefaultPackage, error) {
+
+	var unescaped struct {
+		Packages []DefaultPackage `json:"packages"`
+	}
+	source, err := yaml.YAMLToJSON(yml)
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+
+	rawIn := json.RawMessage(source)
+	bytes, err := rawIn.MarshalJSON()
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+	err = json.Unmarshal(bytes, &unescaped)
+	if err != nil {
+		return []DefaultPackage{}, err
+	}
+	return unescaped.Packages, nil
+}
+
 // Major and minor gets escaped when marshalling in JSON, making compiler fails recognizing selectors for expansion
 func (t *DefaultPackage) JSON() ([]byte, error) {
 	buffer := &bytes.Buffer{}

pkg/tree/compiler_recipe.go

@@ -74,39 +74,92 @@ func (r *CompilerRecipe) Load(path string) error {
 			return errors.Wrap(err, "Error on walk path "+currentpath)
 		}
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
-		pack, err := ReadDefinitionFile(currentpath)
+		switch info.Name() {
-		if err != nil {
+		case DefinitionFile:
-			return err
-		}
-		// Path is set only internally when tree is loaded from disk
-		pack.SetPath(filepath.Dir(currentpath))
 
-		// Instead of rdeps, have a different tree for build deps.
+			pack, err := ReadDefinitionFile(currentpath)
-		compileDefPath := pack.Rel(CompilerDefinitionFile)
-		if helpers.Exists(compileDefPath) {
-			dat, err := ioutil.ReadFile(compileDefPath)
 			if err != nil {
-				return errors.Wrap(err,
+				return err
-					"Error reading file "+CompilerDefinitionFile+" from "+
+			}
-						filepath.Dir(currentpath))
+			// Path is set only internally when tree is loaded from disk
+			pack.SetPath(filepath.Dir(currentpath))
+
+			// Instead of rdeps, have a different tree for build deps.
+			compileDefPath := pack.Rel(CompilerDefinitionFile)
+			if helpers.Exists(compileDefPath) {
+
+				dat, err := helpers.RenderFiles(compileDefPath, currentpath)
+				if err != nil {
+					return errors.Wrap(err,
+						"Error templating file "+CompilerDefinitionFile+" from "+
+							filepath.Dir(currentpath))
+				}
+
+				packbuild, err := pkg.DefaultPackageFromYaml([]byte(dat))
+				if err != nil {
+					return errors.Wrap(err,
+						"Error reading yaml "+CompilerDefinitionFile+" from "+
+							filepath.Dir(currentpath))
+				}
+				pack.Requires(packbuild.GetRequires())
+				pack.Conflicts(packbuild.GetConflicts())
+			}
+
+			_, err = r.Database.CreatePackage(&pack)
+			if err != nil {
+				return errors.Wrap(err, "Error creating package "+pack.GetName())
+			}
+
+		case CollectionFile:
+			dat, err := ioutil.ReadFile(currentpath)
+			if err != nil {
+				return errors.Wrap(err, "Error reading file "+currentpath)
+			}
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			packsRaw, err := pkg.GetRawPackages(dat)
+
+			for _, pack := range packs {
+				pack.SetPath(filepath.Dir(currentpath))
+
+				// Instead of rdeps, have a different tree for build deps.
+				compileDefPath := pack.Rel(CompilerDefinitionFile)
+				if helpers.Exists(compileDefPath) {
+
+					raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
+					buildyaml, err := ioutil.ReadFile(compileDefPath)
+					if err != nil {
+						return errors.Wrap(err, "Error reading file "+currentpath)
+					}
+					dat, err := helpers.RenderHelm(string(buildyaml), raw)
+					if err != nil {
+						return errors.Wrap(err,
+							"Error templating file "+CompilerDefinitionFile+" from "+
+								filepath.Dir(currentpath))
+					}
+
+					packbuild, err := pkg.DefaultPackageFromYaml([]byte(dat))
+					if err != nil {
+						return errors.Wrap(err,
+							"Error reading yaml "+CompilerDefinitionFile+" from "+
+								filepath.Dir(currentpath))
+					}
+					pack.Requires(packbuild.GetRequires())
+					pack.Conflicts(packbuild.GetConflicts())
+				}
+
+				_, err = r.Database.CreatePackage(&pack)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+pack.GetName())
+				}
 			}
-			packbuild, err := pkg.DefaultPackageFromYaml(dat)
-			if err != nil {
-				return errors.Wrap(err,
-					"Error reading yaml "+CompilerDefinitionFile+" from "+
-						filepath.Dir(currentpath))
-			}
-			pack.Requires(packbuild.GetRequires())
-			pack.Conflicts(packbuild.GetConflicts())
-		}
 
-		_, err = r.Database.CreatePackage(&pack)
-		if err != nil {
-			return errors.Wrap(err, "Error creating package "+pack.GetName())
 		}
 
 		return nil

pkg/tree/installer_recipe.go

@@ -85,7 +85,7 @@ func (r *InstallerRecipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
@@ -93,16 +93,35 @@ func (r *InstallerRecipe) Load(path string) error {
 		if err != nil {
 			return errors.Wrap(err, "Error reading file "+currentpath)
 		}
-		pack, err := pkg.DefaultPackageFromYaml(dat)
-		if err != nil {
-			return errors.Wrap(err, "Error reading yaml "+currentpath)
-		}
 
-		// Path is set only internally when tree is loaded from disk
+		switch info.Name() {
-		pack.SetPath(filepath.Dir(currentpath))
+		case DefinitionFile:
-		_, err = r.Database.CreatePackage(&pack)
+			pack, err := pkg.DefaultPackageFromYaml(dat)
-		if err != nil {
+			if err != nil {
-			return errors.Wrap(err, "Error creating package "+pack.GetName())
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+
+			// Path is set only internally when tree is loaded from disk
+			pack.SetPath(filepath.Dir(currentpath))
+			_, err = r.Database.CreatePackage(&pack)
+			if err != nil {
+				return errors.Wrap(err, "Error creating package "+pack.GetName())
+			}
+
+		case CollectionFile:
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			for _, p := range packs {
+				// Path is set only internally when tree is loaded from disk
+				p.SetPath(filepath.Dir(currentpath))
+				_, err = r.Database.CreatePackage(&p)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+p.GetName())
+				}
+			}
+
 		}
 
 		return nil

pkg/tree/recipes.go

@@ -34,6 +34,7 @@ import (
 
 const (
 	DefinitionFile = "definition.yaml"
+	CollectionFile = "collection.yaml"
 )
 
 func NewGeneralRecipe(db pkg.PackageDatabase) Builder { return &Recipe{Database: db} }
@@ -94,7 +95,7 @@ func (r *Recipe) Load(path string) error {
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
 
-		if info.Name() != DefinitionFile {
+		if info.Name() != DefinitionFile && info.Name() != CollectionFile {
 			return nil // Skip with no errors
 		}
 
@@ -102,16 +103,34 @@ func (r *Recipe) Load(path string) error {
 		if err != nil {
 			return errors.Wrap(err, "Error reading file "+currentpath)
 		}
-		pack, err := pkg.DefaultPackageFromYaml(dat)
-		if err != nil {
-			return errors.Wrap(err, "Error reading yaml "+currentpath)
-		}
 
-		// Path is set only internally when tree is loaded from disk
+		switch info.Name() {
-		pack.SetPath(filepath.Dir(currentpath))
+		case DefinitionFile:
-		_, err = r.Database.CreatePackage(&pack)
+			pack, err := pkg.DefaultPackageFromYaml(dat)
-		if err != nil {
+			if err != nil {
-			return errors.Wrap(err, "Error creating package "+pack.GetName())
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+
+			// Path is set only internally when tree is loaded from disk
+			pack.SetPath(filepath.Dir(currentpath))
+			_, err = r.Database.CreatePackage(&pack)
+			if err != nil {
+				return errors.Wrap(err, "Error creating package "+pack.GetName())
+			}
+		case CollectionFile:
+			packs, err := pkg.DefaultPackagesFromYaml(dat)
+			if err != nil {
+				return errors.Wrap(err, "Error reading yaml "+currentpath)
+			}
+			for _, p := range packs {
+				// Path is set only internally when tree is loaded from disk
+				p.SetPath(filepath.Dir(currentpath))
+				_, err = r.Database.CreatePackage(&p)
+				if err != nil {
+					return errors.Wrap(err, "Error creating package "+p.GetName())
+				}
+			}
+
 		}
 
 		return nil