🎨 refactor out common code in tree builder

pkg/tree/collection.go

@@ -0,0 +1,109 @@
+// Copyright © 2019-2022 Ettore Di Giacinto <mudler@luet.io>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package tree
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/api/core/template"
+	"github.com/mudler/luet/pkg/api/core/types"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	"github.com/pkg/errors"
+)
+
+func BuildCollectionParser(srcDir, currentpath, name string, templates []string, db types.PackageDatabase) error {
+	if name != types.PackageCollectionFile {
+		return nil
+	}
+
+	dat, err := ioutil.ReadFile(currentpath)
+	if err != nil {
+		return errors.Wrap(err, "Error reading file "+currentpath)
+	}
+
+	packs, err := types.PackagesFromYAML(dat)
+	if err != nil {
+		return errors.Wrap(err, "Error reading yaml "+currentpath)
+	}
+
+	packsRaw, err := types.GetRawPackages(dat)
+	if err != nil {
+		return errors.Wrap(err, "Error reading raw packages from "+currentpath)
+	}
+
+	for _, pack := range packs {
+		pack.SetPath(filepath.Dir(currentpath))
+		pack.SetTreeDir(srcDir)
+
+		// Instead of rdeps, have a different tree for build deps.
+		compileDefPath := pack.Rel(CompilerDefinitionFile)
+		if fileHelper.Exists(compileDefPath) {
+
+			raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
+			buildyaml, err := ioutil.ReadFile(compileDefPath)
+			if err != nil {
+				return errors.Wrap(err, "Error reading file "+currentpath)
+			}
+			dat, err := template.Render(append(template.ReadFiles(templates...), string(buildyaml)), raw, map[string]interface{}{})
+			if err != nil {
+				return errors.Wrap(err,
+					"Error templating file "+CompilerDefinitionFile+" from "+
+						filepath.Dir(currentpath))
+			}
+
+			packbuild, err := types.PackageFromYaml([]byte(dat))
+			if err != nil {
+				return errors.Wrap(err,
+					"Error reading yaml "+CompilerDefinitionFile+" from "+
+						filepath.Dir(currentpath))
+			}
+			pack.Requires(packbuild.GetRequires())
+
+			pack.Conflicts(packbuild.GetConflicts())
+		}
+
+		_, err = db.CreatePackage(&pack)
+		if err != nil {
+			return errors.Wrap(err, "Error creating package "+pack.GetName())
+		}
+	}
+	return nil
+}
+
+func RuntimeCollectionParser(srcDir, currentpath, name string, templates []string, db types.PackageDatabase) error {
+	if name != types.PackageCollectionFile {
+		return nil
+	}
+
+	dat, err := ioutil.ReadFile(currentpath)
+	if err != nil {
+		return errors.Wrap(err, "Error reading file "+currentpath)
+	}
+	packs, err := types.PackagesFromYAML(dat)
+	if err != nil {
+		return errors.Wrap(err, "Error reading yaml "+currentpath)
+	}
+	for _, p := range packs {
+		// Path is set only internally when tree is loaded from disk
+		p.SetPath(filepath.Dir(currentpath))
+		_, err = db.CreatePackage(&p)
+		if err != nil {
+			return errors.Wrap(err, "Error creating package "+p.GetName())
+		}
+	}
+	return nil
+}

pkg/tree/compiler_recipe.go

@@ -35,8 +35,14 @@ const (
 	CompilerDefinitionFile = "build.yaml"
 )
 
-func NewCompilerRecipe(d types.PackageDatabase) Builder {
+func NewCompilerRecipe(d types.PackageDatabase, fp ...FileParser) Builder {
-	return &CompilerRecipe{Recipe: Recipe{Database: d}}
+	if len(fp) == 0 {
+		fp = []FileParser{
+			BuildCollectionParser,
+			BuildDefinitionParser,
+		}
+	}
+	return &CompilerRecipe{Recipe: Recipe{Database: d}, fileParsers: fp}
 }
 
 func ReadDefinitionFile(path string) (types.Package, error) {
@@ -55,6 +61,7 @@ func ReadDefinitionFile(path string) (types.Package, error) {
 
 // Recipe is the "general" reciper for Trees
 type CompilerRecipe struct {
+	fileParsers []FileParser
 	Recipe
 }
 
@@ -84,100 +91,12 @@ func (r *CompilerRecipe) Load(path string) error {
 			return errors.Wrap(err, "Error on walk path "+currentpath)
 		}
 
-		if info.Name() != types.PackageDefinitionFile && info.Name() != types.PackageCollectionFile {
+		for _, p := range r.fileParsers {
-			return nil // Skip with no errors
+			if err := p(path, currentpath, info.Name(), c, r.Database); err != nil {
-		}
-
-		switch info.Name() {
-		case types.PackageDefinitionFile:
-
-			pack, err := ReadDefinitionFile(currentpath)
-			if err != nil {
 				return err
 			}
-			// Path is set only internally when tree is loaded from disk
-			pack.SetPath(filepath.Dir(currentpath))
-			pack.SetTreeDir(path)
-
-			// Instead of rdeps, have a different tree for build deps.
-			compileDefPath := pack.Rel(CompilerDefinitionFile)
-			if fileHelper.Exists(compileDefPath) {
-				dat, err := template.RenderWithValues(append(c, compileDefPath), currentpath)
-				if err != nil {
-					return errors.Wrap(err,
-						"Error templating file "+CompilerDefinitionFile+" from "+
-							filepath.Dir(currentpath))
 		}
 
-				packbuild, err := types.PackageFromYaml([]byte(dat))
-				if err != nil {
-					return errors.Wrap(err,
-						"Error reading yaml "+CompilerDefinitionFile+" from "+
-							filepath.Dir(currentpath))
-				}
-				pack.Requires(packbuild.GetRequires())
-				pack.Conflicts(packbuild.GetConflicts())
-			}
-
-			_, err = r.Database.CreatePackage(&pack)
-			if err != nil {
-				return errors.Wrap(err, "Error creating package "+pack.GetName())
-			}
-
-		case types.PackageCollectionFile:
-
-			dat, err := ioutil.ReadFile(currentpath)
-			if err != nil {
-				return errors.Wrap(err, "Error reading file "+currentpath)
-			}
-
-			packs, err := types.PackagesFromYAML(dat)
-			if err != nil {
-				return errors.Wrap(err, "Error reading yaml "+currentpath)
-			}
-
-			packsRaw, err := types.GetRawPackages(dat)
-			if err != nil {
-				return errors.Wrap(err, "Error reading raw packages from "+currentpath)
-			}
-
-			for _, pack := range packs {
-				pack.SetPath(filepath.Dir(currentpath))
-				pack.SetTreeDir(path)
-
-				// Instead of rdeps, have a different tree for build deps.
-				compileDefPath := pack.Rel(CompilerDefinitionFile)
-				if fileHelper.Exists(compileDefPath) {
-
-					raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
-					buildyaml, err := ioutil.ReadFile(compileDefPath)
-					if err != nil {
-						return errors.Wrap(err, "Error reading file "+currentpath)
-					}
-					dat, err := template.Render(append(template.ReadFiles(c...), string(buildyaml)), raw, map[string]interface{}{})
-					if err != nil {
-						return errors.Wrap(err,
-							"Error templating file "+CompilerDefinitionFile+" from "+
-								filepath.Dir(currentpath))
-					}
-
-					packbuild, err := types.PackageFromYaml([]byte(dat))
-					if err != nil {
-						return errors.Wrap(err,
-							"Error reading yaml "+CompilerDefinitionFile+" from "+
-								filepath.Dir(currentpath))
-					}
-					pack.Requires(packbuild.GetRequires())
-
-					pack.Conflicts(packbuild.GetConflicts())
-				}
-
-				_, err = r.Database.CreatePackage(&pack)
-				if err != nil {
-					return errors.Wrap(err, "Error creating package "+pack.GetName())
-				}
-			}
-		}
 		return nil
 	}
 

pkg/tree/installer_recipe.go

@@ -36,14 +36,21 @@ const (
 	FinalizerFile = "finalize.yaml"
 )
 
-func NewInstallerRecipe(db types.PackageDatabase) Builder {
+func NewInstallerRecipe(db types.PackageDatabase, fp ...FileParser) Builder {
-	return &InstallerRecipe{Database: db}
+	if len(fp) == 0 {
+		fp = []FileParser{
+			RuntimeCollectionParser,
+			RuntimeDefinitionParser,
+		}
+	}
+	return &InstallerRecipe{Database: db, fileParsers: fp}
 }
 
 // InstallerRecipe is the "general" reciper for Trees
 type InstallerRecipe struct {
 	SourcePath  []string
 	Database    types.PackageDatabase
+	fileParsers []FileParser
 }
 
 func (r *InstallerRecipe) Save(path string) error {
@@ -85,46 +92,11 @@ func (r *InstallerRecipe) Load(path string) error {
 
 	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
-
+		for _, p := range r.fileParsers {
-		if info.Name() != types.PackageDefinitionFile && info.Name() != types.PackageCollectionFile {
+			if err := p(path, currentpath, info.Name(), []string{}, r.Database); err != nil {
-			return nil // Skip with no errors
+				return err
-		}
-
-		dat, err := ioutil.ReadFile(currentpath)
-		if err != nil {
-			return errors.Wrap(err, "Error reading file "+currentpath)
-		}
-
-		switch info.Name() {
-		case types.PackageDefinitionFile:
-			pack, err := types.PackageFromYaml(dat)
-			if err != nil {
-				return errors.Wrap(err, "Error reading yaml "+currentpath)
-			}
-
-			// Path is set only internally when tree is loaded from disk
-			pack.SetPath(filepath.Dir(currentpath))
-			_, err = r.Database.CreatePackage(&pack)
-			if err != nil {
-				return errors.Wrap(err, "Error creating package "+pack.GetName())
-			}
-
-		case types.PackageCollectionFile:
-			packs, err := types.PackagesFromYAML(dat)
-			if err != nil {
-				return errors.Wrap(err, "Error reading yaml "+currentpath)
-			}
-			for _, p := range packs {
-				// Path is set only internally when tree is loaded from disk
-				p.SetPath(filepath.Dir(currentpath))
-				_, err = r.Database.CreatePackage(&p)
-				if err != nil {
-					return errors.Wrap(err, "Error creating package "+p.GetName())
 			}
 		}
-
-		}
-
 		return nil
 	}
 

pkg/tree/package.go

@@ -0,0 +1,88 @@
+// Copyright © 2019-2022 Ettore Di Giacinto <mudler@luet.io>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package tree
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/api/core/template"
+	"github.com/mudler/luet/pkg/api/core/types"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	"github.com/pkg/errors"
+)
+
+func RuntimeDefinitionParser(srcDir, currentpath, name string, templates []string, db types.PackageDatabase) error {
+	if name != types.PackageDefinitionFile {
+		return nil
+	}
+	dat, err := ioutil.ReadFile(currentpath)
+	if err != nil {
+		return errors.Wrap(err, "Error reading file "+currentpath)
+	}
+
+	pack, err := types.PackageFromYaml(dat)
+	if err != nil {
+		return errors.Wrap(err, "Error reading yaml "+currentpath)
+	}
+
+	// Path is set only internally when tree is loaded from disk
+	pack.SetPath(filepath.Dir(currentpath))
+	_, err = db.CreatePackage(&pack)
+	if err != nil {
+		return errors.Wrap(err, "Error creating package "+pack.GetName())
+	}
+	return nil
+}
+
+func BuildDefinitionParser(srcDir, currentpath, name string, templates []string, db types.PackageDatabase) error {
+	if name != types.PackageDefinitionFile {
+		return nil
+	}
+	pack, err := ReadDefinitionFile(currentpath)
+	if err != nil {
+		return err
+	}
+	// Path is set only internally when tree is loaded from disk
+	pack.SetPath(filepath.Dir(currentpath))
+	pack.SetTreeDir(srcDir)
+
+	// Instead of rdeps, have a different tree for build deps.
+	compileDefPath := pack.Rel(CompilerDefinitionFile)
+	if fileHelper.Exists(compileDefPath) {
+		dat, err := template.RenderWithValues(append(templates, compileDefPath), currentpath)
+		if err != nil {
+			return errors.Wrap(err,
+				"Error templating file "+CompilerDefinitionFile+" from "+
+					filepath.Dir(currentpath))
+		}
+
+		packbuild, err := types.PackageFromYaml([]byte(dat))
+		if err != nil {
+			return errors.Wrap(err,
+				"Error reading yaml "+CompilerDefinitionFile+" from "+
+					filepath.Dir(currentpath))
+		}
+		pack.Requires(packbuild.GetRequires())
+		pack.Conflicts(packbuild.GetConflicts())
+	}
+
+	_, err = db.CreatePackage(&pack)
+	if err != nil {
+		return errors.Wrap(err, "Error creating package "+pack.GetName())
+	}
+	return nil
+}

pkg/tree/parser.go

@@ -15,9 +15,13 @@
 
 package tree
 
-import "github.com/mudler/luet/pkg/api/core/types"
+import (
+	"github.com/mudler/luet/pkg/api/core/types"
+)
 
 // parses ebuilds (?) and generates data which is readable by the builder
 type Parser interface {
 	Generate(string) (types.PackageDatabase, error) // Generate scannable luet tree (by builder)
 }
+
+type FileParser func(string, string, string, []string, types.PackageDatabase) error

pkg/tree/recipes.go

@@ -34,10 +34,20 @@ import (
 	"github.com/pkg/errors"
 )
 
-func NewGeneralRecipe(db types.PackageDatabase) Builder { return &Recipe{Database: db} }
+func NewGeneralRecipe(db types.PackageDatabase, fp ...FileParser) Builder {
+	if len(fp) == 0 {
+		fp = []FileParser{
+			RuntimeCollectionParser,
+			RuntimeDefinitionParser,
+		}
+	}
+	return &Recipe{Database: db, fileParsers: fp}
+}
 
 // Recipe is the "general" reciper for Trees
 type Recipe struct {
+	fileParsers []FileParser
+
 	SourcePath []string
 	Database   types.PackageDatabase
 }
@@ -86,53 +96,14 @@ func (r *Recipe) Load(path string) error {
 		r.Database = pkg.NewInMemoryDatabase(false)
 	}
 
-	//r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))
-	// TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean()
-
-	// the function that handles each file or dir
 	var ff = func(currentpath string, info os.FileInfo, err error) error {
-
+		for _, p := range r.fileParsers {
-		if info.Name() != types.PackageDefinitionFile && info.Name() != types.PackageCollectionFile {
+			if err := p(path, currentpath, info.Name(), []string{}, r.Database); err != nil {
-			return nil // Skip with no errors
+				return err
-		}
-
-		dat, err := ioutil.ReadFile(currentpath)
-		if err != nil {
-			return errors.Wrap(err, "Error reading file "+currentpath)
-		}
-
-		switch info.Name() {
-		case types.PackageDefinitionFile:
-			pack, err := types.PackageFromYaml(dat)
-			if err != nil {
-				return errors.Wrap(err, "Error reading yaml "+currentpath)
-			}
-
-			// Path is set only internally when tree is loaded from disk
-			pack.SetPath(filepath.Dir(currentpath))
-			_, err = r.Database.CreatePackage(&pack)
-			if err != nil {
-				return errors.Wrap(err, "Error creating package "+pack.GetName())
-			}
-		case types.PackageCollectionFile:
-			packs, err := types.PackagesFromYAML(dat)
-			if err != nil {
-				return errors.Wrap(err, "Error reading yaml "+currentpath)
-			}
-			for _, p := range packs {
-				// Path is set only internally when tree is loaded from disk
-				p.SetPath(filepath.Dir(currentpath))
-				_, err = r.Database.CreatePackage(&p)
-				if err != nil {
-					return errors.Wrap(err, "Error creating package "+p.GetName())
 			}
 		}
-
-		}
-
 		return nil
 	}
-
 	err := filepath.Walk(path, ff)
 	if err != nil {
 		return err