Tag 0.7.5

This was an issue as we were copying the buildspec instead

cmd/root.go

@@ -38,7 +38,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.7.4"
+	LuetCLIVersion = "0.7.5"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/tree/validate.go

@@ -17,11 +17,15 @@
 package cmd_tree
 
 import (
+	"errors"
 	"fmt"
 	"os"
+	"regexp"
 	"sort"
+	"strconv"
+	"sync"
 
-	//. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/config"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -31,37 +35,21 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func NewTreeValidateCommand() *cobra.Command {
+func validateWorker(i int,
-	var excludes []string
+	wg *sync.WaitGroup,
-	var matches []string
+	c <-chan pkg.Package,
-	var treePaths []string
+	reciper tree.Builder,
+	withSolver bool,
+	regExcludes, regMatches []*regexp.Regexp,
+	excludes, matches []string,
+	errs chan error) {
+
+	defer wg.Done()
 
-	var ans = &cobra.Command{
-		Use:   "validate [OPTIONS]",
-		Short: "Validate a tree or a list of packages",
-		Args:  cobra.OnlyValidArgs,
-		PreRun: func(cmd *cobra.Command, args []string) {
-			if len(treePaths) < 1 {
-				Fatal("Mandatory tree param missing.")
-			}
-		},
-		Run: func(cmd *cobra.Command, args []string) {
 	var depSolver solver.PackageSolver
-			var errstr string
-
-			errors := make([]string, 0)
-
 	brokenPkgs := 0
 	brokenDeps := 0
-			withSolver, _ := cmd.Flags().GetBool("with-solver")
+	var errstr string
-
-			reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
-			for _, treePath := range treePaths {
-				err := reciper.Load(treePath)
-				if err != nil {
-					Fatal("Error on load tree ", err)
-				}
-			}
 
 	emptyInstallationDb := pkg.NewInMemoryDatabase(false)
 	if withSolver {
@@ -70,17 +58,7 @@ func NewTreeValidateCommand() *cobra.Command {
 			emptyInstallationDb)
 	}
 
-			regExcludes, err := helpers.CreateRegexArray(excludes)
+	for p := range c {
-			if err != nil {
-				Fatal(err.Error())
-			}
-			regMatches, err := helpers.CreateRegexArray(matches)
-			if err != nil {
-				Fatal(err.Error())
-			}
-
-			for _, p := range reciper.GetDatabase().World() {
-
 		found, err := reciper.GetDatabase().FindPackages(
 			&pkg.DefaultPackage{
 				Name:     p.GetName(),
@@ -100,7 +78,7 @@ func NewTreeValidateCommand() *cobra.Command {
 				errstr,
 			))
 
-					errors = append(errors,
+			errs <- errors.New(
 				fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
 					p.GetCategory(), p.GetName(), p.GetVersion(),
 					errstr,
@@ -172,7 +150,7 @@ func NewTreeValidateCommand() *cobra.Command {
 					errstr,
 				))
 
-						errors = append(errors,
+				errs <- errors.New(
 					fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
 						p.GetCategory(), p.GetName(), p.GetVersion(),
 						r.GetCategory(), r.GetName(), r.GetVersion(),
@@ -204,7 +182,7 @@ func NewTreeValidateCommand() *cobra.Command {
 							err.Error(),
 						))
 
-								errors = append(errors,
+						errs <- errors.New(
 							fmt.Sprintf("%s/%s-%s: solver broken for Dep %s/%s-%s - %s",
 								p.GetCategory(), p.GetName(), p.GetVersion(),
 								r.GetCategory(), r.GetName(), r.GetVersion(),
@@ -223,16 +201,82 @@ func NewTreeValidateCommand() *cobra.Command {
 			brokenPkgs++
 		}
 	}
+}
 
-			sort.Strings(errors)
+func NewTreeValidateCommand() *cobra.Command {
-			for _, e := range errors {
+	var excludes []string
+	var matches []string
+	var treePaths []string
+
+	var ans = &cobra.Command{
+		Use:   "validate [OPTIONS]",
+		Short: "Validate a tree or a list of packages",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			if len(treePaths) < 1 {
+				Fatal("Mandatory tree param missing.")
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			concurrency := LuetCfg.GetGeneral().Concurrency
+
+			withSolver, _ := cmd.Flags().GetBool("with-solver")
+
+			reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
+			for _, treePath := range treePaths {
+				err := reciper.Load(treePath)
+				if err != nil {
+					Fatal("Error on load tree ", err)
+				}
+			}
+
+			regExcludes, err := helpers.CreateRegexArray(excludes)
+			if err != nil {
+				Fatal(err.Error())
+			}
+			regMatches, err := helpers.CreateRegexArray(matches)
+			if err != nil {
+				Fatal(err.Error())
+			}
+
+			all := make(chan pkg.Package)
+			errs := make(chan error)
+
+			var wg = new(sync.WaitGroup)
+
+			for i := 0; i < concurrency; i++ {
+				wg.Add(1)
+				go validateWorker(i, wg, all,
+					reciper, withSolver, regExcludes, regMatches, excludes, matches,
+					errs)
+			}
+			for _, p := range reciper.GetDatabase().World() {
+				all <- p
+			}
+			close(all)
+
+			// Wait separately and once done close the channel
+			go func() {
+				wg.Wait()
+				close(errs)
+			}()
+
+			stringerrs := []string{}
+			for e := range errs {
+				stringerrs = append(stringerrs, e.Error())
+			}
+			sort.Strings(stringerrs)
+			for _, e := range stringerrs {
 				fmt.Println(e)
 			}
-			fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
 
-			if brokenPkgs > 0 {
+			// fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
-				os.Exit(1)
+			if len(stringerrs) != 0 {
+				Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
+				//	if brokenPkgs > 0 {
+				//os.Exit(1)
 			} else {
+				Info("All good! :white_check_mark:")
 				os.Exit(0)
 			}
 		},

pkg/installer/installer.go

@@ -222,7 +222,11 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 		FILES:
 			for _, f := range artefact.GetFiles() {
 				if helpers.Exists(filepath.Join(s.Target, f)) {
-					toMerge = append(toMerge, ArtifactMatch{Artifact: artefact, Package: artefact.GetCompileSpec().GetPackage()})
+					p, err := repo.GetTree().GetDatabase().FindPackage(artefact.GetCompileSpec().GetPackage())
+					if err != nil {
+						return err
+					}
+					toMerge = append(toMerge, ArtifactMatch{Artifact: artefact, Package: p})
 					break FILES
 				}
 			}

pkg/package/database_boltdb.go

@@ -17,6 +17,7 @@ package pkg
 
 import (
 	"encoding/base64"
+	"fmt"
 	"os"
 	"regexp"
 	"strconv"
@@ -228,7 +229,7 @@ func (db *BoltDatabase) getProvide(p Package) (Package, error) {
 		db.Unlock()
 
 		if !ok {
-			return nil, errors.New("No versions found for package")
+			return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 		}
 
 		for ve, _ := range versions {
@@ -240,7 +241,7 @@ func (db *BoltDatabase) getProvide(p Package) (Package, error) {
 			if match {
 				pa, ok := db.ProvidesDatabase[p.GetPackageName()][ve]
 				if !ok {
-					return nil, errors.New("No versions found for package")
+					return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 				}
 				return pa, nil //pick the first (we shouldn't have providers that are conflicting)
 				// TODO: A find dbcall here would recurse, but would give chance to have providers of providers
@@ -309,7 +310,7 @@ func (db *BoltDatabase) RemovePackage(p Package) error {
 	var found DefaultPackage
 	err = bolt.Select(q.Eq("Name", p.GetName()), q.Eq("Category", p.GetCategory()), q.Eq("Version", p.GetVersion())).Limit(1).Delete(&found)
 	if err != nil {
-		return errors.Wrap(err, "No package found to delete")
+		return errors.New(fmt.Sprintf("Package not found: %s", p.HumanReadableString()))
 	}
 	return nil
 }

pkg/package/database_mem.go

@@ -18,6 +18,7 @@ package pkg
 import (
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"regexp"
 	"sync"
 
@@ -59,7 +60,7 @@ func (db *InMemoryDatabase) Get(s string) (string, error) {
 	defer db.Unlock()
 	pa, ok := db.Database[s]
 	if !ok {
-		return "", errors.New("No key found with that id")
+		return "", errors.New(fmt.Sprintf("No key found for: %s", s))
 	}
 	return pa, nil
 }
@@ -248,7 +249,7 @@ func (db *InMemoryDatabase) FindPackages(p Package) (Packages, error) {
 	}
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
-		return nil, errors.New("No versions found for package")
+		return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 	}
 	var versionsInWorld []Package
 	for ve, _ := range versions {
@@ -277,7 +278,7 @@ func (db *InMemoryDatabase) UpdatePackage(p Package) error {
 
 	return db.Set(p.GetFingerPrint(), enc)
 
-	return errors.New("Package not found")
+	return errors.New(fmt.Sprintf("Package not found: %s", p.HumanReadableString()))
 }
 
 func (db *InMemoryDatabase) GetPackages() []string {
@@ -302,7 +303,7 @@ func (db *InMemoryDatabase) GetPackageFiles(p Package) ([]string, error) {
 
 	pa, ok := db.FileDatabase[p.GetFingerPrint()]
 	if !ok {
-		return pa, errors.New("No key found with that id")
+		return pa, errors.New(fmt.Sprintf("No key found for: %s", p.HumanReadableString()))
 	}
 
 	return pa, nil

pkg/package/package.go

@@ -480,7 +480,11 @@ func DecodePackage(ID string, db PackageDatabase) (Package, error) {
 	return db.GetPackage(ID)
 }
 
-func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Package) (bool, error) {
+func (pack *DefaultPackage) scanRequires(definitiondb PackageDatabase, s Package, visited map[string]interface{}) (bool, error) {
+	if _, ok := visited[pack.HumanReadableString()]; ok {
+		return false, nil
+	}
+	visited[pack.HumanReadableString()] = true
 	p, err := definitiondb.FindPackage(pack)
 	if err != nil {
 		p = pack //relax things
@@ -498,7 +502,7 @@ func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Pac
 				return true, nil
 			}
 		}
-		if contains, err := re.RequiresContains(definitiondb, s); err == nil && contains {
+		if contains, err := re.scanRequires(definitiondb, s, visited); err == nil && contains {
 			return true, nil
 		}
 	}
@@ -506,6 +510,12 @@ func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Pac
 	return false, nil
 }
 
+// RequiresContains recursively scans into the database packages dependencies to find a match with the given package
+// It is used by the solver during uninstall.
+func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Package) (bool, error) {
+	return pack.scanRequires(definitiondb, s, make(map[string]interface{}))
+}
+
 // Best returns the best version of the package (the most bigger) from a list
 // Accepts a versioner interface to change the ordering policy. If null is supplied
 // It defaults to version.WrappedVersioner which supports both semver and debian versioning
@@ -540,7 +550,11 @@ func (set Packages) Unique() Packages {
 	return result
 }
 
-func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db PackageDatabase) ([]bf.Formula, error) {
+func (pack *DefaultPackage) buildFormula(definitiondb PackageDatabase, db PackageDatabase, visited map[string]interface{}) ([]bf.Formula, error) {
+	if _, ok := visited[pack.HumanReadableString()]; ok {
+		return nil, nil
+	}
+	visited[pack.HumanReadableString()] = true
 	p, err := definitiondb.FindPackage(pack)
 	if err != nil {
 		p = pack // Relax failures and trust the def
@@ -642,8 +656,8 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 		}
 		B := bf.Var(encodedB)
 		formulas = append(formulas, bf.Or(bf.Not(A), B))
-
+		r := required.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
-		f, err := required.BuildFormula(definitiondb, db)
+		f, err := r.buildFormula(definitiondb, db, visited)
 		if err != nil {
 			return nil, err
 		}
@@ -672,8 +686,8 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 						B := bf.Var(encodedB)
 						formulas = append(formulas, bf.Or(bf.Not(A),
 							bf.Not(B)))
-
+						r := p.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
-						f, err := p.BuildFormula(definitiondb, db)
+						f, err := r.buildFormula(definitiondb, db, visited)
 						if err != nil {
 							return nil, err
 						}
@@ -692,7 +706,8 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 		formulas = append(formulas, bf.Or(bf.Not(A),
 			bf.Not(B)))
 
-		f, err := required.BuildFormula(definitiondb, db)
+		r := required.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
+		f, err := r.buildFormula(definitiondb, db, visited)
 		if err != nil {
 			return nil, err
 		}
@@ -703,6 +718,10 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 	return formulas, nil
 }
 
+func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db PackageDatabase) ([]bf.Formula, error) {
+	return pack.buildFormula(definitiondb, db, make(map[string]interface{}))
+}
+
 func (p *DefaultPackage) Explain() {
 
 	fmt.Println("====================")