Tag 0.7.5

This was an issue as we were copying the buildspec instead

cmd/root.go

@@ -38,7 +38,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.7.4"
+	LuetCLIVersion = "0.7.5"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/tree/validate.go

@@ -17,11 +17,15 @@
 package cmd_tree
 
 import (
+	"errors"
 	"fmt"
 	"os"
+	"regexp"
 	"sort"
+	"strconv"
+	"sync"
 
-	//. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/config"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -31,6 +35,174 @@ import (
 	"github.com/spf13/cobra"
 )
 
+func validateWorker(i int,
+	wg *sync.WaitGroup,
+	c <-chan pkg.Package,
+	reciper tree.Builder,
+	withSolver bool,
+	regExcludes, regMatches []*regexp.Regexp,
+	excludes, matches []string,
+	errs chan error) {
+
+	defer wg.Done()
+
+	var depSolver solver.PackageSolver
+	brokenPkgs := 0
+	brokenDeps := 0
+	var errstr string
+
+	emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+	if withSolver {
+		depSolver = solver.NewSolver(pkg.NewInMemoryDatabase(false),
+			reciper.GetDatabase(),
+			emptyInstallationDb)
+	}
+
+	for p := range c {
+		found, err := reciper.GetDatabase().FindPackages(
+			&pkg.DefaultPackage{
+				Name:     p.GetName(),
+				Category: p.GetCategory(),
+				Version:  ">=0",
+			},
+		)
+
+		if err != nil || len(found) < 1 {
+			if err != nil {
+				errstr = err.Error()
+			} else {
+				errstr = "No packages"
+			}
+			Error(fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				errstr,
+			))
+
+			errs <- errors.New(
+				fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					errstr,
+				))
+
+			brokenPkgs++
+		}
+
+		pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
+			p.GetVersion())
+
+		validpkg := true
+
+		if len(matches) > 0 {
+			matched := false
+			for _, rgx := range regMatches {
+				if rgx.MatchString(pkgstr) {
+					matched = true
+					break
+				}
+			}
+
+			if !matched {
+				continue
+			}
+		}
+
+		if len(excludes) > 0 {
+			excluded := false
+			for _, rgx := range regExcludes {
+				if rgx.MatchString(pkgstr) {
+					excluded = true
+					break
+				}
+			}
+
+			if excluded {
+				continue
+			}
+		}
+		Info("Checking package "+fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()), "with", len(p.GetRequires()), "dependencies.")
+
+		all := p.GetRequires()
+		all = append(all, p.GetConflicts()...)
+		for _, r := range all {
+			var deps pkg.Packages
+			var err error
+			if r.IsSelector() {
+				deps, err = reciper.GetDatabase().FindPackages(
+					&pkg.DefaultPackage{
+						Name:     r.GetName(),
+						Category: r.GetCategory(),
+						Version:  r.GetVersion(),
+					},
+				)
+			} else {
+				deps = append(deps, r)
+			}
+
+			if err != nil || len(deps) < 1 {
+				if err != nil {
+					errstr = err.Error()
+				} else {
+					errstr = "No packages"
+				}
+				Error(fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					r.GetCategory(), r.GetName(), r.GetVersion(),
+					errstr,
+				))
+
+				errs <- errors.New(
+					fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
+						p.GetCategory(), p.GetName(), p.GetVersion(),
+						r.GetCategory(), r.GetName(), r.GetVersion(),
+						errstr))
+
+				brokenDeps++
+
+				validpkg = false
+
+			} else {
+
+				Debug("Find packages for dep",
+					fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+
+				if withSolver {
+					Spinner(32)
+					solution, err := depSolver.Install(pkg.Packages{r})
+					ass := solution.SearchByName(r.GetPackageName())
+					if err == nil {
+						_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+					}
+					SpinnerStop()
+
+					if err != nil {
+
+						Error(fmt.Sprintf("%s/%s-%s: solver broken for dep %s/%s-%s - %s",
+							p.GetCategory(), p.GetName(), p.GetVersion(),
+							r.GetCategory(), r.GetName(), r.GetVersion(),
+							err.Error(),
+						))
+
+						errs <- errors.New(
+							fmt.Sprintf("%s/%s-%s: solver broken for Dep %s/%s-%s - %s",
+								p.GetCategory(), p.GetName(), p.GetVersion(),
+								r.GetCategory(), r.GetName(), r.GetVersion(),
+								err.Error()))
+
+						brokenDeps++
+						validpkg = false
+					}
+
+				}
+			}
+
+		}
+
+		if !validpkg {
+			brokenPkgs++
+		}
+	}
+}
+
 func NewTreeValidateCommand() *cobra.Command {
 	var excludes []string
 	var matches []string
@@ -46,13 +218,8 @@ func NewTreeValidateCommand() *cobra.Command {
 			}
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			var depSolver solver.PackageSolver
-			var errstr string
+			concurrency := LuetCfg.GetGeneral().Concurrency
 
-			errors := make([]string, 0)
-
-			brokenPkgs := 0
-			brokenDeps := 0
 			withSolver, _ := cmd.Flags().GetBool("with-solver")
 
 			reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
@@ -63,13 +230,6 @@ func NewTreeValidateCommand() *cobra.Command {
 				}
 			}
 
-			emptyInstallationDb := pkg.NewInMemoryDatabase(false)
-			if withSolver {
-				depSolver = solver.NewSolver(pkg.NewInMemoryDatabase(false),
-					reciper.GetDatabase(),
-					emptyInstallationDb)
-			}
-
 			regExcludes, err := helpers.CreateRegexArray(excludes)
 			if err != nil {
 				Fatal(err.Error())
@@ -79,160 +239,44 @@ func NewTreeValidateCommand() *cobra.Command {
 				Fatal(err.Error())
 			}
 
-			for _, p := range reciper.GetDatabase().World() {
+			all := make(chan pkg.Package)
+			errs := make(chan error)
 
-				found, err := reciper.GetDatabase().FindPackages(
-					&pkg.DefaultPackage{
-						Name:     p.GetName(),
-						Category: p.GetCategory(),
-						Version:  ">=0",
-					},
-				)
+			var wg = new(sync.WaitGroup)
 
-				if err != nil || len(found) < 1 {
-					if err != nil {
-						errstr = err.Error()
-					} else {
-						errstr = "No packages"
-					}
-					Error(fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
-						p.GetCategory(), p.GetName(), p.GetVersion(),
-						errstr,
-					))
-
-					errors = append(errors,
-						fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
-							p.GetCategory(), p.GetName(), p.GetVersion(),
-							errstr,
-						))
-
-					brokenPkgs++
-				}
-
-				pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
-					p.GetVersion())
-
-				validpkg := true
-
-				if len(matches) > 0 {
-					matched := false
-					for _, rgx := range regMatches {
-						if rgx.MatchString(pkgstr) {
-							matched = true
-							break
-						}
-					}
-
-					if !matched {
-						continue
-					}
-				}
-
-				if len(excludes) > 0 {
-					excluded := false
-					for _, rgx := range regExcludes {
-						if rgx.MatchString(pkgstr) {
-							excluded = true
-							break
-						}
-					}
-
-					if excluded {
-						continue
-					}
-				}
-				Info("Checking package "+fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()), "with", len(p.GetRequires()), "dependencies.")
-
-				all := p.GetRequires()
-				all = append(all, p.GetConflicts()...)
-				for _, r := range all {
-					var deps pkg.Packages
-					var err error
-					if r.IsSelector() {
-						deps, err = reciper.GetDatabase().FindPackages(
-							&pkg.DefaultPackage{
-								Name:     r.GetName(),
-								Category: r.GetCategory(),
-								Version:  r.GetVersion(),
-							},
-						)
-					} else {
-						deps = append(deps, r)
-					}
-
-					if err != nil || len(deps) < 1 {
-						if err != nil {
-							errstr = err.Error()
-						} else {
-							errstr = "No packages"
-						}
-						Error(fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
-							p.GetCategory(), p.GetName(), p.GetVersion(),
-							r.GetCategory(), r.GetName(), r.GetVersion(),
-							errstr,
-						))
-
-						errors = append(errors,
-							fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
-								p.GetCategory(), p.GetName(), p.GetVersion(),
-								r.GetCategory(), r.GetName(), r.GetVersion(),
-								errstr))
-
-						brokenDeps++
-
-						validpkg = false
-
-					} else {
-
-						Debug("Find packages for dep",
-							fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
-
-						if withSolver {
-							Spinner(32)
-							solution, err := depSolver.Install(pkg.Packages{r})
-							ass := solution.SearchByName(r.GetPackageName())
-							if err == nil {
-								_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
-							}
-							SpinnerStop()
-
-							if err != nil {
-
-								Error(fmt.Sprintf("%s/%s-%s: solver broken for dep %s/%s-%s - %s",
-									p.GetCategory(), p.GetName(), p.GetVersion(),
-									r.GetCategory(), r.GetName(), r.GetVersion(),
-									err.Error(),
-								))
-
-								errors = append(errors,
-									fmt.Sprintf("%s/%s-%s: solver broken for Dep %s/%s-%s - %s",
-										p.GetCategory(), p.GetName(), p.GetVersion(),
-										r.GetCategory(), r.GetName(), r.GetVersion(),
-										err.Error()))
-
-								brokenDeps++
-								validpkg = false
-							}
-
-						}
-					}
-
-				}
-
-				if !validpkg {
-					brokenPkgs++
-				}
+			for i := 0; i < concurrency; i++ {
+				wg.Add(1)
+				go validateWorker(i, wg, all,
+					reciper, withSolver, regExcludes, regMatches, excludes, matches,
+					errs)
 			}
+			for _, p := range reciper.GetDatabase().World() {
+				all <- p
+			}
+			close(all)
 
-			sort.Strings(errors)
-			for _, e := range errors {
+			// Wait separately and once done close the channel
+			go func() {
+				wg.Wait()
+				close(errs)
+			}()
+
+			stringerrs := []string{}
+			for e := range errs {
+				stringerrs = append(stringerrs, e.Error())
+			}
+			sort.Strings(stringerrs)
+			for _, e := range stringerrs {
 				fmt.Println(e)
 			}
-			fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
 
-			if brokenPkgs > 0 {
-				os.Exit(1)
+			// fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
+			if len(stringerrs) != 0 {
+				Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
+				//	if brokenPkgs > 0 {
+				//os.Exit(1)
 			} else {
+				Info("All good! :white_check_mark:")
 				os.Exit(0)
 			}
 		},

pkg/installer/installer.go

@@ -222,7 +222,11 @@ func (l *LuetInstaller) Reclaim(s *System) error {
 		FILES:
 			for _, f := range artefact.GetFiles() {
 				if helpers.Exists(filepath.Join(s.Target, f)) {
-					toMerge = append(toMerge, ArtifactMatch{Artifact: artefact, Package: artefact.GetCompileSpec().GetPackage()})
+					p, err := repo.GetTree().GetDatabase().FindPackage(artefact.GetCompileSpec().GetPackage())
+					if err != nil {
+						return err
+					}
+					toMerge = append(toMerge, ArtifactMatch{Artifact: artefact, Package: p})
 					break FILES
 				}
 			}

pkg/package/database_boltdb.go

@@ -17,6 +17,7 @@ package pkg
 
 import (
 	"encoding/base64"
+	"fmt"
 	"os"
 	"regexp"
 	"strconv"
@@ -228,7 +229,7 @@ func (db *BoltDatabase) getProvide(p Package) (Package, error) {
 		db.Unlock()
 
 		if !ok {
-			return nil, errors.New("No versions found for package")
+			return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 		}
 
 		for ve, _ := range versions {
@@ -240,7 +241,7 @@ func (db *BoltDatabase) getProvide(p Package) (Package, error) {
 			if match {
 				pa, ok := db.ProvidesDatabase[p.GetPackageName()][ve]
 				if !ok {
-					return nil, errors.New("No versions found for package")
+					return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 				}
 				return pa, nil //pick the first (we shouldn't have providers that are conflicting)
 				// TODO: A find dbcall here would recurse, but would give chance to have providers of providers
@@ -309,7 +310,7 @@ func (db *BoltDatabase) RemovePackage(p Package) error {
 	var found DefaultPackage
 	err = bolt.Select(q.Eq("Name", p.GetName()), q.Eq("Category", p.GetCategory()), q.Eq("Version", p.GetVersion())).Limit(1).Delete(&found)
 	if err != nil {
-		return errors.Wrap(err, "No package found to delete")
+		return errors.New(fmt.Sprintf("Package not found: %s", p.HumanReadableString()))
 	}
 	return nil
 }

pkg/package/database_mem.go

@@ -18,6 +18,7 @@ package pkg
 import (
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"regexp"
 	"sync"
 
@@ -59,7 +60,7 @@ func (db *InMemoryDatabase) Get(s string) (string, error) {
 	defer db.Unlock()
 	pa, ok := db.Database[s]
 	if !ok {
-		return "", errors.New("No key found with that id")
+		return "", errors.New(fmt.Sprintf("No key found for: %s", s))
 	}
 	return pa, nil
 }
@@ -248,7 +249,7 @@ func (db *InMemoryDatabase) FindPackages(p Package) (Packages, error) {
 	}
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
-		return nil, errors.New("No versions found for package")
+		return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 	}
 	var versionsInWorld []Package
 	for ve, _ := range versions {
@@ -277,7 +278,7 @@ func (db *InMemoryDatabase) UpdatePackage(p Package) error {
 
 	return db.Set(p.GetFingerPrint(), enc)
 
-	return errors.New("Package not found")
+	return errors.New(fmt.Sprintf("Package not found: %s", p.HumanReadableString()))
 }
 
 func (db *InMemoryDatabase) GetPackages() []string {
@@ -302,7 +303,7 @@ func (db *InMemoryDatabase) GetPackageFiles(p Package) ([]string, error) {
 
 	pa, ok := db.FileDatabase[p.GetFingerPrint()]
 	if !ok {
-		return pa, errors.New("No key found with that id")
+		return pa, errors.New(fmt.Sprintf("No key found for: %s", p.HumanReadableString()))
 	}
 
 	return pa, nil

pkg/package/package.go

@@ -480,7 +480,11 @@ func DecodePackage(ID string, db PackageDatabase) (Package, error) {
 	return db.GetPackage(ID)
 }
 
-func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Package) (bool, error) {
+func (pack *DefaultPackage) scanRequires(definitiondb PackageDatabase, s Package, visited map[string]interface{}) (bool, error) {
+	if _, ok := visited[pack.HumanReadableString()]; ok {
+		return false, nil
+	}
+	visited[pack.HumanReadableString()] = true
 	p, err := definitiondb.FindPackage(pack)
 	if err != nil {
 		p = pack //relax things
@@ -498,7 +502,7 @@ func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Pac
 				return true, nil
 			}
 		}
-		if contains, err := re.RequiresContains(definitiondb, s); err == nil && contains {
+		if contains, err := re.scanRequires(definitiondb, s, visited); err == nil && contains {
 			return true, nil
 		}
 	}
@@ -506,6 +510,12 @@ func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Pac
 	return false, nil
 }
 
+// RequiresContains recursively scans into the database packages dependencies to find a match with the given package
+// It is used by the solver during uninstall.
+func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Package) (bool, error) {
+	return pack.scanRequires(definitiondb, s, make(map[string]interface{}))
+}
+
 // Best returns the best version of the package (the most bigger) from a list
 // Accepts a versioner interface to change the ordering policy. If null is supplied
 // It defaults to version.WrappedVersioner which supports both semver and debian versioning
@@ -540,7 +550,11 @@ func (set Packages) Unique() Packages {
 	return result
 }
 
-func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db PackageDatabase) ([]bf.Formula, error) {
+func (pack *DefaultPackage) buildFormula(definitiondb PackageDatabase, db PackageDatabase, visited map[string]interface{}) ([]bf.Formula, error) {
+	if _, ok := visited[pack.HumanReadableString()]; ok {
+		return nil, nil
+	}
+	visited[pack.HumanReadableString()] = true
 	p, err := definitiondb.FindPackage(pack)
 	if err != nil {
 		p = pack // Relax failures and trust the def
@@ -642,8 +656,8 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 		}
 		B := bf.Var(encodedB)
 		formulas = append(formulas, bf.Or(bf.Not(A), B))
-
-		f, err := required.BuildFormula(definitiondb, db)
+		r := required.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
+		f, err := r.buildFormula(definitiondb, db, visited)
 		if err != nil {
 			return nil, err
 		}
@@ -672,8 +686,8 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 						B := bf.Var(encodedB)
 						formulas = append(formulas, bf.Or(bf.Not(A),
 							bf.Not(B)))
-
-						f, err := p.BuildFormula(definitiondb, db)
+						r := p.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
+						f, err := r.buildFormula(definitiondb, db, visited)
 						if err != nil {
 							return nil, err
 						}
@@ -692,7 +706,8 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 		formulas = append(formulas, bf.Or(bf.Not(A),
 			bf.Not(B)))
 
-		f, err := required.BuildFormula(definitiondb, db)
+		r := required.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage
+		f, err := r.buildFormula(definitiondb, db, visited)
 		if err != nil {
 			return nil, err
 		}
@@ -703,6 +718,10 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 	return formulas, nil
 }
 
+func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db PackageDatabase) ([]bf.Formula, error) {
+	return pack.buildFormula(definitiondb, db, make(map[string]interface{}))
+}
+
 func (p *DefaultPackage) Explain() {
 
 	fmt.Println("====================")