cmd/tree/validate: Integrate validation of buildtime deps

By default both runtime and buildtime deps are checked.
With the option --only-buildtime is possible analyze only
buildtime deps or instead with the option --only-runtime
only the runtime deps.

Signed-off-by: Daniele Rondina <geaaru@sabayonlinux.org>

cmd/tree/validate.go

@@ -35,36 +35,53 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func validateWorker(i int,
+type ValidateOpts struct {
-	wg *sync.WaitGroup,
+	WithSolver    bool
-	c <-chan pkg.Package,
+	OnlyRuntime   bool
-	reciper tree.Builder,
+	OnlyBuildtime bool
-	withSolver bool,
+	RegExcludes   []*regexp.Regexp
-	regExcludes, regMatches []*regexp.Regexp,
+	RegMatches    []*regexp.Regexp
-	excludes, matches []string,
+	Excludes      []string
-	errs chan error) {
+	Matches       []string
 
-	defer wg.Done()
+	// Runtime validate stuff
+	RuntimeCacheDeps *pkg.InMemoryDatabase
+	RuntimeReciper   *tree.InstallerRecipe
+
+	// Buildtime validate stuff
+	BuildtimeCacheDeps *pkg.InMemoryDatabase
+	BuildtimeReciper   *tree.CompilerRecipe
+
+	Mutex      sync.Mutex
+	BrokenPkgs int
+	BrokenDeps int
+}
+
+func (o *ValidateOpts) IncrBrokenPkgs() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenPkgs++
+}
+
+func (o *ValidateOpts) IncrBrokenDeps() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenDeps++
+}
+
+func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, reciper tree.Builder, cacheDeps *pkg.InMemoryDatabase) error {
+	var errstr string
+	var ans error
 
 	var depSolver solver.PackageSolver
-	var cacheDeps *pkg.InMemoryDatabase
-	brokenPkgs := 0
-	brokenDeps := 0
-	var errstr string
 
+	if opts.WithSolver {
 		emptyInstallationDb := pkg.NewInMemoryDatabase(false)
-	if withSolver {
 		depSolver = solver.NewSolver(pkg.NewInMemoryDatabase(false),
 			reciper.GetDatabase(),
 			emptyInstallationDb)
-
-		// Use Singleton in memory cache for speedup dependencies
-		// analysis
-		cacheDeps = pkg.NewInMemoryDatabase(true).(*pkg.InMemoryDatabase)
 	}
 
-	for p := range c {
-
 	found, err := reciper.GetDatabase().FindPackages(
 		&pkg.DefaultPackage{
 			Name:     p.GetName(),
@@ -79,28 +96,50 @@ func validateWorker(i int,
 		} else {
 			errstr = "No packages"
 		}
-			Error(fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
+		Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+			checkType,
 			p.GetCategory(), p.GetName(), p.GetVersion(),
 			errstr,
 		))
 
-			errs <- errors.New(
+		opts.IncrBrokenDeps()
-				fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
+
+		return errors.New(
+			fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+				checkType,
 				p.GetCategory(), p.GetName(), p.GetVersion(),
 				errstr,
 			))
-
-			brokenPkgs++
 	}
 
+	// Ensure that we use the right package from right recipier for deps
+	pReciper, err := reciper.GetDatabase().FindPackage(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  p.GetVersion(),
+		},
+	)
+	if err != nil {
+		errstr = fmt.Sprintf("[%9s] %s/%s-%s: Error on retrieve package - %s.",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			err.Error(),
+		)
+		Error(errstr)
+
+		return errors.New(errstr)
+	}
+	p = pReciper
+
 	pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
 		p.GetVersion())
 
 	validpkg := true
 
-		if len(matches) > 0 {
+	if len(opts.Matches) > 0 {
 		matched := false
-			for _, rgx := range regMatches {
+		for _, rgx := range opts.RegMatches {
 			if rgx.MatchString(pkgstr) {
 				matched = true
 				break
@@ -108,13 +147,13 @@ func validateWorker(i int,
 		}
 
 		if !matched {
-				continue
+			return nil
 		}
 	}
 
-		if len(excludes) > 0 {
+	if len(opts.Excludes) > 0 {
 		excluded := false
-			for _, rgx := range regExcludes {
+		for _, rgx := range opts.RegExcludes {
 			if rgx.MatchString(pkgstr) {
 				excluded = true
 				break
@@ -122,11 +161,11 @@ func validateWorker(i int,
 		}
 
 		if excluded {
-				continue
+			return nil
 		}
 	}
 
-		Info("Checking package "+
+	Info(fmt.Sprintf("[%9s] Checking package ", checkType)+
 		fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
 		"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
 
@@ -154,30 +193,33 @@ func validateWorker(i int,
 			} else {
 				errstr = "No packages"
 			}
-				Error(fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
+			Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+				checkType,
 				p.GetCategory(), p.GetName(), p.GetVersion(),
 				r.GetCategory(), r.GetName(), r.GetVersion(),
 				errstr,
 			))
 
-				errs <- errors.New(
+			opts.IncrBrokenDeps()
-					fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
+
+			ans = errors.New(
+				fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+					checkType,
 					p.GetCategory(), p.GetName(), p.GetVersion(),
 					r.GetCategory(), r.GetName(), r.GetVersion(),
 					errstr))
 
-				brokenDeps++
-
 			validpkg = false
 
 		} else {
 
-				Debug("Find packages for dep",
+			Debug(fmt.Sprintf("[%9s] Find packages for dep", checkType),
 				fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
 
-				if withSolver {
+			if opts.WithSolver {
 
-					Info(fmt.Sprintf("  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
+				Info(fmt.Sprintf("[%9s]  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
+					checkType,
 					idx+1, len(all),
 					p.GetCategory(), p.GetName(), p.GetVersion(),
 					r.GetCategory(), r.GetName(), r.GetVersion(),
@@ -186,7 +228,7 @@ func validateWorker(i int,
 				// Check if the solver is already been done for the deep
 				_, err := cacheDeps.Get(r.HashFingerprint(""))
 				if err == nil {
-						Debug("  :direct_hit: Cache Hit for dep",
+					Debug(fmt.Sprintf("[%9s]  :direct_hit: Cache Hit for dep", checkType),
 						fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
 					continue
 				}
@@ -201,19 +243,21 @@ func validateWorker(i int,
 
 				if err != nil {
 
-						Error(fmt.Sprintf("%s/%s-%s: solver broken for dep %s/%s-%s - %s",
+					Error(fmt.Sprintf("[%9s] %s/%s-%s: solver broken for dep %s/%s-%s - %s",
+						checkType,
 						p.GetCategory(), p.GetName(), p.GetVersion(),
 						r.GetCategory(), r.GetName(), r.GetVersion(),
 						err.Error(),
 					))
 
-						errs <- errors.New(
+					ans = errors.New(
-							fmt.Sprintf("%s/%s-%s: solver broken for Dep %s/%s-%s - %s",
+						fmt.Sprintf("[%9s] %s/%s-%s: solver broken for Dep %s/%s-%s - %s",
+							checkType,
 							p.GetCategory(), p.GetName(), p.GetVersion(),
 							r.GetCategory(), r.GetName(), r.GetVersion(),
 							err.Error()))
 
-						brokenDeps++
+					opts.IncrBrokenDeps()
 					validpkg = false
 				}
 
@@ -226,45 +270,145 @@ func validateWorker(i int,
 	}
 
 	if !validpkg {
-			brokenPkgs++
+		opts.IncrBrokenPkgs()
+	}
+
+	return ans
+}
+
+func validateWorker(i int,
+	wg *sync.WaitGroup,
+	c <-chan pkg.Package,
+	opts *ValidateOpts,
+	errs chan error) {
+
+	defer wg.Done()
+
+	for p := range c {
+
+		if opts.OnlyBuildtime {
+			// Check buildtime compiler/deps
+			err := validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				errs <- err
+			}
+		} else if opts.OnlyRuntime {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				errs <- err
+			}
+
+		} else {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				errs <- err
+				return
+			}
+
+			// Check buildtime compiler/deps
+			err = validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				errs <- err
+			}
+
+		}
+
 	}
 }
+
+func initOpts(opts *ValidateOpts, onlyRuntime, onlyBuildtime, withSolver bool, treePaths []string) {
+	var err error
+
+	opts.OnlyBuildtime = onlyBuildtime
+	opts.OnlyRuntime = onlyRuntime
+	opts.WithSolver = withSolver
+	opts.RuntimeReciper = nil
+	opts.BuildtimeReciper = nil
+	opts.BrokenPkgs = 0
+	opts.BrokenDeps = 0
+
+	if onlyBuildtime {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+	} else if onlyRuntime {
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	} else {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	}
+
+	opts.RuntimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+	opts.BuildtimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+
+	for _, treePath := range treePaths {
+		Info(fmt.Sprintf("Loading :deciduous_tree: %s...", treePath))
+		if opts.BuildtimeReciper != nil {
+			err = opts.BuildtimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
+		}
+		if opts.RuntimeReciper != nil {
+			err = opts.RuntimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
+		}
+	}
+
+	opts.RegExcludes, err = helpers.CreateRegexArray(opts.Excludes)
+	if err != nil {
+		Fatal(err.Error())
+	}
+	opts.RegMatches, err = helpers.CreateRegexArray(opts.Matches)
+	if err != nil {
+		Fatal(err.Error())
+	}
+
 }
 
 func NewTreeValidateCommand() *cobra.Command {
 	var excludes []string
 	var matches []string
 	var treePaths []string
+	var opts ValidateOpts
 
 	var ans = &cobra.Command{
 		Use:   "validate [OPTIONS]",
 		Short: "Validate a tree or a list of packages",
 		Args:  cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
 			if len(treePaths) < 1 {
 				Fatal("Mandatory tree param missing.")
 			}
+			if onlyRuntime && onlyBuildtime {
+				Fatal("Both --only-runtime and --only-buildtime options are not possibile.")
+			}
 		},
 		Run: func(cmd *cobra.Command, args []string) {
+			var reciper tree.Builder
+
 			concurrency := LuetCfg.GetGeneral().Concurrency
 
 			withSolver, _ := cmd.Flags().GetBool("with-solver")
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
 
-			reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
+			opts.Excludes = excludes
-			for _, treePath := range treePaths {
+			opts.Matches = matches
-				err := reciper.Load(treePath)
+			initOpts(&opts, onlyRuntime, onlyBuildtime, withSolver, treePaths)
-				if err != nil {
-					Fatal("Error on load tree ", err)
-				}
-			}
 
-			regExcludes, err := helpers.CreateRegexArray(excludes)
+			// We need at least one valid reciper for get list of the packages.
-			if err != nil {
+			if onlyBuildtime {
-				Fatal(err.Error())
+				reciper = opts.BuildtimeReciper
-			}
+			} else {
-			regMatches, err := helpers.CreateRegexArray(matches)
+				reciper = opts.RuntimeReciper
-			if err != nil {
-				Fatal(err.Error())
 			}
 
 			all := make(chan pkg.Package)
@@ -274,9 +418,7 @@ func NewTreeValidateCommand() *cobra.Command {
 
 			for i := 0; i < concurrency; i++ {
 				wg.Add(1)
-				go validateWorker(i, wg, all,
+				go validateWorker(i, wg, all, &opts, errs)
-					reciper, withSolver, regExcludes, regMatches, excludes, matches,
-					errs)
 			}
 			for _, p := range reciper.GetDatabase().World() {
 				all <- p
@@ -310,6 +452,8 @@ func NewTreeValidateCommand() *cobra.Command {
 		},
 	}
 
+	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
+	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")
 	ans.Flags().BoolP("with-solver", "s", false,
 		"Enable check of requires also with solver.")
 	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{},