Make also deps recalculation concurrent

pkg/package/database_boltdb.go

@@ -123,6 +123,22 @@ func (db *BoltDatabase) GetPackages() []string {
 	return ids
 }
 
+func (db *BoltDatabase) GetAllPackages(packages chan Package) error {
+	bolt, err := storm.Open(db.Path)
+	if err != nil {
+		return err
+	}
+	defer bolt.Close()
+	// Fetching records one by one (useful when the bucket contains a lot of records)
+	query := bolt.Select()
+
+	return query.Each(new(DefaultPackage), func(record interface{}) error {
+		u := record.(*DefaultPackage)
+		packages <- u
+		return err
+	})
+}
+
 // Encode encodes the package to string.
 // It returns an ID which can be used to retrieve the package later on.
 func (db *BoltDatabase) CreatePackage(p Package) (string, error) {

pkg/package/database_mem.go

@@ -88,6 +88,11 @@ func (db *InMemoryDatabase) GetPackage(ID string) (Package, error) {
 	return p, nil
 }
 
+// Not implemented
+func (db *InMemoryDatabase) GetAllPackages(packages chan Package) error {
+	return errors.New("Not implemented")
+}
+
 // Encode encodes the package to string.
 // It returns an ID which can be used to retrieve the package later on.
 func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {

pkg/package/package.go

@@ -63,6 +63,7 @@ type PackageSet interface {
 	Clean() error
 	FindPackage(Package) (Package, error)
 	UpdatePackage(p Package) error
+	GetAllPackages(packages chan Package) error
 }
 
 type Tree interface {

pkg/tree/builder/gentoo/gentoo.go

@@ -19,16 +19,13 @@ package gentoo
 // https://gist.github.com/adnaan/6ca68c7985c6f851def3
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 	"sync"
-	"time"
-
-	"github.com/briandowns/spinner"
 
+	. "github.com/mudler/luet/pkg/logger"
 	tree "github.com/mudler/luet/pkg/tree"
 
 	pkg "github.com/mudler/luet/pkg/package"
@@ -78,9 +75,10 @@ func (gb *GentooBuilder) worker(i int, wg *sync.WaitGroup, s chan string, t pkg.
 	defer wg.Done()
 
 	for path := range s {
+		SpinnerText(" "+path, "Scanning ")
 		err := gb.scanEbuild(path, t)
 		if err != nil {
-			fmt.Println("Error scanning ebuild: " + path)
+			Error("scanning ebuild: " + path)
 		}
 	}
 
@@ -93,9 +91,8 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) {
 		return nil, err
 	}
 	var toScan = make(chan string)
-	s := spinner.New(spinner.CharSets[11], 100*time.Millisecond) // Build our new spinner
-	s.Start()                                                    // Start the spinner
-	defer s.Stop()
+	Spinner(27)
+	defer SpinnerStop()
 
 	tree := &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewBoltDatabase(tmpfile.Name())}}
 
@@ -125,5 +122,6 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) {
 
 	close(toScan)
 	wg.Wait()
-	return tree, tree.ResolveDeps()
+	Info("Resolving deps")
+	return tree, tree.ResolveDeps(10)
 }

pkg/tree/tree.go

@@ -7,7 +7,9 @@ package tree
 
 import (
 	"errors"
-	"fmt"
+	"sync"
+
+	. "github.com/mudler/luet/pkg/logger"
 
 	pkg "github.com/mudler/luet/pkg/package"
 )
@@ -62,20 +64,16 @@ func (gt *DefaultTree) FindPackage(pack pkg.Package) (pkg.Package, error) {
 	return nil, errors.New("No package found")
 }
 
-// Search for deps/conflicts in db and replaces it with packages in the db
-func (t *DefaultTree) ResolveDeps() error {
-	for _, pid := range t.GetPackageSet().GetPackages() {
-
-		p, err := t.GetPackageSet().GetPackage(pid)
-		if err != nil {
-			return err
-		}
+func (gb *DefaultTree) depsWorker(i int, wg *sync.WaitGroup, c chan pkg.Package) error {
+	defer wg.Done()
 
+	for p := range c {
+		SpinnerText(" "+p.GetName(), "Deps ")
 		for _, r := range p.GetRequires() {
 
-			foundPackage, err := t.GetPackageSet().FindPackage(r)
+			foundPackage, err := gb.GetPackageSet().FindPackage(r)
 			if err != nil {
-				fmt.Println("Warning: Unmatched dependency - no package found in the database for this requirement clause")
+				Warning("Unmatched dependency - no package found in the database for this requirement clause")
 				continue
 				//return err
 			}
@@ -88,7 +86,7 @@ func (t *DefaultTree) ResolveDeps() error {
 
 		for _, r := range p.GetConflicts() {
 
-			foundPackage, err := t.GetPackageSet().FindPackage(r)
+			foundPackage, err := gb.GetPackageSet().FindPackage(r)
 			if err != nil {
 				continue
 				//return err
@@ -100,10 +98,29 @@ func (t *DefaultTree) ResolveDeps() error {
 			r = found
 		}
 
-		if err = t.GetPackageSet().UpdatePackage(p); err != nil {
+		if err := gb.GetPackageSet().UpdatePackage(p); err != nil {
 			return err
 		}
-
 	}
+
+	return nil
+}
+
+// Search for deps/conflicts in db and replaces it with packages in the db
+func (t *DefaultTree) ResolveDeps(concurrency int) error {
+	all := make(chan pkg.Package)
+
+	var wg = new(sync.WaitGroup)
+	for i := 0; i < concurrency; i++ {
+		wg.Add(1)
+		go t.depsWorker(i, wg, all)
+	}
+
+	if err := t.GetPackageSet().GetAllPackages(all); err != nil {
+		return err
+	}
+
+	close(all)
+	wg.Wait()
 	return nil
 }