From ef1d0e55738db24c9c769045dde4f896ddccdf87 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 1 Nov 2019 16:26:03 +0100 Subject: [PATCH] Make also deps recalculation concurrent --- pkg/package/database_boltdb.go | 16 +++++++++++ pkg/package/database_mem.go | 5 ++++ pkg/package/package.go | 1 + pkg/tree/builder/gentoo/gentoo.go | 16 +++++------ pkg/tree/tree.go | 45 +++++++++++++++++++++---------- 5 files changed, 60 insertions(+), 23 deletions(-) diff --git a/pkg/package/database_boltdb.go b/pkg/package/database_boltdb.go index 20215d92..436f479b 100644 --- a/pkg/package/database_boltdb.go +++ b/pkg/package/database_boltdb.go @@ -123,6 +123,22 @@ func (db *BoltDatabase) GetPackages() []string { return ids } +func (db *BoltDatabase) GetAllPackages(packages chan Package) error { + bolt, err := storm.Open(db.Path) + if err != nil { + return err + } + defer bolt.Close() + // Fetching records one by one (useful when the bucket contains a lot of records) + query := bolt.Select() + + return query.Each(new(DefaultPackage), func(record interface{}) error { + u := record.(*DefaultPackage) + packages <- u + return err + }) +} + // Encode encodes the package to string. // It returns an ID which can be used to retrieve the package later on. func (db *BoltDatabase) CreatePackage(p Package) (string, error) { diff --git a/pkg/package/database_mem.go b/pkg/package/database_mem.go index a7096ba1..049b4bd7 100644 --- a/pkg/package/database_mem.go +++ b/pkg/package/database_mem.go @@ -88,6 +88,11 @@ func (db *InMemoryDatabase) GetPackage(ID string) (Package, error) { return p, nil } +// Not implemented +func (db *InMemoryDatabase) GetAllPackages(packages chan Package) error { + return errors.New("Not implemented") +} + // Encode encodes the package to string. // It returns an ID which can be used to retrieve the package later on. func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) { diff --git a/pkg/package/package.go b/pkg/package/package.go index d9c725e1..1105038b 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -63,6 +63,7 @@ type PackageSet interface { Clean() error FindPackage(Package) (Package, error) UpdatePackage(p Package) error + GetAllPackages(packages chan Package) error } type Tree interface { diff --git a/pkg/tree/builder/gentoo/gentoo.go b/pkg/tree/builder/gentoo/gentoo.go index dabdaba2..c8e70823 100644 --- a/pkg/tree/builder/gentoo/gentoo.go +++ b/pkg/tree/builder/gentoo/gentoo.go @@ -19,16 +19,13 @@ package gentoo // https://gist.github.com/adnaan/6ca68c7985c6f851def3 import ( - "fmt" "io/ioutil" "os" "path/filepath" "strings" "sync" - "time" - - "github.com/briandowns/spinner" + . "github.com/mudler/luet/pkg/logger" tree "github.com/mudler/luet/pkg/tree" pkg "github.com/mudler/luet/pkg/package" @@ -78,9 +75,10 @@ func (gb *GentooBuilder) worker(i int, wg *sync.WaitGroup, s chan string, t pkg. defer wg.Done() for path := range s { + SpinnerText(" "+path, "Scanning ") err := gb.scanEbuild(path, t) if err != nil { - fmt.Println("Error scanning ebuild: " + path) + Error("scanning ebuild: " + path) } } @@ -93,9 +91,8 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) { return nil, err } var toScan = make(chan string) - s := spinner.New(spinner.CharSets[11], 100*time.Millisecond) // Build our new spinner - s.Start() // Start the spinner - defer s.Stop() + Spinner(27) + defer SpinnerStop() tree := &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewBoltDatabase(tmpfile.Name())}} @@ -125,5 +122,6 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) { close(toScan) wg.Wait() - return tree, tree.ResolveDeps() + Info("Resolving deps") + return tree, tree.ResolveDeps(10) } diff --git a/pkg/tree/tree.go b/pkg/tree/tree.go index 337e22e2..34f55b27 100644 --- a/pkg/tree/tree.go +++ b/pkg/tree/tree.go @@ -7,7 +7,9 @@ package tree import ( "errors" - "fmt" + "sync" + + . "github.com/mudler/luet/pkg/logger" pkg "github.com/mudler/luet/pkg/package" ) @@ -62,20 +64,16 @@ func (gt *DefaultTree) FindPackage(pack pkg.Package) (pkg.Package, error) { return nil, errors.New("No package found") } -// Search for deps/conflicts in db and replaces it with packages in the db -func (t *DefaultTree) ResolveDeps() error { - for _, pid := range t.GetPackageSet().GetPackages() { - - p, err := t.GetPackageSet().GetPackage(pid) - if err != nil { - return err - } +func (gb *DefaultTree) depsWorker(i int, wg *sync.WaitGroup, c chan pkg.Package) error { + defer wg.Done() + for p := range c { + SpinnerText(" "+p.GetName(), "Deps ") for _, r := range p.GetRequires() { - foundPackage, err := t.GetPackageSet().FindPackage(r) + foundPackage, err := gb.GetPackageSet().FindPackage(r) if err != nil { - fmt.Println("Warning: Unmatched dependency - no package found in the database for this requirement clause") + Warning("Unmatched dependency - no package found in the database for this requirement clause") continue //return err } @@ -88,7 +86,7 @@ func (t *DefaultTree) ResolveDeps() error { for _, r := range p.GetConflicts() { - foundPackage, err := t.GetPackageSet().FindPackage(r) + foundPackage, err := gb.GetPackageSet().FindPackage(r) if err != nil { continue //return err @@ -100,10 +98,29 @@ func (t *DefaultTree) ResolveDeps() error { r = found } - if err = t.GetPackageSet().UpdatePackage(p); err != nil { + if err := gb.GetPackageSet().UpdatePackage(p); err != nil { return err } - } + + return nil +} + +// Search for deps/conflicts in db and replaces it with packages in the db +func (t *DefaultTree) ResolveDeps(concurrency int) error { + all := make(chan pkg.Package) + + var wg = new(sync.WaitGroup) + for i := 0; i < concurrency; i++ { + wg.Add(1) + go t.depsWorker(i, wg, all) + } + + if err := t.GetPackageSet().GetAllPackages(all); err != nil { + return err + } + + close(all) + wg.Wait() return nil }