From 8c0ad2b571569a9c4c4dd5d1152783f57a5561a5 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 2 Nov 2019 18:02:55 +0100 Subject: [PATCH] Use InMemoryDB instead of Bolt in gentoo parser Also refactor things a bit --- pkg/tree/builder/gentoo/gentoo.go | 39 ++++++++++--------- pkg/tree/builder/gentoo/simpleparser.go | 8 ++-- pkg/tree/recipes.go | 2 +- pkg/tree/tree.go | 50 +++++++++++++------------ 4 files changed, 53 insertions(+), 46 deletions(-) diff --git a/pkg/tree/builder/gentoo/gentoo.go b/pkg/tree/builder/gentoo/gentoo.go index f206946a..fa3c8cd3 100644 --- a/pkg/tree/builder/gentoo/gentoo.go +++ b/pkg/tree/builder/gentoo/gentoo.go @@ -19,9 +19,9 @@ package gentoo // https://gist.github.com/adnaan/6ca68c7985c6f851def3 import ( - "io/ioutil" "os" "path/filepath" + "strconv" "strings" "sync" @@ -53,7 +53,11 @@ func (gt *GentooTree) Prelude() string { } func (gb *GentooBuilder) scanEbuild(path string, t pkg.Tree) error { - + defer func() { + if r := recover(); r != nil { + Error(r) + } + }() pkgs, err := gb.EbuildParser.ScanEbuild(path, t) if err != nil { return err @@ -70,31 +74,31 @@ func (gb *GentooBuilder) scanEbuild(path string, t pkg.Tree) error { return nil } -func (gb *GentooBuilder) worker(i int, wg *sync.WaitGroup, s chan string, t pkg.Tree) error { +func (gb *GentooBuilder) worker(i int, wg *sync.WaitGroup, s <-chan string, t pkg.Tree) { defer wg.Done() for path := range s { - Info("Scanning", path) + Info("#"+strconv.Itoa(i), "parsing", path) err := gb.scanEbuild(path, t) if err != nil { - Error("scanning ebuild: " + path) + Error(path, ":", err.Error()) } } - return nil } func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) { - tmpfile, err := ioutil.TempFile("", "boltdb") - if err != nil { - return nil, err - } + // tmpfile, err := ioutil.TempFile("", "boltdb") + // if err != nil { + // return nil, err + // } var toScan = make(chan string) Spinner(27) defer SpinnerStop() + tree := &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewInMemoryDatabase(false)}} - tree := &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewBoltDatabase(tmpfile.Name())}} - + //tree := &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewBoltDatabase(tmpfile.Name())}} + Debug("Concurrency", gb.Concurrency) // the waitgroup will allow us to wait for all the goroutines to finish at the end var wg = new(sync.WaitGroup) for i := 0; i < gb.Concurrency; i++ { @@ -103,7 +107,7 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) { } // TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean() - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -111,17 +115,18 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) { return nil } if strings.Contains(info.Name(), "ebuild") { + Debug("Enqueueing", path) toScan <- path } return nil }) - if err != nil { - return tree, err - } close(toScan) Debug("Waiting for goroutines to finish") - wg.Wait() + wg.Wait() // FIXME: With BoltDB as backend goroutines timeouts and deadlocks + if err != nil { + return tree, err + } Info("Scan finished") Info("Resolving deps") return tree, tree.ResolveDeps(gb.Concurrency) diff --git a/pkg/tree/builder/gentoo/simpleparser.go b/pkg/tree/builder/gentoo/simpleparser.go index b531f7b1..717ba8fb 100644 --- a/pkg/tree/builder/gentoo/simpleparser.go +++ b/pkg/tree/builder/gentoo/simpleparser.go @@ -55,11 +55,8 @@ func SourceFile(ctx context.Context, path string) (map[string]expand.Variable, e // ScanEbuild returns a list of packages (always one with SimpleEbuildParser) decoded from an ebuild. func (ep *SimpleEbuildParser) ScanEbuild(path string, tree pkg.Tree) ([]pkg.Package, error) { - defer func() { - if r := recover(); r != nil { - Error(r) - } - }() + Debug("Starting parsing of ebuild", path) + file := filepath.Base(path) file = strings.Replace(file, ".ebuild", "", -1) @@ -124,6 +121,7 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string, tree pkg.Tree) ([]pkg.Pack } } + Debug("Finished processing ebuild", path) //TODO: Deps and conflicts return []pkg.Package{pack}, nil diff --git a/pkg/tree/recipes.go b/pkg/tree/recipes.go index 754425b8..6b9d60bf 100644 --- a/pkg/tree/recipes.go +++ b/pkg/tree/recipes.go @@ -51,7 +51,7 @@ func (r *Recipe) Load(path string) error { r.PackageTree = NewDefaultTree() } - tmpfile, err := ioutil.TempFile("", "boltdb") + tmpfile, err := ioutil.TempFile("", "luet") if err != nil { return err } diff --git a/pkg/tree/tree.go b/pkg/tree/tree.go index 34f55b27..e7cbcecb 100644 --- a/pkg/tree/tree.go +++ b/pkg/tree/tree.go @@ -64,7 +64,7 @@ func (gt *DefaultTree) FindPackage(pack pkg.Package) (pkg.Package, error) { return nil, errors.New("No package found") } -func (gb *DefaultTree) depsWorker(i int, wg *sync.WaitGroup, c chan pkg.Package) error { +func (gb *DefaultTree) depsWorker(i int, wg *sync.WaitGroup, c <-chan pkg.Package) error { defer wg.Done() for p := range c { @@ -72,35 +72,42 @@ func (gb *DefaultTree) depsWorker(i int, wg *sync.WaitGroup, c chan pkg.Package) for _, r := range p.GetRequires() { foundPackage, err := gb.GetPackageSet().FindPackage(r) - if err != nil { - Warning("Unmatched dependency - no package found in the database for this requirement clause") - continue - //return err + if err == nil { + + found, ok := foundPackage.(*pkg.DefaultPackage) + if !ok { + panic("Simpleparser should deal only with DefaultPackages") + } + r = found + } else { + Warning("Unmatched require for", r.GetName()) } - found, ok := foundPackage.(*pkg.DefaultPackage) - if !ok { - panic("Simpleparser should deal only with DefaultPackages") - } - r = found } + Debug("Walking conflicts for", p.GetName()) for _, r := range p.GetConflicts() { + Debug("conflict", r.GetName()) foundPackage, err := gb.GetPackageSet().FindPackage(r) - if err != nil { - continue - //return err + if err == nil { + + found, ok := foundPackage.(*pkg.DefaultPackage) + if !ok { + panic("Simpleparser should deal only with DefaultPackages") + } + r = found + } else { + Warning("Unmatched conflict for", r.GetName()) + } - found, ok := foundPackage.(*pkg.DefaultPackage) - if !ok { - panic("Simpleparser should deal only with DefaultPackages") - } - r = found } + Debug("Finished processing", p.GetName()) if err := gb.GetPackageSet().UpdatePackage(p); err != nil { return err } + Debug("Update done", p.GetName()) + } return nil @@ -116,11 +123,8 @@ func (t *DefaultTree) ResolveDeps(concurrency int) error { go t.depsWorker(i, wg, all) } - if err := t.GetPackageSet().GetAllPackages(all); err != nil { - return err - } - + err := t.GetPackageSet().GetAllPackages(all) close(all) wg.Wait() - return nil + return err }