kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-07-16 16:31:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use InMemoryDB instead of Bolt in gentoo parser

Browse Source 
Also refactor things a bit
This commit is contained in:
Ettore Di Giacinto 2019-11-02 18:02:55 +01:00
parent 9ec316312f
commit 8c0ad2b571
No known key found for this signature in database
GPG Key ID: 1ADA699B145A2D1C
4 changed files with 53 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
pkg/tree/builder/gentoo/gentoo.go
View File

@ -19,9 +19,9 @@ package gentoo
// https://gist.github.com/adnaan/6ca68c7985c6f851def3
import (
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
@ -53,7 +53,11 @@ func (gt *GentooTree) Prelude() string {
}
func (gb *GentooBuilder) scanEbuild(path string, t pkg.Tree) error {
defer func() {
if r := recover(); r != nil {
Error(r)
}
}()
pkgs, err := gb.EbuildParser.ScanEbuild(path, t)
if err != nil {
return err
@ -70,31 +74,31 @@ func (gb *GentooBuilder) scanEbuild(path string, t pkg.Tree) error {
return nil
}
func (gb *GentooBuilder) worker(i int, wg *sync.WaitGroup, s chan string, t pkg.Tree) error {
func (gb *GentooBuilder) worker(i int, wg *sync.WaitGroup, s <-chan string, t pkg.Tree) {
defer wg.Done()
for path := range s {
Info("Scanning", path)
Info("#"+strconv.Itoa(i), "parsing", path)
err := gb.scanEbuild(path, t)
if err != nil {
Error("scanning ebuild: " + path)
Error(path, ":", err.Error())
}
}
return nil
}
func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) {
tmpfile, err := ioutil.TempFile("", "boltdb")
if err != nil {
return nil, err
}
// tmpfile, err := ioutil.TempFile("", "boltdb")
// if err != nil {
// return nil, err
// }
var toScan = make(chan string)
Spinner(27)
defer SpinnerStop()
tree := &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewInMemoryDatabase(false)}}
tree := &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewBoltDatabase(tmpfile.Name())}}
//tree := &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewBoltDatabase(tmpfile.Name())}}
Debug("Concurrency", gb.Concurrency)
// the waitgroup will allow us to wait for all the goroutines to finish at the end
var wg = new(sync.WaitGroup)
for i := 0; i < gb.Concurrency; i++ {
@ -103,7 +107,7 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) {
}
// TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean()
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
@ -111,17 +115,18 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) {
return nil
}
if strings.Contains(info.Name(), "ebuild") {
Debug("Enqueueing", path)
toScan <- path
}
return nil
})
if err != nil {
return tree, err
}
close(toScan)
Debug("Waiting for goroutines to finish")
wg.Wait()
wg.Wait() // FIXME: With BoltDB as backend goroutines timeouts and deadlocks
if err != nil {
return tree, err
}
Info("Scan finished")
Info("Resolving deps")
return tree, tree.ResolveDeps(gb.Concurrency)

8
pkg/tree/builder/gentoo/simpleparser.go
View File

@ -55,11 +55,8 @@ func SourceFile(ctx context.Context, path string) (map[string]expand.Variable, e
// ScanEbuild returns a list of packages (always one with SimpleEbuildParser) decoded from an ebuild.
func (ep *SimpleEbuildParser) ScanEbuild(path string, tree pkg.Tree) ([]pkg.Package, error) {
defer func() {
if r := recover(); r != nil {
Error(r)
}
}()
Debug("Starting parsing of ebuild", path)
file := filepath.Base(path)
file = strings.Replace(file, ".ebuild", "", -1)
@ -124,6 +121,7 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string, tree pkg.Tree) ([]pkg.Pack
}
}
Debug("Finished processing ebuild", path)
//TODO: Deps and conflicts
return []pkg.Package{pack}, nil

2
pkg/tree/recipes.go
View File

@ -51,7 +51,7 @@ func (r *Recipe) Load(path string) error {
r.PackageTree = NewDefaultTree()
}
tmpfile, err := ioutil.TempFile("", "boltdb")
tmpfile, err := ioutil.TempFile("", "luet")
if err != nil {
return err
}

50
pkg/tree/tree.go
View File

@ -64,7 +64,7 @@ func (gt *DefaultTree) FindPackage(pack pkg.Package) (pkg.Package, error) {
return nil, errors.New("No package found")
}
func (gb *DefaultTree) depsWorker(i int, wg *sync.WaitGroup, c chan pkg.Package) error {
func (gb *DefaultTree) depsWorker(i int, wg *sync.WaitGroup, c <-chan pkg.Package) error {
defer wg.Done()
for p := range c {
@ -72,35 +72,42 @@ func (gb *DefaultTree) depsWorker(i int, wg *sync.WaitGroup, c chan pkg.Package)
for _, r := range p.GetRequires() {
foundPackage, err := gb.GetPackageSet().FindPackage(r)
if err != nil {
Warning("Unmatched dependency - no package found in the database for this requirement clause")
continue
//return err
if err == nil {
found, ok := foundPackage.(*pkg.DefaultPackage)
if !ok {
panic("Simpleparser should deal only with DefaultPackages")
}
r = found
} else {
Warning("Unmatched require for", r.GetName())
}
found, ok := foundPackage.(*pkg.DefaultPackage)
if !ok {
panic("Simpleparser should deal only with DefaultPackages")
}
r = found
}
Debug("Walking conflicts for", p.GetName())
for _, r := range p.GetConflicts() {
Debug("conflict", r.GetName())
foundPackage, err := gb.GetPackageSet().FindPackage(r)
if err != nil {
continue
//return err
if err == nil {
found, ok := foundPackage.(*pkg.DefaultPackage)
if !ok {
panic("Simpleparser should deal only with DefaultPackages")
}
r = found
} else {
Warning("Unmatched conflict for", r.GetName())
}
found, ok := foundPackage.(*pkg.DefaultPackage)
if !ok {
panic("Simpleparser should deal only with DefaultPackages")
}
r = found
}
Debug("Finished processing", p.GetName())
if err := gb.GetPackageSet().UpdatePackage(p); err != nil {
return err
}
Debug("Update done", p.GetName())
}
return nil
@ -116,11 +123,8 @@ func (t *DefaultTree) ResolveDeps(concurrency int) error {
go t.depsWorker(i, wg, all)
}
if err := t.GetPackageSet().GetAllPackages(all); err != nil {
return err
}
err := t.GetPackageSet().GetAllPackages(all)
close(all)
wg.Wait()
return nil
return err
}