From f3623cbb383c5e5bd8c92b49e585246145378dd5 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 3 Nov 2019 13:37:45 +0100 Subject: [PATCH] Allow to switch between database types in Gentoo Parser Define types and allows to switch between boltdb and inmemory --- cmd/convert.go | 4 +- pkg/tree/builder/gentoo/gentoo.go | 45 ++++++++++++++------ pkg/tree/builder/gentoo/gentoo_test.go | 2 +- pkg/tree/builder/gentoo/simpleparser_test.go | 2 +- pkg/tree/recipes_test.go | 6 +-- 5 files changed, 38 insertions(+), 21 deletions(-) diff --git a/cmd/convert.go b/cmd/convert.go index d48fae91..f382fc8c 100644 --- a/cmd/convert.go +++ b/cmd/convert.go @@ -47,9 +47,9 @@ var convertCmd = &cobra.Command{ var builder tree.Parser switch t { case "gentoo": - builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c) + builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c, gentoo.InMemory) default: // dup - builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c) + builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c, gentoo.InMemory) } packageTree, err := builder.Generate(input) diff --git a/pkg/tree/builder/gentoo/gentoo.go b/pkg/tree/builder/gentoo/gentoo.go index fa3c8cd3..613bcf69 100644 --- a/pkg/tree/builder/gentoo/gentoo.go +++ b/pkg/tree/builder/gentoo/gentoo.go @@ -19,6 +19,7 @@ package gentoo // https://gist.github.com/adnaan/6ca68c7985c6f851def3 import ( + "io/ioutil" "os" "path/filepath" "strconv" @@ -31,13 +32,21 @@ import ( pkg "github.com/mudler/luet/pkg/package" ) -func NewGentooBuilder(e EbuildParser, concurrency int) tree.Parser { +type MemoryDB int + +const ( + InMemory MemoryDB = iota + BoltDB MemoryDB = iota +) + +func NewGentooBuilder(e EbuildParser, concurrency int, db MemoryDB) tree.Parser { return &GentooBuilder{EbuildParser: e, Concurrency: concurrency} } type GentooBuilder struct { EbuildParser EbuildParser Concurrency int + DBType MemoryDB } type GentooTree struct { @@ -88,22 +97,32 @@ func (gb *GentooBuilder) worker(i int, wg *sync.WaitGroup, s <-chan string, t pk } func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) { - // tmpfile, err := ioutil.TempFile("", "boltdb") - // if err != nil { - // return nil, err - // } + var toScan = make(chan string) Spinner(27) defer SpinnerStop() - tree := &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewInMemoryDatabase(false)}} + var gtree *GentooTree + + // Support for + switch gb.DBType { + case InMemory: + gtree = &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewInMemoryDatabase(false)}} + case BoltDB: + tmpfile, err := ioutil.TempFile("", "boltdb") + if err != nil { + return nil, err + } + gtree = &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewBoltDatabase(tmpfile.Name())}} + default: + gtree = &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewInMemoryDatabase(false)}} + } - //tree := &GentooTree{DefaultTree: &tree.DefaultTree{Packages: pkg.NewBoltDatabase(tmpfile.Name())}} Debug("Concurrency", gb.Concurrency) // the waitgroup will allow us to wait for all the goroutines to finish at the end var wg = new(sync.WaitGroup) for i := 0; i < gb.Concurrency; i++ { wg.Add(1) - go gb.worker(i, wg, toScan, tree) + go gb.worker(i, wg, toScan, gtree) } // TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean() @@ -115,19 +134,17 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) { return nil } if strings.Contains(info.Name(), "ebuild") { - Debug("Enqueueing", path) toScan <- path } return nil }) close(toScan) - Debug("Waiting for goroutines to finish") - wg.Wait() // FIXME: With BoltDB as backend goroutines timeouts and deadlocks + wg.Wait() if err != nil { - return tree, err + return gtree, err } - Info("Scan finished") + Info("Resolving deps") - return tree, tree.ResolveDeps(gb.Concurrency) + return gtree, gtree.ResolveDeps(gb.Concurrency) } diff --git a/pkg/tree/builder/gentoo/gentoo_test.go b/pkg/tree/builder/gentoo/gentoo_test.go index 447695f0..d836f6a6 100644 --- a/pkg/tree/builder/gentoo/gentoo_test.go +++ b/pkg/tree/builder/gentoo/gentoo_test.go @@ -34,7 +34,7 @@ var _ = Describe("GentooBuilder", func() { Context("Simple test", func() { It("parses correctly deps", func() { - gb := NewGentooBuilder(&FakeParser{}, 20) + gb := NewGentooBuilder(&FakeParser{}, 20, InMemory) tree, err := gb.Generate("../../../../tests/fixtures/overlay") Expect(err).ToNot(HaveOccurred()) Expect(len(tree.GetPackageSet().GetPackages())).To(Equal(10)) diff --git a/pkg/tree/builder/gentoo/simpleparser_test.go b/pkg/tree/builder/gentoo/simpleparser_test.go index 39493295..dd6bae66 100644 --- a/pkg/tree/builder/gentoo/simpleparser_test.go +++ b/pkg/tree/builder/gentoo/simpleparser_test.go @@ -26,7 +26,7 @@ var _ = Describe("GentooBuilder", func() { Context("Simple test", func() { It("parses correctly deps", func() { - gb := NewGentooBuilder(&SimpleEbuildParser{}, 20) + gb := NewGentooBuilder(&SimpleEbuildParser{}, 20, InMemory) tree, err := gb.Generate("../../../../tests/fixtures/overlay") Expect(err).ToNot(HaveOccurred()) defer func() { diff --git a/pkg/tree/recipes_test.go b/pkg/tree/recipes_test.go index 55272ce3..c9aa8927 100644 --- a/pkg/tree/recipes_test.go +++ b/pkg/tree/recipes_test.go @@ -29,7 +29,7 @@ var _ = Describe("Recipe", func() { Expect(err).ToNot(HaveOccurred()) defer os.RemoveAll(tmpdir) // clean up - gb := gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, 20) + gb := gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, 20, gentoo.InMemory) tree, err := gb.Generate("../../tests/fixtures/overlay") Expect(err).ToNot(HaveOccurred()) defer func() { @@ -51,7 +51,7 @@ var _ = Describe("Recipe", func() { Expect(err).ToNot(HaveOccurred()) defer os.RemoveAll(tmpdir) // clean up - gb := gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, 20) + gb := gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, 20, gentoo.InMemory) tree, err := gb.Generate("../../tests/fixtures/overlay") Expect(err).ToNot(HaveOccurred()) defer func() { @@ -88,7 +88,7 @@ var _ = Describe("Recipe", func() { Expect(err).ToNot(HaveOccurred()) defer os.RemoveAll(tmpdir) // clean up - gb := gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, 20) + gb := gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, 20, gentoo.InMemory) tree, err := gb.Generate("../../tests/fixtures/overlay") Expect(err).ToNot(HaveOccurred()) defer func() {