Add boltdb implementation for packageset and db

Makefile

@@ -6,12 +6,15 @@ VERSION := $(shell git describe --tags || echo dev)
 VERSION := $(shell echo $(VERSION) | sed -e 's/^v//g')
 ITTERATION := $(shell date +%s)
 BUILD_PLATFORMS ?= -osarch="linux/amd64" -osarch="linux/386" -osarch="linux/arm"
+ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
 .PHONY: all
 all: deps build
 
 .PHONY: test
 test:
+	go get github.com/onsi/ginkgo/ginkgo
+	go get github.com/onsi/gomega/...
 	ginkgo -r ./...
 
 .PHONY: coverage
@@ -54,3 +57,8 @@ lint:
 	# Checking project code style...
 	golint ./... | grep -v "be unexported"
 
+.PHONY: test-docker
+test-docker:
+	docker run -v $(ROOT_DIR):/go/src/github.com/mudler/luet \
+				--workdir /go/src/github.com/mudler/luet -ti golang:latest \
+				bash -c "make test"

pkg/package/database.go

@@ -18,12 +18,14 @@ package pkg
 // Database is a merely simple in-memory db.
 // FIXME: Use a proper structure or delegate to third-party
 type PackageDatabase interface {
+	PackageSet
+
 	Get(s string) (string, error)
 	Set(k, v string) error
 
 	Create([]byte) (string, error)
 	Retrieve(ID string) ([]byte, error)
 
-	GetPackage(ID string) (Package, error)
-	CreatePackage(p Package) (string, error)
+	FindPackage(name, version string) (Package, error)
+	UpdatePackage(p Package) error
 }

pkg/package/database_boltdb.go

@@ -0,0 +1,144 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+import (
+	"errors"
+	"strconv"
+
+	storm "github.com/asdine/storm"
+	"github.com/asdine/storm/q"
+)
+
+//var BoltInstance PackageDatabase
+
+type BoltDatabase struct {
+	Path string
+}
+
+func NewBoltDatabase(path string) PackageDatabase {
+	// if BoltInstance == nil {
+	// 	BoltInstance = &BoltDatabase{Path: path}
+	// }
+	//return BoltInstance, nil
+	return &BoltDatabase{Path: path}
+}
+
+func (db *BoltDatabase) Get(s string) (string, error) {
+	return "", errors.New("Not implemented")
+}
+
+func (db *BoltDatabase) Set(k, v string) error {
+	return errors.New("Not implemented")
+
+}
+
+func (db *BoltDatabase) Create(v []byte) (string, error) {
+	return "", errors.New("Not implemented")
+}
+
+func (db *BoltDatabase) Retrieve(ID string) ([]byte, error) {
+	return []byte{}, errors.New("Not implemented")
+}
+
+func (db *BoltDatabase) FindPackage(name, version string) (Package, error) {
+	p := &DefaultPackage{}
+	bolt, err := storm.Open(db.Path)
+	if err != nil {
+		return nil, err
+	}
+	defer bolt.Close()
+
+	err = bolt.Select(q.Eq("Name", name), q.Eq("Version", version)).Limit(1).First(p)
+	if err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+func (db *BoltDatabase) UpdatePackage(p Package) error {
+
+	bolt, err := storm.Open(db.Path)
+	if err != nil {
+		return err
+	}
+	defer bolt.Close()
+
+	dp, ok := p.(*DefaultPackage)
+	if !ok {
+		return errors.New("Bolt DB support only DefaultPackage type for now")
+	}
+	err = bolt.Update(dp)
+	if err != nil {
+		return err
+	}
+
+	return err
+}
+
+func (db *BoltDatabase) GetPackage(ID string) (Package, error) {
+	p := &DefaultPackage{}
+	bolt, err := storm.Open(db.Path)
+	if err != nil {
+		return nil, err
+	}
+	defer bolt.Close()
+	iid, err := strconv.Atoi(ID)
+	if err != nil {
+		return nil, err
+	}
+	err = bolt.One("ID", iid, p)
+	return p, err
+}
+
+func (db *BoltDatabase) GetPackages() []string {
+	ids := []string{}
+	bolt, err := storm.Open(db.Path)
+	if err != nil {
+		return []string{}
+	}
+	defer bolt.Close()
+	// Fetching records one by one (useful when the bucket contains a lot of records)
+	query := bolt.Select()
+
+	query.Each(new(DefaultPackage), func(record interface{}) error {
+		u := record.(*DefaultPackage)
+		ids = append(ids, strconv.Itoa(u.ID))
+		return nil
+	})
+	return ids
+}
+
+// Encode encodes the package to string.
+// It returns an ID which can be used to retrieve the package later on.
+func (db *BoltDatabase) CreatePackage(p Package) (string, error) {
+	bolt, err := storm.Open(db.Path)
+	if err != nil {
+		return "", err
+	}
+	defer bolt.Close()
+
+	dp, ok := p.(*DefaultPackage)
+	if !ok {
+		return "", errors.New("Bolt DB support only DefaultPackage type for now")
+	}
+	err = bolt.Save(dp)
+	if err != nil {
+		return "", err
+	}
+
+	return strconv.Itoa(dp.ID), err
+}

pkg/package/database_mem.go

@@ -108,3 +108,15 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 	}
 	return ID, nil
 }
+
+func (db *InMemoryDatabase) FindPackage(name, version string) (Package, error) {
+	return nil, errors.New("Not implemented")
+}
+
+func (db *InMemoryDatabase) UpdatePackage(p Package) error {
+	return errors.New("Not implemented")
+}
+
+func (db *InMemoryDatabase) GetPackages() []string {
+	return []string{}
+}

pkg/package/package.go

@@ -53,30 +53,19 @@ type Package interface {
 }
 
 type PackageSet interface {
-	GetPackages() []Package
-	AddPackages(pkgs []Package)
-}
-
-type DefaultPackages struct {
-	Packages []Package
+	GetPackages() []string //Ids
+	CreatePackage(pkg Package) (string, error)
+	GetPackage(ID string) (Package, error)
 }
 
 type Tree interface {
 	GetPackageSet() PackageSet
+	Prelude() string // A tree might have a prelude to be able to consume a tree
 }
 
-func (pkgs *DefaultPackages) GetPackages() []Package {
-	return pkgs.Packages
-}
-
-func (p *DefaultPackages) AddPackages(pkgs []Package) {
-	p.Packages = append(p.Packages, pkgs...)
-}
-
-func NewPackages(p []Package) PackageSet { return &DefaultPackages{Packages: p} }
-
 // DefaultPackage represent a standard package definition
 type DefaultPackage struct {
+	ID               int `storm:"id,increment"` // primary key with auto increment
 	Name             string
 	Version          string
 	Category         string

pkg/package/package_test.go

@@ -94,8 +94,8 @@ var _ = Describe("Package", func() {
 			f, err := a1.BuildFormula()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(f)).To(Equal(2))
-			Expect(f[0].String()).To(Equal("or(not(d698a46c), 0dc06f33)"))
-			Expect(f[1].String()).To(Equal("or(not(d698a46c), not(2e57efc4))"))
+			Expect(f[0].String()).To(Equal("or(not(fd21d375), c85f382e)"))
+			Expect(f[1].String()).To(Equal("or(not(fd21d375), not(f512c160))"))
 		})
 	})
 

pkg/tree/builder.go

@@ -21,8 +21,7 @@ import (
 
 // reads a luet tree and generates the package lists
 type Builder interface {
-	Scan(string) error // compiles a tree
-	Load(string) error // pre-scanned tree
-
-	Generate() (pkg.PackageSet, error) // generates world
+	Save(string) error // A tree might be saved to a folder structure (human editable)
+	Load(string) error // A tree might be loaded from a db (e.g. bolt) and written to folder
+	Tree() pkg.Tree    // generates world
 }

pkg/tree/builder/gentoo/gentoo.go

@@ -19,6 +19,7 @@ package gentoo
 // https://gist.github.com/adnaan/6ca68c7985c6f851def3
 
 import (
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -34,7 +35,10 @@ func NewGentooBuilder(e EbuildParser) tree.Parser {
 
 type GentooBuilder struct{ EbuildParser EbuildParser }
 
-type GentooTree struct{ Packages pkg.PackageSet }
+type GentooTree struct {
+	Packages pkg.PackageSet
+	DBPath   string
+}
 
 type EbuildParser interface {
 	ScanEbuild(path string) ([]pkg.Package, error)
@@ -44,10 +48,21 @@ func (gt *GentooTree) GetPackageSet() pkg.PackageSet {
 	return gt.Packages
 }
 
-func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) {
-	tree := &GentooTree{Packages: pkg.NewPackages([]pkg.Package{})}
+func (gt *GentooTree) Prelude() string {
+	return "/usr/portage/"
+}
 
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) {
+	tmpfile, err := ioutil.TempFile("", "boltdb")
+	if err != nil {
+		return nil, err
+	}
+
+	//defer os.Remove(tmpfile.Name()) // clean up
+
+	tree := &GentooTree{Packages: pkg.NewBoltDatabase(tmpfile.Name()), DBPath: tmpfile.Name()}
+
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -59,8 +74,12 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) {
 			if err != nil {
 				return err
 			}
-			tree.Packages.AddPackages(pkgs)
-
+			for _, p := range pkgs {
+				_, err := tree.GetPackageSet().CreatePackage(p)
+				if err != nil {
+					return err
+				}
+			}
 		}
 		return nil
 	})

pkg/tree/builder/gentoo/simpleparser.go

@@ -19,16 +19,36 @@ package gentoo
 // https://gist.github.com/adnaan/6ca68c7985c6f851def3
 
 import (
+	"context"
 	"errors"
+	"fmt"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 
 	pkg "github.com/mudler/luet/pkg/package"
+	"mvdan.cc/sh/expand"
+	"mvdan.cc/sh/shell"
+	"mvdan.cc/sh/syntax"
 )
 
 // SimpleEbuildParser ignores USE flags and generates just 1-1 package
 type SimpleEbuildParser struct {
+	World pkg.PackageDatabase
+}
+
+func SourceFile(ctx context.Context, path string) (map[string]expand.Variable, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("could not open: %v", err)
+	}
+	defer f.Close()
+	file, err := syntax.NewParser(syntax.StopAt("src")).Parse(f, path)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse: %v", err)
+	}
+	return shell.SourceNode(ctx, file)
 }
 
 // ScanEbuild returns a list of packages (always one with SimpleEbuildParser) decoded from an ebuild.
@@ -46,6 +66,27 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) ([]pkg.Package, error) {
 	if len(packageInfo) != 1 || len(packageInfo[0]) != 12 {
 		return []pkg.Package{}, errors.New("Failed decoding ebuild: " + path)
 	}
-	//TODO: Deps and conflicts
-	return []pkg.Package{&pkg.DefaultPackage{Name: packageInfo[0][2], Version: packageInfo[0][7]}}, nil
+
+	vars, err := SourceFile(context.TODO(), path)
+	if err != nil {
+		//	return []pkg.Package{}, err
+	}
+	//fmt.Println("Scanning", path)
+	//fmt.Println(vars)
+	pack := &pkg.DefaultPackage{Name: packageInfo[0][2], Version: packageInfo[0][7]}
+	rdepend, ok := vars["RDEPEND"]
+	if ok {
+		rdepends := strings.Split(rdepend.String(), "\n")
+
+		pack.PackageRequires = []*pkg.DefaultPackage{}
+		for _, rr := range rdepends {
+
+			//TODO: Resolve to db or create a new one.
+			pack.PackageRequires = append(pack.PackageRequires, &pkg.DefaultPackage{Name: rr})
+		}
+
+	}
+
+	//TODO: Deps and conflicts
+	return []pkg.Package{pack}, nil
 }

pkg/tree/builder/gentoo/simpleparser_test.go

@@ -31,10 +31,11 @@ var _ = Describe("GentooBuilder", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(tree.GetPackageSet().GetPackages())).To(Equal(10))
 
-			for _, p := range tree.GetPackageSet().GetPackages() {
+			for _, pid := range tree.GetPackageSet().GetPackages() {
+				p, err := tree.GetPackageSet().GetPackage(pid)
+				Expect(err).ToNot(HaveOccurred())
 				Expect(p.GetName()).To(ContainSubstring("pinentry"))
 				Expect(p.GetVersion()).To(ContainSubstring("1."))
-
 			}
 
 		})