diff --git a/Makefile b/Makefile index 307b8ed8..775641c6 100644 --- a/Makefile +++ b/Makefile @@ -6,12 +6,15 @@ VERSION := $(shell git describe --tags || echo dev) VERSION := $(shell echo $(VERSION) | sed -e 's/^v//g') ITTERATION := $(shell date +%s) BUILD_PLATFORMS ?= -osarch="linux/amd64" -osarch="linux/386" -osarch="linux/arm" +ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) .PHONY: all all: deps build .PHONY: test test: + go get github.com/onsi/ginkgo/ginkgo + go get github.com/onsi/gomega/... ginkgo -r ./... .PHONY: coverage @@ -54,3 +57,8 @@ lint: # Checking project code style... golint ./... | grep -v "be unexported" +.PHONY: test-docker +test-docker: + docker run -v $(ROOT_DIR):/go/src/github.com/mudler/luet \ + --workdir /go/src/github.com/mudler/luet -ti golang:latest \ + bash -c "make test" diff --git a/pkg/package/database.go b/pkg/package/database.go index bd046bd5..58c3d983 100644 --- a/pkg/package/database.go +++ b/pkg/package/database.go @@ -18,12 +18,14 @@ package pkg // Database is a merely simple in-memory db. // FIXME: Use a proper structure or delegate to third-party type PackageDatabase interface { + PackageSet + Get(s string) (string, error) Set(k, v string) error Create([]byte) (string, error) Retrieve(ID string) ([]byte, error) - GetPackage(ID string) (Package, error) - CreatePackage(p Package) (string, error) + FindPackage(name, version string) (Package, error) + UpdatePackage(p Package) error } diff --git a/pkg/package/database_boltdb.go b/pkg/package/database_boltdb.go new file mode 100644 index 00000000..040e2f42 --- /dev/null +++ b/pkg/package/database_boltdb.go @@ -0,0 +1,144 @@ +// Copyright © 2019 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package pkg + +import ( + "errors" + "strconv" + + storm "github.com/asdine/storm" + "github.com/asdine/storm/q" +) + +//var BoltInstance PackageDatabase + +type BoltDatabase struct { + Path string +} + +func NewBoltDatabase(path string) PackageDatabase { + // if BoltInstance == nil { + // BoltInstance = &BoltDatabase{Path: path} + // } + //return BoltInstance, nil + return &BoltDatabase{Path: path} +} + +func (db *BoltDatabase) Get(s string) (string, error) { + return "", errors.New("Not implemented") +} + +func (db *BoltDatabase) Set(k, v string) error { + return errors.New("Not implemented") + +} + +func (db *BoltDatabase) Create(v []byte) (string, error) { + return "", errors.New("Not implemented") +} + +func (db *BoltDatabase) Retrieve(ID string) ([]byte, error) { + return []byte{}, errors.New("Not implemented") +} + +func (db *BoltDatabase) FindPackage(name, version string) (Package, error) { + p := &DefaultPackage{} + bolt, err := storm.Open(db.Path) + if err != nil { + return nil, err + } + defer bolt.Close() + + err = bolt.Select(q.Eq("Name", name), q.Eq("Version", version)).Limit(1).First(p) + if err != nil { + return nil, err + } + return p, nil +} + +func (db *BoltDatabase) UpdatePackage(p Package) error { + + bolt, err := storm.Open(db.Path) + if err != nil { + return err + } + defer bolt.Close() + + dp, ok := p.(*DefaultPackage) + if !ok { + return errors.New("Bolt DB support only DefaultPackage type for now") + } + err = bolt.Update(dp) + if err != nil { + return err + } + + return err +} + +func (db *BoltDatabase) GetPackage(ID string) (Package, error) { + p := &DefaultPackage{} + bolt, err := storm.Open(db.Path) + if err != nil { + return nil, err + } + defer bolt.Close() + iid, err := strconv.Atoi(ID) + if err != nil { + return nil, err + } + err = bolt.One("ID", iid, p) + return p, err +} + +func (db *BoltDatabase) GetPackages() []string { + ids := []string{} + bolt, err := storm.Open(db.Path) + if err != nil { + return []string{} + } + defer bolt.Close() + // Fetching records one by one (useful when the bucket contains a lot of records) + query := bolt.Select() + + query.Each(new(DefaultPackage), func(record interface{}) error { + u := record.(*DefaultPackage) + ids = append(ids, strconv.Itoa(u.ID)) + return nil + }) + return ids +} + +// Encode encodes the package to string. +// It returns an ID which can be used to retrieve the package later on. +func (db *BoltDatabase) CreatePackage(p Package) (string, error) { + bolt, err := storm.Open(db.Path) + if err != nil { + return "", err + } + defer bolt.Close() + + dp, ok := p.(*DefaultPackage) + if !ok { + return "", errors.New("Bolt DB support only DefaultPackage type for now") + } + err = bolt.Save(dp) + if err != nil { + return "", err + } + + return strconv.Itoa(dp.ID), err +} diff --git a/pkg/package/database_mem.go b/pkg/package/database_mem.go index 7bd29567..084ee9ed 100644 --- a/pkg/package/database_mem.go +++ b/pkg/package/database_mem.go @@ -108,3 +108,15 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) { } return ID, nil } + +func (db *InMemoryDatabase) FindPackage(name, version string) (Package, error) { + return nil, errors.New("Not implemented") +} + +func (db *InMemoryDatabase) UpdatePackage(p Package) error { + return errors.New("Not implemented") +} + +func (db *InMemoryDatabase) GetPackages() []string { + return []string{} +} diff --git a/pkg/package/package.go b/pkg/package/package.go index ae8512a5..5352d708 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -53,30 +53,19 @@ type Package interface { } type PackageSet interface { - GetPackages() []Package - AddPackages(pkgs []Package) -} - -type DefaultPackages struct { - Packages []Package + GetPackages() []string //Ids + CreatePackage(pkg Package) (string, error) + GetPackage(ID string) (Package, error) } type Tree interface { GetPackageSet() PackageSet + Prelude() string // A tree might have a prelude to be able to consume a tree } -func (pkgs *DefaultPackages) GetPackages() []Package { - return pkgs.Packages -} - -func (p *DefaultPackages) AddPackages(pkgs []Package) { - p.Packages = append(p.Packages, pkgs...) -} - -func NewPackages(p []Package) PackageSet { return &DefaultPackages{Packages: p} } - // DefaultPackage represent a standard package definition type DefaultPackage struct { + ID int `storm:"id,increment"` // primary key with auto increment Name string Version string Category string diff --git a/pkg/package/package_test.go b/pkg/package/package_test.go index e5cbf826..3b04e706 100644 --- a/pkg/package/package_test.go +++ b/pkg/package/package_test.go @@ -94,8 +94,8 @@ var _ = Describe("Package", func() { f, err := a1.BuildFormula() Expect(err).ToNot(HaveOccurred()) Expect(len(f)).To(Equal(2)) - Expect(f[0].String()).To(Equal("or(not(d698a46c), 0dc06f33)")) - Expect(f[1].String()).To(Equal("or(not(d698a46c), not(2e57efc4))")) + Expect(f[0].String()).To(Equal("or(not(fd21d375), c85f382e)")) + Expect(f[1].String()).To(Equal("or(not(fd21d375), not(f512c160))")) }) }) diff --git a/pkg/tree/builder.go b/pkg/tree/builder.go index 590b7e9a..34900c7d 100644 --- a/pkg/tree/builder.go +++ b/pkg/tree/builder.go @@ -21,8 +21,7 @@ import ( // reads a luet tree and generates the package lists type Builder interface { - Scan(string) error // compiles a tree - Load(string) error // pre-scanned tree - - Generate() (pkg.PackageSet, error) // generates world + Save(string) error // A tree might be saved to a folder structure (human editable) + Load(string) error // A tree might be loaded from a db (e.g. bolt) and written to folder + Tree() pkg.Tree // generates world } diff --git a/pkg/tree/builder/gentoo/gentoo.go b/pkg/tree/builder/gentoo/gentoo.go index 161462b4..02404660 100644 --- a/pkg/tree/builder/gentoo/gentoo.go +++ b/pkg/tree/builder/gentoo/gentoo.go @@ -19,6 +19,7 @@ package gentoo // https://gist.github.com/adnaan/6ca68c7985c6f851def3 import ( + "io/ioutil" "os" "path/filepath" "strings" @@ -34,7 +35,10 @@ func NewGentooBuilder(e EbuildParser) tree.Parser { type GentooBuilder struct{ EbuildParser EbuildParser } -type GentooTree struct{ Packages pkg.PackageSet } +type GentooTree struct { + Packages pkg.PackageSet + DBPath string +} type EbuildParser interface { ScanEbuild(path string) ([]pkg.Package, error) @@ -44,10 +48,21 @@ func (gt *GentooTree) GetPackageSet() pkg.PackageSet { return gt.Packages } -func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) { - tree := &GentooTree{Packages: pkg.NewPackages([]pkg.Package{})} +func (gt *GentooTree) Prelude() string { + return "/usr/portage/" +} - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { +func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) { + tmpfile, err := ioutil.TempFile("", "boltdb") + if err != nil { + return nil, err + } + + //defer os.Remove(tmpfile.Name()) // clean up + + tree := &GentooTree{Packages: pkg.NewBoltDatabase(tmpfile.Name()), DBPath: tmpfile.Name()} + + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -59,8 +74,12 @@ func (gb *GentooBuilder) Generate(dir string) (pkg.Tree, error) { if err != nil { return err } - tree.Packages.AddPackages(pkgs) - + for _, p := range pkgs { + _, err := tree.GetPackageSet().CreatePackage(p) + if err != nil { + return err + } + } } return nil }) diff --git a/pkg/tree/builder/gentoo/simpleparser.go b/pkg/tree/builder/gentoo/simpleparser.go index b8f6c936..07d108ac 100644 --- a/pkg/tree/builder/gentoo/simpleparser.go +++ b/pkg/tree/builder/gentoo/simpleparser.go @@ -19,16 +19,36 @@ package gentoo // https://gist.github.com/adnaan/6ca68c7985c6f851def3 import ( + "context" "errors" + "fmt" + "os" "path/filepath" "regexp" "strings" pkg "github.com/mudler/luet/pkg/package" + "mvdan.cc/sh/expand" + "mvdan.cc/sh/shell" + "mvdan.cc/sh/syntax" ) // SimpleEbuildParser ignores USE flags and generates just 1-1 package type SimpleEbuildParser struct { + World pkg.PackageDatabase +} + +func SourceFile(ctx context.Context, path string) (map[string]expand.Variable, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("could not open: %v", err) + } + defer f.Close() + file, err := syntax.NewParser(syntax.StopAt("src")).Parse(f, path) + if err != nil { + return nil, fmt.Errorf("could not parse: %v", err) + } + return shell.SourceNode(ctx, file) } // ScanEbuild returns a list of packages (always one with SimpleEbuildParser) decoded from an ebuild. @@ -46,6 +66,27 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) ([]pkg.Package, error) { if len(packageInfo) != 1 || len(packageInfo[0]) != 12 { return []pkg.Package{}, errors.New("Failed decoding ebuild: " + path) } + + vars, err := SourceFile(context.TODO(), path) + if err != nil { + // return []pkg.Package{}, err + } + //fmt.Println("Scanning", path) + //fmt.Println(vars) + pack := &pkg.DefaultPackage{Name: packageInfo[0][2], Version: packageInfo[0][7]} + rdepend, ok := vars["RDEPEND"] + if ok { + rdepends := strings.Split(rdepend.String(), "\n") + + pack.PackageRequires = []*pkg.DefaultPackage{} + for _, rr := range rdepends { + + //TODO: Resolve to db or create a new one. + pack.PackageRequires = append(pack.PackageRequires, &pkg.DefaultPackage{Name: rr}) + } + + } + //TODO: Deps and conflicts - return []pkg.Package{&pkg.DefaultPackage{Name: packageInfo[0][2], Version: packageInfo[0][7]}}, nil + return []pkg.Package{pack}, nil } diff --git a/pkg/tree/builder/gentoo/simpleparser_test.go b/pkg/tree/builder/gentoo/simpleparser_test.go index 138b6d5e..761fa4d7 100644 --- a/pkg/tree/builder/gentoo/simpleparser_test.go +++ b/pkg/tree/builder/gentoo/simpleparser_test.go @@ -31,10 +31,11 @@ var _ = Describe("GentooBuilder", func() { Expect(err).ToNot(HaveOccurred()) Expect(len(tree.GetPackageSet().GetPackages())).To(Equal(10)) - for _, p := range tree.GetPackageSet().GetPackages() { + for _, pid := range tree.GetPackageSet().GetPackages() { + p, err := tree.GetPackageSet().GetPackage(pid) + Expect(err).ToNot(HaveOccurred()) Expect(p.GetName()).To(ContainSubstring("pinentry")) Expect(p.GetVersion()).To(ContainSubstring("1.")) - } })