Support concurrency in InMemoryDB and reach feature parity with boltdb

pkg/package/database_boltdb.go

@@ -19,14 +19,18 @@ import (
 	"errors"
 	"os"
 	"strconv"
+	"sync"
+	"time"
 
 	storm "github.com/asdine/storm"
 	"github.com/asdine/storm/q"
+	"go.etcd.io/bbolt"
 )
 
 //var BoltInstance PackageDatabase
 
 type BoltDatabase struct {
+	sync.Mutex
 	Path string
 }
 
@@ -57,7 +61,7 @@ func (db *BoltDatabase) Retrieve(ID string) ([]byte, error) {
 
 func (db *BoltDatabase) FindPackage(tofind Package) (Package, error) {
 	p := &DefaultPackage{}
-	bolt, err := storm.Open(db.Path)
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
 		return nil, err
 	}
@@ -72,7 +76,7 @@ func (db *BoltDatabase) FindPackage(tofind Package) (Package, error) {
 
 func (db *BoltDatabase) UpdatePackage(p Package) error {
 
-	bolt, err := storm.Open(db.Path)
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
 		return err
 	}
@@ -87,12 +91,12 @@ func (db *BoltDatabase) UpdatePackage(p Package) error {
 		return err
 	}
 
-	return err
+	return nil
 }
 
 func (db *BoltDatabase) GetPackage(ID string) (Package, error) {
 	p := &DefaultPackage{}
-	bolt, err := storm.Open(db.Path)
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
 		return nil, err
 	}
@@ -107,7 +111,7 @@ func (db *BoltDatabase) GetPackage(ID string) (Package, error) {
 
 func (db *BoltDatabase) GetPackages() []string {
 	ids := []string{}
-	bolt, err := storm.Open(db.Path)
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
 		return []string{}
 	}
@@ -124,25 +128,37 @@ func (db *BoltDatabase) GetPackages() []string {
 }
 
 func (db *BoltDatabase) GetAllPackages(packages chan Package) error {
-	bolt, err := storm.Open(db.Path)
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
 		return err
 	}
 	defer bolt.Close()
 	// Fetching records one by one (useful when the bucket contains a lot of records)
-	query := bolt.Select()
+	//query := bolt.Select()
 
-	return query.Each(new(DefaultPackage), func(record interface{}) error {
+	var packs []Package
-		u := record.(*DefaultPackage)
+	err = bolt.All(&packs)
-		packages <- u
+	if err != nil {
 		return err
-	})
+	}
+
+	for _, r := range packs {
+		packages <- r
+	}
+
+	return nil
+
+	// return query.Each(new(DefaultPackage), func(record interface{}) error {
+	// 	u := record.(*DefaultPackage)
+	// 	packages <- u
+	// 	return err
+	// })
 }
 
 // Encode encodes the package to string.
 // It returns an ID which can be used to retrieve the package later on.
 func (db *BoltDatabase) CreatePackage(p Package) (string, error) {
-	bolt, err := storm.Open(db.Path)
+	bolt, err := storm.Open(db.Path, storm.BoltOptions(0600, &bbolt.Options{Timeout: 30 * time.Second}))
 	if err != nil {
 		return "", err
 	}
@@ -161,5 +177,7 @@ func (db *BoltDatabase) CreatePackage(p Package) (string, error) {
 }
 
 func (db *BoltDatabase) Clean() error {
+	db.Lock()
+	defer db.Unlock()
 	return os.RemoveAll(db.Path)
 }

pkg/package/database_mem.go

@@ -21,24 +21,36 @@ import (
 	"errors"
 	"fmt"
 	"hash/crc32"
+	"sync"
 )
 
 var DBInMemoryInstance PackageDatabase
 
 type InMemoryDatabase struct {
+	*sync.Mutex
 	Database map[string]string
 }
 
-func NewInMemoryDatabase() PackageDatabase {
+func NewInMemoryDatabase(singleton bool) PackageDatabase {
 	// In memoryDB is a singleton
-	if DBInMemoryInstance == nil {
+	if singleton && DBInMemoryInstance == nil {
-		DBInMemoryInstance = &InMemoryDatabase{Database: map[string]string{}}
+		DBInMemoryInstance = &InMemoryDatabase{
+			Mutex: &sync.Mutex{},
+
+			Database: map[string]string{}}
+	}
+	if !singleton {
+		return &InMemoryDatabase{
+			Mutex: &sync.Mutex{},
+
+			Database: map[string]string{}}
 	}
 	return DBInMemoryInstance
 }
 
 func (db *InMemoryDatabase) Get(s string) (string, error) {
-
+	db.Lock()
+	defer db.Unlock()
 	pa, ok := db.Database[s]
 	if !ok {
 		return "", errors.New("No key found with that id")
@@ -47,6 +59,8 @@ func (db *InMemoryDatabase) Get(s string) (string, error) {
 }
 
 func (db *InMemoryDatabase) Set(k, v string) error {
+	db.Lock()
+	defer db.Unlock()
 	db.Database[k] = v
 
 	return nil
@@ -55,7 +69,7 @@ func (db *InMemoryDatabase) Set(k, v string) error {
 func (db *InMemoryDatabase) Create(v []byte) (string, error) {
 	enc := base64.StdEncoding.EncodeToString(v)
 	crc32q := crc32.MakeTable(0xD5828281)
-	ID := fmt.Sprintf("%08x", crc32.Checksum([]byte(enc), crc32q))
+	ID := fmt.Sprintf("%08x", crc32.Checksum([]byte(enc), crc32q)) // TODO: Replace with package fingerprint?
 
 	return ID, db.Set(ID, enc)
 }
@@ -88,15 +102,21 @@ func (db *InMemoryDatabase) GetPackage(ID string) (Package, error) {
 	return p, nil
 }
 
-// Not implemented
 func (db *InMemoryDatabase) GetAllPackages(packages chan Package) error {
-	return errors.New("Not implemented")
+	packs := db.GetPackages()
+	for _, p := range packs {
+		pack, err := db.GetPackage(p)
+		if err != nil {
+			return err
+		}
+		packages <- pack
+	}
+	return nil
 }
 
 // Encode encodes the package to string.
 // It returns an ID which can be used to retrieve the package later on.
 func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
-
 	pd, ok := p.(*DefaultPackage)
 	if !ok {
 		return "", errors.New("InMemoryDatabase suports only DefaultPackage")
@@ -114,16 +134,73 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 	return ID, nil
 }
 
+func (db *InMemoryDatabase) encodePackage(p Package) (string, string, error) {
+	pd, ok := p.(*DefaultPackage)
+	if !ok {
+		return "", "", errors.New("InMemoryDatabase suports only DefaultPackage")
+	}
+
+	res, err := json.Marshal(pd)
+	if err != nil {
+		return "", "", err
+	}
+	enc := base64.StdEncoding.EncodeToString(res)
+	crc32q := crc32.MakeTable(0xD5828281)
+	ID := fmt.Sprintf("%08x", crc32.Checksum([]byte(enc), crc32q)) // TODO: Replace with package fingerprint?
+
+	return ID, enc, nil
+}
+
 func (db *InMemoryDatabase) FindPackage(p Package) (Package, error) {
-	return nil, errors.New("Not implemented")
+
+	// TODO: Replace this piece, when IDs are fingerprint, findpackage becames O(1)
+
+	for _, k := range db.GetPackages() {
+		pack, err := db.GetPackage(k)
+		if err != nil {
+			return nil, err
+		}
+		if pack.GetFingerPrint() == p.GetFingerPrint() {
+			return pack, nil
+		}
+	}
+	return nil, errors.New("Package not found")
 }
 
 func (db *InMemoryDatabase) UpdatePackage(p Package) error {
-	return errors.New("Not implemented")
+	var id string
+	found := false
+	for _, k := range db.GetPackages() {
+		pack, err := db.GetPackage(k)
+		if err != nil {
+			return err
+		}
+		if pack.GetFingerPrint() == p.GetFingerPrint() {
+			id = k
+			found = true
+			break
+		}
+	}
+	if found {
+
+		_, enc, err := db.encodePackage(p)
+		if err != nil {
+			return err
+		}
+
+		return db.Set(id, enc)
+	}
+	return errors.New("Package not found")
 }
 
 func (db *InMemoryDatabase) GetPackages() []string {
-	return []string{}
+	keys := []string{}
+	db.Lock()
+	defer db.Unlock()
+	for k, _ := range db.Database {
+		keys = append(keys, k)
+	}
+	return keys
 }
 
 func (db *InMemoryDatabase) Clean() error {

pkg/package/database_test.go

@@ -0,0 +1,60 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg_test
+
+import (
+	. "github.com/mudler/luet/pkg/package"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Database", func() {
+
+	db := NewInMemoryDatabase(false)
+	Context("Simple package", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		//	a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		//		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
+		//		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+		It("Saves and get data back correctly", func() {
+
+			ID, err := db.CreatePackage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			pack, err := db.GetPackage(ID)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack).To(Equal(a))
+
+		})
+
+		It("Gets all", func() {
+
+			ids := db.GetPackages()
+
+			Expect(ids).To(Equal([]string{"9164d667"}))
+
+		})
+		It("Find packages", func() {
+
+			pack, err := db.FindPackage(a)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack).To(Equal(a))
+
+		})
+	})
+
+})