Move revdeps computation to db

cmd/search.go

@@ -135,8 +135,8 @@ var searchCmd = &cobra.Command{
 							})
 					}
 				} else {
-					visited := make(map[string]interface{})
+					packs, _:= m.Repo.GetTree().GetDatabase().GetRevdeps(m.Package)
-					for _, revdep := range m.Package.ExpandedRevdeps(m.Repo.GetTree().GetDatabase(), visited) {
+					for _, revdep := range packs{
 						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
 							Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
 							results.Packages = append(results.Packages,
@@ -189,9 +189,8 @@ var searchCmd = &cobra.Command{
 							})
 					}
 				} else {
-					visited := make(map[string]interface{})
+					packs,_:=system.Database.GetRevdeps(pack)
-
+					for _, revdep := range packs {
-					for _, revdep := range pack.ExpandedRevdeps(system.Database, visited) {
 						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
 							Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
 							results.Packages = append(results.Packages,

cmd/tree/pkglist.go

@@ -168,9 +168,8 @@ func NewTreePkglistCommand() *cobra.Command {
 
 				if addPkg {
 					if revdeps {
-
+						packs, _ := reciper.GetDatabase().GetRevdeps(p)
-						visited := make(map[string]interface{})
+						for _, revdep := range packs {
-						for _, revdep := range p.ExpandedRevdeps(reciper.GetDatabase(), visited) {
 							if full {
 								pkgstr = pkgDetail(revdep)
 							} else if verbose {

pkg/package/database.go

@@ -28,6 +28,7 @@ type PackageDatabase interface {
 }
 
 type PackageSet interface {
+	GetRevdeps(p Package) (Packages, error)
 	GetPackages() []string //Ids
 	CreatePackage(pkg Package) (string, error)
 	GetPackage(ID string) (Package, error)

pkg/package/database_boltdb.go

@@ -86,6 +86,17 @@ func (db *BoltDatabase) Retrieve(ID string) ([]byte, error) {
 	return enc, nil
 }
 
+// GetRevdeps uses a new inmemory db to calcuate revdeps
+// TODO: Have a memory instance for boltdb, so we don't compute each time we get called
+// as this is REALLY expensive. But we don't perform usually those operations in a file db.
+func (db *BoltDatabase) GetRevdeps(p Package) (Packages, error) {
+	memory := NewInMemoryDatabase(false)
+	for _, p := range db.World() {
+		memory.CreatePackage(p)
+	}
+	return memory.GetRevdeps(p)
+}
+
 func (db *BoltDatabase) FindPackage(tofind Package) (Package, error) {
 	// Provides: Return the replaced package here
 	if provided, err := db.getProvide(tofind); err == nil {

pkg/package/database_mem.go

@@ -31,6 +31,7 @@ var DBInMemoryInstance = &InMemoryDatabase{
 	Database:         map[string]string{},
 	CacheNoVersion:   map[string]map[string]interface{}{},
 	ProvidesDatabase: map[string]map[string]Package{},
+	RevDepsDatabase:  map[string]map[string]Package{},
 }
 
 type InMemoryDatabase struct {
@@ -39,6 +40,7 @@ type InMemoryDatabase struct {
 	FileDatabase     map[string][]string
 	CacheNoVersion   map[string]map[string]interface{}
 	ProvidesDatabase map[string]map[string]Package
+	RevDepsDatabase  map[string]map[string]Package
 }
 
 func NewInMemoryDatabase(singleton bool) PackageDatabase {
@@ -50,6 +52,7 @@ func NewInMemoryDatabase(singleton bool) PackageDatabase {
 			Database:         map[string]string{},
 			CacheNoVersion:   map[string]map[string]interface{}{},
 			ProvidesDatabase: map[string]map[string]Package{},
+			RevDepsDatabase:  map[string]map[string]Package{},
 		}
 	}
 	return DBInMemoryInstance
@@ -125,6 +128,47 @@ func (db *InMemoryDatabase) GetAllPackages(packages chan Package) error {
 	return nil
 }
 
+func (db *InMemoryDatabase) getRevdeps(p Package, visited map[string]interface{}) (Packages, error) {
+	var versionsInWorld Packages
+	if _, ok := visited[p.HumanReadableString()]; ok {
+		return versionsInWorld, nil
+	}
+	visited[p.HumanReadableString()] = true
+
+	var res Packages
+	packs, err := db.FindPackages(p)
+	if err != nil {
+		return res, err
+	}
+	for _, pp := range packs {
+		//	db.Lock()
+		list := db.RevDepsDatabase[pp.GetFingerPrint()]
+		//	db.Unlock()
+		for _, revdep := range list {
+			dep, err := db.FindPackage(revdep)
+			if err != nil {
+				return res, err
+			}
+			res = append(res, dep)
+
+			packs, err := db.getRevdeps(dep, visited)
+			if err != nil {
+				return res, err
+			}
+			res = append(res, packs...)
+
+		}
+	}
+	return res.Unique(), nil
+}
+
+// GetRevdeps returns the package reverse dependencies,
+// matching also selectors in versions (>, <, >=, <=)
+// TODO: Code should use db explictly
+func (db *InMemoryDatabase) GetRevdeps(p Package) (Packages, error) {
+	return db.getRevdeps(p, make(map[string]interface{}))
+}
+
 // Encode encodes the package to string.
 // It returns an ID which can be used to retrieve the package later on.
 func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
@@ -143,9 +187,16 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 		return "", err
 	}
 
+	db.populateCaches(pd)
+
+	return ID, nil
+}
+
+func (db *InMemoryDatabase) populateCaches(p Package) {
+	pd, _ := p.(*DefaultPackage)
+
 	// Create extra cache between package -> []versions
 	db.Lock()
-	defer db.Unlock()
 
 	// Provides: Store package provides, we will reuse this when walking deps
 	for _, provide := range pd.Provides {
@@ -157,21 +208,41 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) {
 		db.ProvidesDatabase[provide.GetPackageName()][provide.GetVersion()] = p
 	}
 
-	_, ok = db.CacheNoVersion[p.GetPackageName()]
+	_, ok := db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
 		db.CacheNoVersion[p.GetPackageName()] = make(map[string]interface{})
 	}
 	db.CacheNoVersion[p.GetPackageName()][p.GetVersion()] = nil
+	db.Unlock()
 
-	return ID, nil
+	for _, re := range pd.GetRequires() {
+		packages, _ := db.FindPackages(re)
+		db.Lock()
+
+		for _, pa := range packages {
+			_, ok := db.RevDepsDatabase[pa.GetFingerPrint()]
+			if !ok {
+				db.RevDepsDatabase[pa.GetFingerPrint()] = make(map[string]Package)
+			}
+			db.RevDepsDatabase[pa.GetFingerPrint()][pd.GetFingerPrint()] = pd
+		}
+		_, ok := db.RevDepsDatabase[re.GetFingerPrint()]
+		if !ok {
+			db.RevDepsDatabase[re.GetFingerPrint()] = make(map[string]Package)
+		}
+		db.RevDepsDatabase[re.GetFingerPrint()][pd.GetFingerPrint()] = pd
+		db.Unlock()
+	}
 }
 
 func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
+
 	db.Lock()
+
 	pa, ok := db.ProvidesDatabase[p.GetPackageName()][p.GetVersion()]
 	if !ok {
 		versions, ok := db.ProvidesDatabase[p.GetPackageName()]
-		db.Unlock()
+		defer db.Unlock()
 
 		if !ok {
 			return nil, errors.New("No versions found for package")
@@ -195,6 +266,7 @@ func (db *InMemoryDatabase) getProvide(p Package) (Package, error) {
 		return nil, errors.New("No package provides this")
 	}
 	db.Unlock()
+
 	return db.FindPackage(pa)
 }
 
@@ -229,8 +301,9 @@ func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) {
 	if provided, err := db.getProvide(p); err == nil {
 		p = provided
 	}
-
+	db.Lock()
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
+	db.Unlock()
 	if !ok {
 		return nil, errors.New("No versions found for package")
 	}
@@ -247,29 +320,38 @@ func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) {
 
 // FindPackages return the list of the packages beloging to cat/name (any versions in requested range)
 func (db *InMemoryDatabase) FindPackages(p Package) (Packages, error) {
-
+	if !p.IsSelector() {
+		pack, err := db.FindPackage(p)
+		if err != nil {
+			return []Package{}, err
+		}
+		return []Package{pack}, nil
+	}
 	// Provides: Treat as the replaced package here
 	if provided, err := db.getProvide(p); err == nil {
 		p = provided
 	}
+
+	db.Lock()
+	var matches []*DefaultPackage
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
+	for ve := range versions {
+		match, _ := p.SelectorMatchVersion(ve, nil)
+		if match {
+			matches = append(matches, &DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
+		}
+	}
+	db.Unlock()
 	if !ok {
 		return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString()))
 	}
 	var versionsInWorld []Package
-	for ve, _ := range versions {
+	for _, p := range matches {
-		match, err := p.SelectorMatchVersion(ve, nil)
+		w, err := db.FindPackage(p)
 		if err != nil {
-			return nil, errors.Wrap(err, "Error on match selector")
+			return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")
-		}
-
-		if match {
-			w, err := db.FindPackage(&DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve})
-			if err != nil {
-				return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen")
-			}
-			versionsInWorld = append(versionsInWorld, w)
 		}
+		versionsInWorld = append(versionsInWorld, w)
 	}
 	return Packages(versionsInWorld), nil
 }

pkg/package/package.go

@@ -49,7 +49,6 @@ type Package interface {
 	Requires([]*DefaultPackage) Package
 	Conflicts([]*DefaultPackage) Package
 	Revdeps(PackageDatabase) Packages
-	ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages
 	LabelDeps(PackageDatabase, string) Packages
 
 	GetProvides() []*DefaultPackage
@@ -513,8 +512,7 @@ func walkPackage(p Package, definitiondb PackageDatabase, visited map[string]int
 	}
 	visited[p.HumanReadableString()] = true
 
-	revdepvisited := make(map[string]interface{})
+	revdeps, _ := definitiondb.GetRevdeps(p)
-	revdeps := p.ExpandedRevdeps(definitiondb, revdepvisited)
 	for _, r := range revdeps {
 		versionsInWorld = append(versionsInWorld, r)
 	}
@@ -548,52 +546,6 @@ func (p *DefaultPackage) Related(definitiondb PackageDatabase) Packages {
 	return walkPackage(p, definitiondb, map[string]interface{}{})
 }
 
-// ExpandedRevdeps returns the package reverse dependencies,
-// matching also selectors in versions (>, <, >=, <=)
-func (p *DefaultPackage) ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages {
-	var versionsInWorld Packages
-	if _, ok := visited[p.HumanReadableString()]; ok {
-		return versionsInWorld
-	}
-	visited[p.HumanReadableString()] = true
-
-	for _, w := range definitiondb.World() {
-		if w.Matches(p) {
-			continue
-		}
-		match := false
-
-		for _, re := range w.GetRequires() {
-			if re.Matches(p) {
-				match = true
-			}
-			if !match {
-
-				packages, _ := re.Expand(definitiondb)
-				for _, pa := range packages {
-					if pa.Matches(p) {
-						match = true
-					}
-				}
-			}
-
-			//	if ok, _ := w.RequiresContains(definitiondb, p); ok {
-
-		}
-
-		if match {
-			versionsInWorld = append(versionsInWorld, w)
-
-			versionsInWorld = append(versionsInWorld, w.ExpandedRevdeps(definitiondb, visited).Unique()...)
-
-		}
-		//	}
-
-	}
-	//visited[p.HumanReadableString()] = true
-	return versionsInWorld.Unique()
-}
-
 func (p *DefaultPackage) LabelDeps(definitiondb PackageDatabase, labelKey string) Packages {
 	var pkgsWithLabelInWorld Packages
 	// TODO: check if integrate some index to improve

pkg/package/package_test.go

@@ -220,8 +220,8 @@ var _ = Describe("Package", func() {
 				_, err := definitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			visited := make(map[string]interface{})
+			lst, err := definitions.GetRevdeps(a)
-			lst := a.ExpandedRevdeps(definitions, visited)
+			Expect(err).ToNot(HaveOccurred())
 			Expect(lst).To(ContainElement(c))
 			Expect(lst).To(ContainElement(d))
 			Expect(lst).To(ContainElement(e))
@@ -242,9 +242,9 @@ var _ = Describe("Package", func() {
 				_, err := definitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			visited := make(map[string]interface{})
 
-			lst := a.ExpandedRevdeps(definitions, visited)
+			lst, err := definitions.GetRevdeps(a)
+			Expect(err).ToNot(HaveOccurred())
 			Expect(lst).To(ContainElement(b))
 			Expect(lst).To(ContainElement(c))
 			Expect(lst).To(ContainElement(d))
@@ -266,9 +266,8 @@ var _ = Describe("Package", func() {
 				_, err := definitions.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			visited := make(map[string]interface{})
+			lst, err := definitions.GetRevdeps(a)
-
+			Expect(err).ToNot(HaveOccurred())
-			lst := a.ExpandedRevdeps(definitions, visited)
 			Expect(lst).To(ContainElement(b))
 			Expect(lst).To(ContainElement(c))
 			Expect(lst).To(ContainElement(d))

pkg/solver/parallel.go

@@ -273,9 +273,11 @@ func (s *Parallel) Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error) {
 	for _, p := range ls {
 		temporarySet.CreatePackage(p)
 	}
-	visited := make(map[string]interface{})
-	revdeps := p.ExpandedRevdeps(temporarySet, visited)
 
+	revdeps, err := temporarySet.GetRevdeps(p)
+	if err != nil {
+		return false, errors.Wrap(err, "error scanning revdeps")
+	}
 	var revdepsErr error
 	for _, r := range revdeps {
 		if revdepsErr == nil {

pkg/solver/parallel_test.go

@@ -401,7 +401,7 @@ var _ = Describe("Parallel", func() {
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(4))
+			Expect(len(solution)).To(Equal(3))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -529,7 +529,7 @@ var _ = Describe("Parallel", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(6))
+			Expect(len(solution)).To(Equal(5))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -570,7 +570,7 @@ var _ = Describe("Parallel", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(6))
+			Expect(len(solution)).To(Equal(5))
 			Expect(err).ToNot(HaveOccurred())
 		})
 

pkg/solver/solver.go

@@ -211,8 +211,8 @@ func (s *Solver) BuildPartialWorld(includeInstalled bool) (bf.Formula, error) {
 	if len(formulas) != 0 {
 		return bf.And(formulas...), nil
 	}
-	return bf.True, nil
 
+	return bf.True, nil
 }
 
 func (s *Solver) getList(db pkg.PackageDatabase, lsp pkg.Packages) (pkg.Packages, error) {
@@ -255,8 +255,11 @@ func (s *Solver) Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error) {
 	for _, p := range ls {
 		temporarySet.CreatePackage(p)
 	}
-	visited := make(map[string]interface{})
+
-	revdeps := p.ExpandedRevdeps(temporarySet, visited)
+	revdeps, err := temporarySet.GetRevdeps(p)
+	if err != nil {
+		return false, errors.Wrap(err, "error scanning revdeps")
+	}
 
 	var revdepsErr error
 	for _, r := range revdeps {
@@ -518,7 +521,7 @@ func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAsser
 		}
 	}
 	// Then try to uninstall the versions in the system, and store that tree
-	r, err := s.Uninstall(checkconflicts, false, toUninstall...)
+	r, err := s.Uninstall(checkconflicts, false, toUninstall.Unique()...)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't uninstall candidates ")
 	}
@@ -532,7 +535,9 @@ func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAsser
 	if len(toInstall) == 0 {
 		return toUninstall, PackagesAssertions{}, nil
 	}
-	assertions, err := s2.Install(toInstall)
+
+	assertions, err := s2.Install(toInstall.Unique())
+
 	return toUninstall, assertions, err
 	// To that tree, ask to install the versions that should be upgraded, and try to solve
 	// Return the solution
@@ -604,6 +609,7 @@ func (s *Solver) Uninstall(checkconflicts, full bool, packs ...pkg.Package) (pkg
 
 	s2 := NewSolver(Options{Type: SingleCoreSimple}, pkg.NewInMemoryDatabase(false), s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
 	s2.SetResolver(s.Resolver)
+
 	// Get the requirements to install the candidate
 	asserts, err := s2.Install(toRemove)
 	if err != nil {
@@ -652,6 +658,7 @@ func (s *Solver) BuildFormula() (bf.Formula, error) {
 	}
 
 	for _, wanted := range s.Wanted {
+
 		encodedW, err := wanted.Encode(s.SolverDatabase)
 		if err != nil {
 			return nil, err

pkg/solver/solver_test.go

@@ -401,7 +401,7 @@ var _ = Describe("Solver", func() {
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(4))
+			Expect(len(solution)).To(Equal(3))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -529,7 +529,7 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(6))
+			Expect(len(solution)).To(Equal(5))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -570,7 +570,7 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(6))
+			Expect(len(solution)).To(Equal(5))
 			Expect(err).ToNot(HaveOccurred())
 		})