diff --git a/cmd/search.go b/cmd/search.go index 3d7b6d84..84b122d7 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -135,8 +135,8 @@ var searchCmd = &cobra.Command{ }) } } else { - visited := make(map[string]interface{}) - for _, revdep := range m.Package.ExpandedRevdeps(m.Repo.GetTree().GetDatabase(), visited) { + packs, _:= m.Repo.GetTree().GetDatabase().GetRevdeps(m.Package) + for _, revdep := range packs{ if !revdep.IsHidden() || revdep.IsHidden() && hidden { Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString())) results.Packages = append(results.Packages, @@ -189,9 +189,8 @@ var searchCmd = &cobra.Command{ }) } } else { - visited := make(map[string]interface{}) - - for _, revdep := range pack.ExpandedRevdeps(system.Database, visited) { + packs,_:=system.Database.GetRevdeps(pack) + for _, revdep := range packs { if !revdep.IsHidden() || revdep.IsHidden() && hidden { Info(fmt.Sprintf(":package:%s", pack.HumanReadableString())) results.Packages = append(results.Packages, diff --git a/cmd/tree/pkglist.go b/cmd/tree/pkglist.go index 5b8486a1..91723459 100644 --- a/cmd/tree/pkglist.go +++ b/cmd/tree/pkglist.go @@ -168,9 +168,8 @@ func NewTreePkglistCommand() *cobra.Command { if addPkg { if revdeps { - - visited := make(map[string]interface{}) - for _, revdep := range p.ExpandedRevdeps(reciper.GetDatabase(), visited) { + packs, _ := reciper.GetDatabase().GetRevdeps(p) + for _, revdep := range packs { if full { pkgstr = pkgDetail(revdep) } else if verbose { diff --git a/pkg/package/database.go b/pkg/package/database.go index 82983250..e5c69f10 100644 --- a/pkg/package/database.go +++ b/pkg/package/database.go @@ -28,6 +28,7 @@ type PackageDatabase interface { } type PackageSet interface { + GetRevdeps(p Package) (Packages, error) GetPackages() []string //Ids CreatePackage(pkg Package) (string, error) GetPackage(ID string) (Package, error) diff --git a/pkg/package/database_boltdb.go b/pkg/package/database_boltdb.go index 9c52fc71..785748c4 100644 --- a/pkg/package/database_boltdb.go +++ b/pkg/package/database_boltdb.go @@ -86,6 +86,17 @@ func (db *BoltDatabase) Retrieve(ID string) ([]byte, error) { return enc, nil } +// GetRevdeps uses a new inmemory db to calcuate revdeps +// TODO: Have a memory instance for boltdb, so we don't compute each time we get called +// as this is REALLY expensive. But we don't perform usually those operations in a file db. +func (db *BoltDatabase) GetRevdeps(p Package) (Packages, error) { + memory := NewInMemoryDatabase(false) + for _, p := range db.World() { + memory.CreatePackage(p) + } + return memory.GetRevdeps(p) +} + func (db *BoltDatabase) FindPackage(tofind Package) (Package, error) { // Provides: Return the replaced package here if provided, err := db.getProvide(tofind); err == nil { diff --git a/pkg/package/database_mem.go b/pkg/package/database_mem.go index c36d8fdf..970d20df 100644 --- a/pkg/package/database_mem.go +++ b/pkg/package/database_mem.go @@ -31,6 +31,7 @@ var DBInMemoryInstance = &InMemoryDatabase{ Database: map[string]string{}, CacheNoVersion: map[string]map[string]interface{}{}, ProvidesDatabase: map[string]map[string]Package{}, + RevDepsDatabase: map[string]map[string]Package{}, } type InMemoryDatabase struct { @@ -39,6 +40,7 @@ type InMemoryDatabase struct { FileDatabase map[string][]string CacheNoVersion map[string]map[string]interface{} ProvidesDatabase map[string]map[string]Package + RevDepsDatabase map[string]map[string]Package } func NewInMemoryDatabase(singleton bool) PackageDatabase { @@ -50,6 +52,7 @@ func NewInMemoryDatabase(singleton bool) PackageDatabase { Database: map[string]string{}, CacheNoVersion: map[string]map[string]interface{}{}, ProvidesDatabase: map[string]map[string]Package{}, + RevDepsDatabase: map[string]map[string]Package{}, } } return DBInMemoryInstance @@ -125,6 +128,47 @@ func (db *InMemoryDatabase) GetAllPackages(packages chan Package) error { return nil } +func (db *InMemoryDatabase) getRevdeps(p Package, visited map[string]interface{}) (Packages, error) { + var versionsInWorld Packages + if _, ok := visited[p.HumanReadableString()]; ok { + return versionsInWorld, nil + } + visited[p.HumanReadableString()] = true + + var res Packages + packs, err := db.FindPackages(p) + if err != nil { + return res, err + } + for _, pp := range packs { + // db.Lock() + list := db.RevDepsDatabase[pp.GetFingerPrint()] + // db.Unlock() + for _, revdep := range list { + dep, err := db.FindPackage(revdep) + if err != nil { + return res, err + } + res = append(res, dep) + + packs, err := db.getRevdeps(dep, visited) + if err != nil { + return res, err + } + res = append(res, packs...) + + } + } + return res.Unique(), nil +} + +// GetRevdeps returns the package reverse dependencies, +// matching also selectors in versions (>, <, >=, <=) +// TODO: Code should use db explictly +func (db *InMemoryDatabase) GetRevdeps(p Package) (Packages, error) { + return db.getRevdeps(p, make(map[string]interface{})) +} + // Encode encodes the package to string. // It returns an ID which can be used to retrieve the package later on. func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) { @@ -143,9 +187,16 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) { return "", err } + db.populateCaches(pd) + + return ID, nil +} + +func (db *InMemoryDatabase) populateCaches(p Package) { + pd, _ := p.(*DefaultPackage) + // Create extra cache between package -> []versions db.Lock() - defer db.Unlock() // Provides: Store package provides, we will reuse this when walking deps for _, provide := range pd.Provides { @@ -157,21 +208,41 @@ func (db *InMemoryDatabase) CreatePackage(p Package) (string, error) { db.ProvidesDatabase[provide.GetPackageName()][provide.GetVersion()] = p } - _, ok = db.CacheNoVersion[p.GetPackageName()] + _, ok := db.CacheNoVersion[p.GetPackageName()] if !ok { db.CacheNoVersion[p.GetPackageName()] = make(map[string]interface{}) } db.CacheNoVersion[p.GetPackageName()][p.GetVersion()] = nil + db.Unlock() - return ID, nil + for _, re := range pd.GetRequires() { + packages, _ := db.FindPackages(re) + db.Lock() + + for _, pa := range packages { + _, ok := db.RevDepsDatabase[pa.GetFingerPrint()] + if !ok { + db.RevDepsDatabase[pa.GetFingerPrint()] = make(map[string]Package) + } + db.RevDepsDatabase[pa.GetFingerPrint()][pd.GetFingerPrint()] = pd + } + _, ok := db.RevDepsDatabase[re.GetFingerPrint()] + if !ok { + db.RevDepsDatabase[re.GetFingerPrint()] = make(map[string]Package) + } + db.RevDepsDatabase[re.GetFingerPrint()][pd.GetFingerPrint()] = pd + db.Unlock() + } } func (db *InMemoryDatabase) getProvide(p Package) (Package, error) { + db.Lock() + pa, ok := db.ProvidesDatabase[p.GetPackageName()][p.GetVersion()] if !ok { versions, ok := db.ProvidesDatabase[p.GetPackageName()] - db.Unlock() + defer db.Unlock() if !ok { return nil, errors.New("No versions found for package") @@ -195,6 +266,7 @@ func (db *InMemoryDatabase) getProvide(p Package) (Package, error) { return nil, errors.New("No package provides this") } db.Unlock() + return db.FindPackage(pa) } @@ -229,8 +301,9 @@ func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) { if provided, err := db.getProvide(p); err == nil { p = provided } - + db.Lock() versions, ok := db.CacheNoVersion[p.GetPackageName()] + db.Unlock() if !ok { return nil, errors.New("No versions found for package") } @@ -247,29 +320,38 @@ func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) { // FindPackages return the list of the packages beloging to cat/name (any versions in requested range) func (db *InMemoryDatabase) FindPackages(p Package) (Packages, error) { - + if !p.IsSelector() { + pack, err := db.FindPackage(p) + if err != nil { + return []Package{}, err + } + return []Package{pack}, nil + } // Provides: Treat as the replaced package here if provided, err := db.getProvide(p); err == nil { p = provided } + + db.Lock() + var matches []*DefaultPackage versions, ok := db.CacheNoVersion[p.GetPackageName()] + for ve := range versions { + match, _ := p.SelectorMatchVersion(ve, nil) + if match { + matches = append(matches, &DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve}) + } + } + db.Unlock() if !ok { return nil, errors.New(fmt.Sprintf("No versions found for: %s", p.HumanReadableString())) } var versionsInWorld []Package - for ve, _ := range versions { - match, err := p.SelectorMatchVersion(ve, nil) + for _, p := range matches { + w, err := db.FindPackage(p) if err != nil { - return nil, errors.Wrap(err, "Error on match selector") - } - - if match { - w, err := db.FindPackage(&DefaultPackage{Name: p.GetName(), Category: p.GetCategory(), Version: ve}) - if err != nil { - return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen") - } - versionsInWorld = append(versionsInWorld, w) + return nil, errors.Wrap(err, "Cache mismatch - this shouldn't happen") } + versionsInWorld = append(versionsInWorld, w) } return Packages(versionsInWorld), nil } diff --git a/pkg/package/package.go b/pkg/package/package.go index 9d9ffbd4..3ecd1524 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -49,7 +49,6 @@ type Package interface { Requires([]*DefaultPackage) Package Conflicts([]*DefaultPackage) Package Revdeps(PackageDatabase) Packages - ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages LabelDeps(PackageDatabase, string) Packages GetProvides() []*DefaultPackage @@ -513,8 +512,7 @@ func walkPackage(p Package, definitiondb PackageDatabase, visited map[string]int } visited[p.HumanReadableString()] = true - revdepvisited := make(map[string]interface{}) - revdeps := p.ExpandedRevdeps(definitiondb, revdepvisited) + revdeps, _ := definitiondb.GetRevdeps(p) for _, r := range revdeps { versionsInWorld = append(versionsInWorld, r) } @@ -548,52 +546,6 @@ func (p *DefaultPackage) Related(definitiondb PackageDatabase) Packages { return walkPackage(p, definitiondb, map[string]interface{}{}) } -// ExpandedRevdeps returns the package reverse dependencies, -// matching also selectors in versions (>, <, >=, <=) -func (p *DefaultPackage) ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages { - var versionsInWorld Packages - if _, ok := visited[p.HumanReadableString()]; ok { - return versionsInWorld - } - visited[p.HumanReadableString()] = true - - for _, w := range definitiondb.World() { - if w.Matches(p) { - continue - } - match := false - - for _, re := range w.GetRequires() { - if re.Matches(p) { - match = true - } - if !match { - - packages, _ := re.Expand(definitiondb) - for _, pa := range packages { - if pa.Matches(p) { - match = true - } - } - } - - // if ok, _ := w.RequiresContains(definitiondb, p); ok { - - } - - if match { - versionsInWorld = append(versionsInWorld, w) - - versionsInWorld = append(versionsInWorld, w.ExpandedRevdeps(definitiondb, visited).Unique()...) - - } - // } - - } - //visited[p.HumanReadableString()] = true - return versionsInWorld.Unique() -} - func (p *DefaultPackage) LabelDeps(definitiondb PackageDatabase, labelKey string) Packages { var pkgsWithLabelInWorld Packages // TODO: check if integrate some index to improve diff --git a/pkg/package/package_test.go b/pkg/package/package_test.go index 3e5e40e6..d2d11a87 100644 --- a/pkg/package/package_test.go +++ b/pkg/package/package_test.go @@ -220,8 +220,8 @@ var _ = Describe("Package", func() { _, err := definitions.CreatePackage(p) Expect(err).ToNot(HaveOccurred()) } - visited := make(map[string]interface{}) - lst := a.ExpandedRevdeps(definitions, visited) + lst, err := definitions.GetRevdeps(a) + Expect(err).ToNot(HaveOccurred()) Expect(lst).To(ContainElement(c)) Expect(lst).To(ContainElement(d)) Expect(lst).To(ContainElement(e)) @@ -242,9 +242,9 @@ var _ = Describe("Package", func() { _, err := definitions.CreatePackage(p) Expect(err).ToNot(HaveOccurred()) } - visited := make(map[string]interface{}) - lst := a.ExpandedRevdeps(definitions, visited) + lst, err := definitions.GetRevdeps(a) + Expect(err).ToNot(HaveOccurred()) Expect(lst).To(ContainElement(b)) Expect(lst).To(ContainElement(c)) Expect(lst).To(ContainElement(d)) @@ -266,9 +266,8 @@ var _ = Describe("Package", func() { _, err := definitions.CreatePackage(p) Expect(err).ToNot(HaveOccurred()) } - visited := make(map[string]interface{}) - - lst := a.ExpandedRevdeps(definitions, visited) + lst, err := definitions.GetRevdeps(a) + Expect(err).ToNot(HaveOccurred()) Expect(lst).To(ContainElement(b)) Expect(lst).To(ContainElement(c)) Expect(lst).To(ContainElement(d)) diff --git a/pkg/solver/parallel.go b/pkg/solver/parallel.go index 6e566932..226a285b 100644 --- a/pkg/solver/parallel.go +++ b/pkg/solver/parallel.go @@ -273,9 +273,11 @@ func (s *Parallel) Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error) { for _, p := range ls { temporarySet.CreatePackage(p) } - visited := make(map[string]interface{}) - revdeps := p.ExpandedRevdeps(temporarySet, visited) + revdeps, err := temporarySet.GetRevdeps(p) + if err != nil { + return false, errors.Wrap(err, "error scanning revdeps") + } var revdepsErr error for _, r := range revdeps { if revdepsErr == nil { diff --git a/pkg/solver/parallel_test.go b/pkg/solver/parallel_test.go index f157e822..718ec260 100644 --- a/pkg/solver/parallel_test.go +++ b/pkg/solver/parallel_test.go @@ -401,7 +401,7 @@ var _ = Describe("Parallel", func() { Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: false})) Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true})) - Expect(len(solution)).To(Equal(4)) + Expect(len(solution)).To(Equal(3)) Expect(err).ToNot(HaveOccurred()) }) @@ -529,7 +529,7 @@ var _ = Describe("Parallel", func() { Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false})) Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true})) - Expect(len(solution)).To(Equal(6)) + Expect(len(solution)).To(Equal(5)) Expect(err).ToNot(HaveOccurred()) }) @@ -570,7 +570,7 @@ var _ = Describe("Parallel", func() { Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false})) Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true})) - Expect(len(solution)).To(Equal(6)) + Expect(len(solution)).To(Equal(5)) Expect(err).ToNot(HaveOccurred()) }) diff --git a/pkg/solver/solver.go b/pkg/solver/solver.go index fc4479b0..a7e60b8f 100644 --- a/pkg/solver/solver.go +++ b/pkg/solver/solver.go @@ -211,8 +211,8 @@ func (s *Solver) BuildPartialWorld(includeInstalled bool) (bf.Formula, error) { if len(formulas) != 0 { return bf.And(formulas...), nil } - return bf.True, nil + return bf.True, nil } func (s *Solver) getList(db pkg.PackageDatabase, lsp pkg.Packages) (pkg.Packages, error) { @@ -255,8 +255,11 @@ func (s *Solver) Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error) { for _, p := range ls { temporarySet.CreatePackage(p) } - visited := make(map[string]interface{}) - revdeps := p.ExpandedRevdeps(temporarySet, visited) + + revdeps, err := temporarySet.GetRevdeps(p) + if err != nil { + return false, errors.Wrap(err, "error scanning revdeps") + } var revdepsErr error for _, r := range revdeps { @@ -518,7 +521,7 @@ func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAsser } } // Then try to uninstall the versions in the system, and store that tree - r, err := s.Uninstall(checkconflicts, false, toUninstall...) + r, err := s.Uninstall(checkconflicts, false, toUninstall.Unique()...) if err != nil { return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't uninstall candidates ") } @@ -532,7 +535,9 @@ func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAsser if len(toInstall) == 0 { return toUninstall, PackagesAssertions{}, nil } - assertions, err := s2.Install(toInstall) + + assertions, err := s2.Install(toInstall.Unique()) + return toUninstall, assertions, err // To that tree, ask to install the versions that should be upgraded, and try to solve // Return the solution @@ -604,6 +609,7 @@ func (s *Solver) Uninstall(checkconflicts, full bool, packs ...pkg.Package) (pkg s2 := NewSolver(Options{Type: SingleCoreSimple}, pkg.NewInMemoryDatabase(false), s.DefinitionDatabase, pkg.NewInMemoryDatabase(false)) s2.SetResolver(s.Resolver) + // Get the requirements to install the candidate asserts, err := s2.Install(toRemove) if err != nil { @@ -652,6 +658,7 @@ func (s *Solver) BuildFormula() (bf.Formula, error) { } for _, wanted := range s.Wanted { + encodedW, err := wanted.Encode(s.SolverDatabase) if err != nil { return nil, err diff --git a/pkg/solver/solver_test.go b/pkg/solver/solver_test.go index 97442be9..e7502470 100644 --- a/pkg/solver/solver_test.go +++ b/pkg/solver/solver_test.go @@ -401,7 +401,7 @@ var _ = Describe("Solver", func() { Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: false})) Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true})) - Expect(len(solution)).To(Equal(4)) + Expect(len(solution)).To(Equal(3)) Expect(err).ToNot(HaveOccurred()) }) @@ -529,7 +529,7 @@ var _ = Describe("Solver", func() { Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false})) Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true})) - Expect(len(solution)).To(Equal(6)) + Expect(len(solution)).To(Equal(5)) Expect(err).ToNot(HaveOccurred()) }) @@ -570,7 +570,7 @@ var _ = Describe("Solver", func() { Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false})) Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true})) - Expect(len(solution)).To(Equal(6)) + Expect(len(solution)).To(Equal(5)) Expect(err).ToNot(HaveOccurred()) })