From f6a4b634c103bdcc03b075f6b421b4e89f7e1b01 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 30 Oct 2020 19:12:12 +0100 Subject: [PATCH] Don't always walk all World() packages With this change the solver during install now considers only the part of the tree which is required to calculate the solution, it doesn't consider anymore World() as the search space. The search space now is narrowed down to the packages that related to the one which we are considering. In this subset of changes we are also optimizing the Parallel solver avoiding an useless loop. This change boost overall performance on large datasets which don't necessarly have relations touching the whole tree. --- pkg/package/package.go | 43 ++++++++++ pkg/solver/benchmark_test.go | 112 +++++++++++++++++++++++++- pkg/solver/decoder.go | 16 ++-- pkg/solver/decoder_test.go | 10 +-- pkg/solver/parallel.go | 150 +++++++++++++++++++++++------------ pkg/solver/parallel_test.go | 6 +- pkg/solver/resolver_test.go | 22 ++--- pkg/solver/solver.go | 79 +++++++++++++----- pkg/solver/solver_test.go | 6 +- pkg/tree/recipes_test.go | 2 +- pkg/tree/tree_test.go | 32 +++----- 11 files changed, 355 insertions(+), 123 deletions(-) diff --git a/pkg/package/package.go b/pkg/package/package.go index ebcb61e0..e13abe3b 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -40,6 +40,7 @@ import ( // FIXME: Currently some of the methods are returning DefaultPackages due to JSON serialization of the package type Package interface { Encode(PackageDatabase) (string, error) + Related(definitiondb PackageDatabase) Packages BuildFormula(PackageDatabase, PackageDatabase) ([]bf.Formula, error) @@ -451,6 +452,48 @@ func (p *DefaultPackage) Revdeps(definitiondb PackageDatabase) Packages { return versionsInWorld } +func walkPackage(p Package, definitiondb PackageDatabase, visited map[string]interface{}) Packages { + var versionsInWorld Packages + if _, ok := visited[p.HumanReadableString()]; ok { + return versionsInWorld + } + visited[p.HumanReadableString()] = true + + revdepvisited := make(map[string]interface{}) + revdeps := p.ExpandedRevdeps(definitiondb, revdepvisited) + for _, r := range revdeps { + versionsInWorld = append(versionsInWorld, r) + } + + if !p.IsSelector() { + versionsInWorld = append(versionsInWorld, p) + } + + for _, re := range p.GetRequires() { + versions, _ := re.Expand(definitiondb) + for _, r := range versions { + + versionsInWorld = append(versionsInWorld, r) + versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...) + } + + } + for _, re := range p.GetConflicts() { + versions, _ := re.Expand(definitiondb) + for _, r := range versions { + + versionsInWorld = append(versionsInWorld, r) + versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...) + + } + } + return versionsInWorld.Unique() +} + +func (p *DefaultPackage) Related(definitiondb PackageDatabase) Packages { + return walkPackage(p, definitiondb, map[string]interface{}{}) +} + // ExpandedRevdeps returns the package reverse dependencies, // matching also selectors in versions (>, <, >=, <=) func (p *DefaultPackage) ExpandedRevdeps(definitiondb PackageDatabase, visited map[string]interface{}) Packages { diff --git a/pkg/solver/benchmark_test.go b/pkg/solver/benchmark_test.go index c2edc13d..78f95794 100644 --- a/pkg/solver/benchmark_test.go +++ b/pkg/solver/benchmark_test.go @@ -16,11 +16,13 @@ package solver_test import ( + "fmt" + "os" + "strconv" + pkg "github.com/mudler/luet/pkg/package" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "os" - "strconv" . "github.com/mudler/luet/pkg/solver" ) @@ -139,4 +141,110 @@ var _ = Describe("Solver Benchmarks", func() { }, 1) }) + Context("Complex data sets - Parallel Upgrades", func() { + BeforeEach(func() { + db = pkg.NewInMemoryDatabase(false) + dbInstalled = pkg.NewInMemoryDatabase(false) + dbDefinitions = pkg.NewInMemoryDatabase(false) + s = NewSolver(Options{Type: ParallelSimple, Concurrency: 100}, dbInstalled, dbDefinitions, db) + if os.Getenv("BENCHMARK_TESTS") != "true" { + Skip("BENCHMARK_TESTS not enabled") + } + }) + Measure("it should be fast in resolution from a 10000*8 dataset", func(b Benchmarker) { + runtime := b.Time("runtime", func() { + for i := 2; i < 10000; i++ { + C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{}) + D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{}) + B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{}) + A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}) + for _, p := range []pkg.Package{A, B, C, D, E, F, G, H} { + _, err := dbDefinitions.CreatePackage(p) + Expect(err).ToNot(HaveOccurred()) + } + } + + //C := pkg.NewPackage("C", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + G := pkg.NewPackage("G", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + H := pkg.NewPackage("H", "1", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{}) + D := pkg.NewPackage("D", "1", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{}) + B := pkg.NewPackage("B", "1", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{}) + A := pkg.NewPackage("A", "1", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}) + _, err := dbInstalled.CreatePackage(A) + Expect(err).ToNot(HaveOccurred()) + _, err = dbInstalled.CreatePackage(B) + Expect(err).ToNot(HaveOccurred()) + _, err = dbInstalled.CreatePackage(D) + Expect(err).ToNot(HaveOccurred()) + _, err = dbInstalled.CreatePackage(H) + Expect(err).ToNot(HaveOccurred()) + _, err = dbInstalled.CreatePackage(G) + Expect(err).ToNot(HaveOccurred()) + fmt.Println("Upgrade starts") + + packages, ass, err := s.Upgrade(false, true) + Expect(err).ToNot(HaveOccurred()) + + Expect(packages).To(ContainElement(A)) + + G = pkg.NewPackage("G", "9999", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + H = pkg.NewPackage("H", "9999", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{}) + D = pkg.NewPackage("D", "9999", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{}) + B = pkg.NewPackage("B", "9999", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{}) + A = pkg.NewPackage("A", "9999", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}) + Expect(ass).To(ContainElement(PackageAssert{Package: A, Value: true})) + + Expect(len(packages)).To(Equal(5)) + // Expect(len(solution)).To(Equal(6)) + + }) + + Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.") + }, 1) + + PMeasure("it should be fast in resolution from a 50000 dataset with upgrade universe", func(b Benchmarker) { + runtime := b.Time("runtime", func() { + for i := 0; i < 2; i++ { + C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{}) + D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{}) + B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{}) + A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}) + for _, p := range []pkg.Package{A, B, C, D, E, F, G} { + _, err := dbDefinitions.CreatePackage(p) + Expect(err).ToNot(HaveOccurred()) + } + fmt.Println("Creating package, run", i) + } + + G := pkg.NewPackage("G", "1", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{}) + H := pkg.NewPackage("H", "1", []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{}) + D := pkg.NewPackage("D", "1", []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{}) + B := pkg.NewPackage("B", "1", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{}) + A := pkg.NewPackage("A", "1", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}) + _, err := dbInstalled.CreatePackage(A) + Expect(err).ToNot(HaveOccurred()) + fmt.Println("Upgrade starts") + + packages, ass, err := s.UpgradeUniverse(true) + Expect(err).ToNot(HaveOccurred()) + + Expect(ass).To(ContainElement(PackageAssert{Package: pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}), Value: true})) + Expect(packages).To(ContainElement(pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}))) + Expect(packages).To(Equal(5)) + // Expect(len(solution)).To(Equal(6)) + + }) + + Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.") + }, 1) + }) + }) diff --git a/pkg/solver/decoder.go b/pkg/solver/decoder.go index 17bd0ab8..301664c2 100644 --- a/pkg/solver/decoder.go +++ b/pkg/solver/decoder.go @@ -150,22 +150,13 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin orderedAssertions := PackagesAssertions{} unorderedAssertions := PackagesAssertions{} - fingerprints := []string{} tmpMap := map[string]PackageAssert{} graph := topsort.NewGraph() - for _, a := range assertions { graph.AddNode(a.Package.GetFingerPrint()) tmpMap[a.Package.GetFingerPrint()] = a - fingerprints = append(fingerprints, a.Package.GetFingerPrint()) unorderedAssertions = append(unorderedAssertions, a) // Build a list of the ones that must be ordered - - if a.Value { - unorderedAssertions = append(unorderedAssertions, a) // Build a list of the ones that must be ordered - } else { - orderedAssertions = append(orderedAssertions, a) // Keep last the ones which are not meant to be installed - } } sort.Sort(unorderedAssertions) @@ -200,8 +191,11 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin for _, res := range result { a, ok := tmpMap[res] if !ok { - return nil, errors.New("fail looking for " + res) - // continue + //return nil, errors.New("fail looking for " + res) + // Since now we don't return the entire world as part of assertions + // if we don't find any reference must be because fingerprint we are analyzing (which is the one we are ordering against) + // is not part of the assertions, thus we can omit it from the result + continue } orderedAssertions = append(orderedAssertions, a) // orderedAssertions = append(PackagesAssertions{a}, orderedAssertions...) // push upfront diff --git a/pkg/solver/decoder_test.go b/pkg/solver/decoder_test.go index 88ed6d1d..028299ae 100644 --- a/pkg/solver/decoder_test.go +++ b/pkg/solver/decoder_test.go @@ -214,12 +214,10 @@ var _ = Describe("Decoder", func() { hash2 := solution.AssertionHash() // Expect(len(solution)).To(Equal(6)) - Expect(solution[0].Package.GetName()).To(Equal("A")) - Expect(solution[1].Package.GetName()).To(Equal("G")) - Expect(solution[2].Package.GetName()).To(Equal("H")) - Expect(solution[3].Package.GetName()).To(Equal("D")) - Expect(solution[4].Package.GetName()).To(Equal("B")) - Expect(solution[0].Value).ToNot(BeTrue()) + Expect(solution[0].Package.GetName()).To(Equal("G")) + Expect(solution[1].Package.GetName()).To(Equal("H")) + Expect(solution[2].Package.GetName()).To(Equal("D")) + Expect(solution[3].Package.GetName()).To(Equal("B")) Expect(hash).ToNot(Equal("")) Expect(hash2).ToNot(Equal("")) diff --git a/pkg/solver/parallel.go b/pkg/solver/parallel.go index bad7696a..af9df637 100644 --- a/pkg/solver/parallel.go +++ b/pkg/solver/parallel.go @@ -113,12 +113,26 @@ func (s *Parallel) buildParallelFormula(formulas []bf.Formula, packages pkg.Pack wg.Wait() close(results) wg2.Wait() - return bf.And(formulas...), nil + + if len(formulas) != 0 { + return bf.And(formulas...), nil + } + return bf.True, nil } func (s *Parallel) BuildInstalled() (bf.Formula, error) { var formulas []bf.Formula - return s.buildParallelFormula(formulas, s.Installed()) + + var packages pkg.Packages + for _, p := range s.Installed() { + packages = append(packages, p) + for _, dep := range p.Related(s.DefinitionDatabase) { + packages = append(packages, dep) + } + + } + + return s.buildParallelFormula(formulas, packages) } // BuildWorld builds the formula which olds the requirements from the package definitions @@ -134,10 +148,63 @@ func (s *Parallel) BuildWorld(includeInstalled bool) (bf.Formula, error) { //f = bf.And(f, solvable) formulas = append(formulas, solvable) } - return s.buildParallelFormula(formulas, s.World()) } +// BuildWorld builds the formula which olds the requirements from the package definitions +// which are available (global state) +func (s *Parallel) BuildPartialWorld(includeInstalled bool) (bf.Formula, error) { + var formulas []bf.Formula + // NOTE: This block should be enabled in case of very old systems with outdated world sets + if includeInstalled { + solvable, err := s.BuildInstalled() + if err != nil { + return nil, err + } + //f = bf.And(f, solvable) + formulas = append(formulas, solvable) + } + + var wg = new(sync.WaitGroup) + var wg2 = new(sync.WaitGroup) + var packages pkg.Packages + + all := make(chan pkg.Package) + results := make(chan pkg.Package, 1) + for i := 0; i < s.Concurrency; i++ { + wg.Add(1) + go func(wg *sync.WaitGroup, c <-chan pkg.Package) { + defer wg.Done() + for p := range c { + for _, dep := range p.Related(s.DefinitionDatabase) { + results <- dep + } + + } + }(wg, all) + } + wg2.Add(1) + go func() { + defer wg2.Done() + for t := range results { + packages = append(packages, t) + } + }() + + for _, p := range s.Wanted { + all <- p + } + + close(all) + wg.Wait() + close(results) + wg2.Wait() + + return s.buildParallelFormula(formulas, packages) + + //return s.buildParallelFormula(formulas, s.World()) +} + func (s *Parallel) getList(db pkg.PackageDatabase, lsp pkg.Packages) (pkg.Packages, error) { var ls pkg.Packages var wg = new(sync.WaitGroup) @@ -370,12 +437,11 @@ func (s *Parallel) UninstallUniverse(toremove pkg.Packages) (pkg.Packages, error // the Universe db as authoritative // See also on the subject: https://arxiv.org/pdf/1007.1021.pdf func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssertions, error) { + var formulas []bf.Formula // we first figure out which aren't up-to-date // which has to be removed // and which needs to be upgraded - notUptodate := pkg.Packages{} removed := pkg.Packages{} - toUpgrade := pkg.Packages{} // TODO: this is memory expensive, we need to optimize this universe := pkg.NewInMemoryDatabase(false) @@ -387,11 +453,17 @@ func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAsse universe.CreatePackage(p) } + // Build constraints for the whole defdb + r, err := s.BuildWorld(true) + if err != nil { + return nil, nil, errors.Wrap(err, "couldn't build world constraints") + } + var wg = new(sync.WaitGroup) var wg2 = new(sync.WaitGroup) all := make(chan pkg.Package) - results := make(chan []pkg.Package, 1) + results := make(chan bf.Formula, 1) for i := 0; i < s.Concurrency; i++ { wg.Add(1) go func(wg *sync.WaitGroup, c <-chan pkg.Package) { @@ -408,7 +480,12 @@ func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAsse bestmatch := available.Best(nil) // Found a better version available if !bestmatch.Matches(p) { - results <- []pkg.Package{p, bestmatch} + encodedP, _ := p.Encode(universe) + P := bf.Var(encodedP) + results <- bf.And(bf.Not(P), r) + encodedP, _ = bestmatch.Encode(universe) + P = bf.Var(encodedP) + results <- bf.And(P, r) } } }(wg, all) @@ -418,8 +495,7 @@ func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAsse go func() { defer wg2.Done() for t := range results { - notUptodate = append(notUptodate, t[0]) - toUpgrade = append(toUpgrade, t[1]) + formulas = append(formulas, t) } }() @@ -433,50 +509,24 @@ func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAsse close(results) wg2.Wait() - // resolve to packages from the db to be able to encode correctly - oldPackages, err := s.getList(universe, notUptodate) - if err != nil { - return nil, nil, errors.Wrap(err, "couldn't get package marked for removal from universe") - } - - updates, err := s.getList(universe, toUpgrade) - if err != nil { - return nil, nil, errors.Wrap(err, "couldn't get package marked for update from universe") - } - - var formulas []bf.Formula - - // Build constraints for the whole defdb - r, err := s.BuildWorld(true) - if err != nil { - return nil, nil, errors.Wrap(err, "couldn't build world constraints") - } - // Treat removed packages from universe as marked for deletion if dropremoved { - oldPackages = append(oldPackages, removed...) - } - // SAT encode the clauses against the world - for _, p := range oldPackages.Unique() { - encodedP, err := p.Encode(universe) - if err != nil { - return nil, nil, errors.Wrap(err, "couldn't encode package") + // SAT encode the clauses against the world + for _, p := range removed { + encodedP, err := p.Encode(universe) + if err != nil { + return nil, nil, errors.Wrap(err, "couldn't encode package") + } + P := bf.Var(encodedP) + formulas = append(formulas, bf.And(bf.Not(P), r)) } - P := bf.Var(encodedP) - formulas = append(formulas, bf.And(bf.Not(P), r)) - } - - for _, p := range updates { - encodedP, err := p.Encode(universe) - if err != nil { - return nil, nil, errors.Wrap(err, "couldn't encode package") - } - P := bf.Var(encodedP) - formulas = append(formulas, bf.And(P, r)) } markedForRemoval := pkg.Packages{} + if len(formulas) == 0 { + return pkg.Packages{}, PackagesAssertions{}, nil + } model := bf.Solve(bf.And(formulas...)) if model == nil { return nil, nil, errors.New("Failed finding a solution") @@ -506,7 +556,6 @@ func (s *Parallel) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAss toUninstall := pkg.Packages{} toInstall := pkg.Packages{} - availableCache := map[string]pkg.Packages{} for _, p := range s.DefinitionDatabase.World() { // Each one, should be expanded @@ -577,7 +626,9 @@ func (s *Parallel) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAss } } } - + if len(toInstall) == 0 { + return toUninstall, PackagesAssertions{}, nil + } r, e := s2.Install(toInstall) return toUninstall, r, e // To that tree, ask to install the versions that should be upgraded, and try to solve @@ -673,7 +724,8 @@ func (s *Parallel) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Pack // BuildFormula builds the main solving formula that is evaluated by the sat Parallel. func (s *Parallel) BuildFormula() (bf.Formula, error) { var formulas []bf.Formula - r, err := s.BuildWorld(false) + + r, err := s.BuildPartialWorld(false) if err != nil { return nil, err } diff --git a/pkg/solver/parallel_test.go b/pkg/solver/parallel_test.go index ad9c2375..5d4cf0d2 100644 --- a/pkg/solver/parallel_test.go +++ b/pkg/solver/parallel_test.go @@ -108,10 +108,10 @@ var _ = Describe("Parallel", func() { Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: true})) - Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false})) - Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false})) + Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true})) + Expect(solution).ToNot(ContainElement(PackageAssert{Package: D, Value: true})) - Expect(len(solution)).To(Equal(5)) + Expect(len(solution)).To(Equal(3)) }) It("Solves correctly if the selected package to install has requirements", func() { diff --git a/pkg/solver/resolver_test.go b/pkg/solver/resolver_test.go index 793c47a5..49c12309 100644 --- a/pkg/solver/resolver_test.go +++ b/pkg/solver/resolver_test.go @@ -79,13 +79,13 @@ var _ = Describe("Resolver", func() { solution, err := s.Install([]pkg.Package{D, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn Expect(err).ToNot(HaveOccurred()) - Expect(len(solution)).To(Equal(6)) + Expect(len(solution)).To(Equal(3)) - Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false})) - Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false})) + Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true})) + Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true})) - Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false})) + Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true})) }) @@ -112,12 +112,12 @@ var _ = Describe("Resolver", func() { solution, err := s.Install([]pkg.Package{A, D}) Expect(err).ToNot(HaveOccurred()) - Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false})) - Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false})) + Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true})) + Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true})) - Expect(len(solution)).To(Equal(4)) + Expect(len(solution)).To(Equal(2)) }) It("will find out that we can install D and F by ignoring E and A", func() { @@ -142,13 +142,13 @@ var _ = Describe("Resolver", func() { solution, err := s.Install([]pkg.Package{A, D, E, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn Expect(err).ToNot(HaveOccurred()) - Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false})) - Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false})) + Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true})) + Expect(solution).ToNot(ContainElement(PackageAssert{Package: B, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) // Was already installed Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true})) - Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false})) + Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true})) - Expect(len(solution)).To(Equal(6)) + Expect(len(solution)).To(Equal(3)) }) }) diff --git a/pkg/solver/solver.go b/pkg/solver/solver.go index b2dbcd84..de286a6e 100644 --- a/pkg/solver/solver.go +++ b/pkg/solver/solver.go @@ -131,7 +131,15 @@ func (s *Solver) noRulesInstalled() bool { func (s *Solver) BuildInstalled() (bf.Formula, error) { var formulas []bf.Formula + var packages pkg.Packages for _, p := range s.Installed() { + packages = append(packages, p) + for _, dep := range p.Related(s.DefinitionDatabase) { + packages = append(packages, dep) + } + } + + for _, p := range packages { solvable, err := p.BuildFormula(s.DefinitionDatabase, s.SolverDatabase) if err != nil { return nil, err @@ -159,6 +167,7 @@ func (s *Solver) BuildWorld(includeInstalled bool) (bf.Formula, error) { } for _, p := range s.World() { + solvable, err := p.BuildFormula(s.DefinitionDatabase, s.SolverDatabase) if err != nil { return nil, err @@ -168,6 +177,44 @@ func (s *Solver) BuildWorld(includeInstalled bool) (bf.Formula, error) { return bf.And(formulas...), nil } +// BuildWorld builds the formula which olds the requirements from the package definitions +// which are available (global state) +func (s *Solver) BuildPartialWorld(includeInstalled bool) (bf.Formula, error) { + var formulas []bf.Formula + // NOTE: This block shouldf be enabled in case of very old systems with outdated world sets + if includeInstalled { + solvable, err := s.BuildInstalled() + if err != nil { + return nil, err + } + //f = bf.And(f, solvable) + formulas = append(formulas, solvable) + } + + var packages pkg.Packages + for _, p := range s.Wanted { + // packages = append(packages, p) + for _, dep := range p.Related(s.DefinitionDatabase) { + packages = append(packages, dep) + } + + } + + for _, p := range packages { + solvable, err := p.BuildFormula(s.DefinitionDatabase, s.SolverDatabase) + if err != nil { + return nil, err + } + formulas = append(formulas, solvable...) + } + + if len(formulas) != 0 { + return bf.And(formulas...), nil + } + return bf.True, nil + +} + func (s *Solver) getList(db pkg.PackageDatabase, lsp pkg.Packages) (pkg.Packages, error) { var ls pkg.Packages @@ -377,17 +424,6 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert } } - // resolve to packages from the db to be able to encode correctly - oldPackages, err := s.getList(universe, notUptodate) - if err != nil { - return nil, nil, errors.Wrap(err, "couldn't get package marked for removal from universe") - } - - updates, err := s.getList(universe, toUpgrade) - if err != nil { - return nil, nil, errors.Wrap(err, "couldn't get package marked for update from universe") - } - var formulas []bf.Formula // Build constraints for the whole defdb @@ -398,11 +434,11 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert // Treat removed packages from universe as marked for deletion if dropremoved { - oldPackages = append(oldPackages, removed...) + notUptodate = append(notUptodate, removed...) } // SAT encode the clauses against the world - for _, p := range oldPackages.Unique() { + for _, p := range notUptodate.Unique() { encodedP, err := p.Encode(universe) if err != nil { return nil, nil, errors.Wrap(err, "couldn't encode package") @@ -411,7 +447,7 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert formulas = append(formulas, bf.And(bf.Not(P), r)) } - for _, p := range updates { + for _, p := range toUpgrade { encodedP, err := p.Encode(universe) if err != nil { return nil, nil, errors.Wrap(err, "couldn't encode package") @@ -421,6 +457,10 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert } markedForRemoval := pkg.Packages{} + + if len(formulas) == 0 { + return pkg.Packages{}, PackagesAssertions{}, nil + } model := bf.Solve(bf.And(formulas...)) if model == nil { return nil, nil, errors.New("Failed finding a solution") @@ -477,7 +517,6 @@ func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAsser ass = append(ass, PackageAssert{Package: i.(*pkg.DefaultPackage), Value: true}) } } - // Then try to uninstall the versions in the system, and store that tree for _, p := range toUninstall { r, err := s.Uninstall(p, checkconflicts, false) @@ -491,7 +530,9 @@ func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAsser } } } - + if len(toInstall) == 0 { + return toUninstall, PackagesAssertions{}, nil + } r, e := s2.Install(toInstall) return toUninstall, r, e // To that tree, ask to install the versions that should be upgraded, and try to solve @@ -587,16 +628,18 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag // BuildFormula builds the main solving formula that is evaluated by the sat solver. func (s *Solver) BuildFormula() (bf.Formula, error) { var formulas []bf.Formula - r, err := s.BuildWorld(false) + r, err := s.BuildPartialWorld(false) if err != nil { return nil, err } + for _, wanted := range s.Wanted { encodedW, err := wanted.Encode(s.SolverDatabase) if err != nil { return nil, err } W := bf.Var(encodedW) + // allW = append(allW, W) installedWorld := s.Installed() //TODO:Optimize if len(installedWorld) == 0 { @@ -614,8 +657,8 @@ func (s *Solver) BuildFormula() (bf.Formula, error) { } } - formulas = append(formulas, r) + formulas = append(formulas, r) return bf.And(formulas...), nil } diff --git a/pkg/solver/solver_test.go b/pkg/solver/solver_test.go index 04856a38..c249f528 100644 --- a/pkg/solver/solver_test.go +++ b/pkg/solver/solver_test.go @@ -108,10 +108,10 @@ var _ = Describe("Solver", func() { Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: true})) - Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false})) - Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false})) + // Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false})) + //Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: false})) - Expect(len(solution)).To(Equal(5)) + Expect(len(solution)).To(Equal(3)) }) It("Solves correctly if the selected package to install has requirements", func() { diff --git a/pkg/tree/recipes_test.go b/pkg/tree/recipes_test.go index cf0c768d..dca538e0 100644 --- a/pkg/tree/recipes_test.go +++ b/pkg/tree/recipes_test.go @@ -120,7 +120,7 @@ var _ = Describe("Recipe", func() { s := solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false), tree, tree) solution, err := s.Install([]pkg.Package{pack}) Expect(err).ToNot(HaveOccurred()) - Expect(len(solution)).To(Equal(33)) + Expect(len(solution)).To(Equal(14)) var allSol string for _, sol := range solution { diff --git a/pkg/tree/tree_test.go b/pkg/tree/tree_test.go index 622bbc87..b3d03be8 100644 --- a/pkg/tree/tree_test.go +++ b/pkg/tree/tree_test.go @@ -69,31 +69,25 @@ var _ = Describe("Tree", func() { solution, err = solution.Order(generalRecipe.GetDatabase(), pack.GetFingerPrint()) Expect(err).ToNot(HaveOccurred()) - Expect(solution[0].Package.GetName()).To(Equal("a")) - Expect(solution[0].Value).To(BeFalse()) + Expect(solution[0].Package.GetName()).To(Equal("b")) + Expect(solution[0].Value).To(BeTrue()) - Expect(solution[1].Package.GetName()).To(Equal("b")) + Expect(solution[1].Package.GetName()).To(Equal("c")) Expect(solution[1].Value).To(BeTrue()) - Expect(solution[2].Package.GetName()).To(Equal("c")) + Expect(solution[2].Package.GetName()).To(Equal("d")) Expect(solution[2].Value).To(BeTrue()) - - Expect(solution[3].Package.GetName()).To(Equal("d")) - Expect(solution[3].Value).To(BeTrue()) - Expect(len(solution)).To(Equal(4)) + Expect(len(solution)).To(Equal(3)) newsolution := solution.Drop(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"}) - Expect(len(newsolution)).To(Equal(3)) + Expect(len(newsolution)).To(Equal(2)) - Expect(newsolution[0].Package.GetName()).To(Equal("a")) - Expect(newsolution[0].Value).To(BeFalse()) + Expect(newsolution[0].Package.GetName()).To(Equal("b")) + Expect(newsolution[0].Value).To(BeTrue()) - Expect(newsolution[1].Package.GetName()).To(Equal("b")) + Expect(newsolution[1].Package.GetName()).To(Equal("c")) Expect(newsolution[1].Value).To(BeTrue()) - Expect(newsolution[2].Package.GetName()).To(Equal("c")) - Expect(newsolution[2].Value).To(BeTrue()) - } }) }) @@ -146,11 +140,11 @@ var _ = Describe("Tree", func() { base, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "base", Category: "layer", Version: "0.2"}) Expect(err).ToNot(HaveOccurred()) - Expect(solution).To(ContainElement(solver.PackageAssert{Package: pack.(*pkg.DefaultPackage), Value: false})) + Expect(solution).ToNot(ContainElement(solver.PackageAssert{Package: pack.(*pkg.DefaultPackage), Value: true})) Expect(solution).To(ContainElement(solver.PackageAssert{Package: D.(*pkg.DefaultPackage), Value: true})) - Expect(solution).To(ContainElement(solver.PackageAssert{Package: extra.(*pkg.DefaultPackage), Value: false})) - Expect(solution).To(ContainElement(solver.PackageAssert{Package: base.(*pkg.DefaultPackage), Value: false})) - Expect(len(solution)).To(Equal(6)) + Expect(solution).ToNot(ContainElement(solver.PackageAssert{Package: extra.(*pkg.DefaultPackage), Value: true})) + Expect(solution).ToNot(ContainElement(solver.PackageAssert{Package: base.(*pkg.DefaultPackage), Value: true})) + Expect(len(solution)).To(Equal(3)) } }) })