Tag 0.9.13

Similarly, we want just to consider what is being uninstalled and the
new rules of the package that is going to be upgraded

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.9.10"
+	LuetCLIVersion = "0.9.13"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/upgrade.go

@@ -123,7 +123,7 @@ func init() {
 	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
 	upgradeCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
-	upgradeCmd.Flags().Bool("full", true, "Attempts to remove as much packages as possible which aren't required (slow)")
+	upgradeCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
 	upgradeCmd.Flags().Bool("universe", false, "Use ONLY the SAT solver to compute upgrades (experimental)")
 	upgradeCmd.Flags().Bool("clean", false, "Try to drop removed packages (experimental, only when --universe is enabled)")
 	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")

pkg/compiler/compiler.go

@@ -16,6 +16,7 @@
 package compiler
 
 import (
+	"archive/tar"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -345,6 +346,10 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
 	}
 
+	if len(p.GetPreBuildSteps()) == 0 {
+		buildertaggedImage = image
+	}
+
 	// Then we write the step image, which uses the builder one
 	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")); err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
@@ -392,11 +397,15 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 		}
 		return nil
 	}
-
-	Info(pkgTag, ":whale: Generating 'builder' image from", image, "as", buildertaggedImage, "with prelude steps")
-	if err := buildAndPush(builderOpts); err != nil {
-		return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
+	if len(p.GetPreBuildSteps()) != 0 {
+		Info(pkgTag, ":whale: Generating 'builder' image from", image, "as", buildertaggedImage, "with prelude steps")
+		if err := buildAndPush(builderOpts); err != nil {
+			return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
+		}
 	}
+
+	// Even if we might not have any steps to build, we do that so we can tag the image used in this moment and use that to cache it in a registry, or in the system.
+	// acting as a docker tag.
 	Info(pkgTag, ":whale: Generating 'package' image from", buildertaggedImage, "as", packageImage, "with build steps")
 	if err := buildAndPush(runnerOpts); err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
@@ -420,6 +429,23 @@ func (cs *LuetCompiler) genArtifact(p CompilationSpec, builderOpts, runnerOpts C
 		unpack = true
 	}
 
+	if len(p.BuildSteps()) == 0 && len(p.GetPreBuildSteps()) == 0 && !unpack {
+		fakePackage := p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")
+		// We can't generate delta in this case. It implies the package is a virtual, and nothing has to be done really
+
+		file, err := os.Create(fakePackage)
+		if err != nil {
+			return nil, errors.Wrap(err, "Failed creating virtual package")
+		}
+		defer file.Close()
+		tw := tar.NewWriter(file)
+		defer tw.Close()
+
+		artifact := NewPackageArtifact(fakePackage)
+		artifact.SetCompressionType(cs.CompressionType)
+		return artifact, nil
+	}
+
 	// prepare folder content of the image with the package compiled inside
 	if err := cs.Backend.ExportImage(runnerOpts); err != nil {
 		return nil, errors.Wrap(err, "Failed exporting image")

pkg/compiler/spec.go

@@ -247,6 +247,19 @@ ENV PACKAGE_NAME=` + cs.Package.GetName() + `
 ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
 ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
 
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			//var file string
+			// if helpers.IsValidUrl(s) {
+			// 	file = s
+			// } else {
+			// 	file = cs.Rel(s)
+			// }
+			spec = spec + `
+ADD ` + s + ` /luetbuild/`
+		}
+	}
+
 	for _, s := range cs.Env {
 		spec = spec + `
 ENV ` + s

pkg/compiler/spec_test.go

@@ -173,6 +173,8 @@ WORKDIR /luetbuild
 ENV PACKAGE_NAME=a
 ENV PACKAGE_VERSION=1.0
 ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
 ENV test=1
 RUN echo foo > /test
 RUN echo bar > /test2`))

pkg/helpers/file.go

@@ -19,6 +19,8 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
+	"strings"
 	"time"
 
 	copy "github.com/otiai10/copy"
@@ -41,6 +43,8 @@ func OrderFiles(target string, files []string) ([]string, []string) {
 		}
 	}
 
+	dirs := []string{}
+
 	for _, f := range files {
 		target := filepath.Join(target, f)
 		fi, err := os.Lstat(target)
@@ -48,11 +52,16 @@ func OrderFiles(target string, files []string) ([]string, []string) {
 			continue
 		}
 		if m := fi.Mode(); m.IsDir() {
-			newFiles = append(newFiles, f)
+			dirs = append(dirs, f)
 		}
 	}
 
-	return newFiles, notPresent
+	// Compare how many sub paths there are, and push at the end the ones that have less subpaths
+	sort.Slice(dirs, func(i, j int) bool {
+		return len(strings.Split(dirs[i], string(os.PathSeparator))) > len(strings.Split(dirs[j], string(os.PathSeparator)))
+	})
+
+	return append(newFiles, dirs...), notPresent
 }
 
 func ListDir(dir string) ([]string, error) {

pkg/helpers/file_test.go

@@ -60,5 +60,27 @@ var _ = Describe("Helpers", func() {
 			Expect(ordered).To(Equal([]string{"baz", "bar/foo", "foo", "baz2/foo", "bar", "baz2"}))
 			Expect(notExisting).To(Equal([]string{"notexisting"}))
 		})
+
+		It("orders correctly when there are folders with folders", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			err = os.MkdirAll(filepath.Join(testDir, "bar"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+			err = os.MkdirAll(filepath.Join(testDir, "foo"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "bar"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+			err = os.MkdirAll(filepath.Join(testDir, "foo", "baz", "fa"), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			ordered, _ := OrderFiles(testDir, []string{"foo", "foo/bar", "bar", "foo/baz/fa", "foo/baz"})
+			Expect(ordered).To(Equal([]string{"foo/baz/fa", "foo/bar", "foo/baz", "foo", "bar"}))
+		})
 	})
 })

pkg/installer/installer.go

@@ -84,7 +84,7 @@ func (l *LuetInstaller) computeUpgrade(syncedRepos Repositories, s *System) (pkg
 			return uninstall, toInstall, errors.Wrap(err, "Failed solving solution for upgrade")
 		}
 	} else {
-		uninstall, solution, err = solv.Upgrade(!l.Options.FullUninstall, l.Options.NoDeps)
+		uninstall, solution, err = solv.Upgrade(l.Options.FullUninstall, true)
 		if err != nil {
 			return uninstall, toInstall, errors.Wrap(err, "Failed solving solution for upgrade")
 		}
@@ -241,10 +241,6 @@ func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, to
 	syncedRepos.SyncDatabase(allRepos)
 	toInstall = syncedRepos.ResolveSelectors(toInstall)
 
-	if err := l.download(syncedRepos, toInstall); err != nil {
-		return errors.Wrap(err, "Pre-downloading packages")
-	}
-
 	// We don't want any conflict with the installed to raise during the upgrade.
 	// In this way we both force uninstalls and we avoid to check with conflicts
 	// against the current system state which is pending to deletion
@@ -253,8 +249,43 @@ func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, to
 	// now the solver enforces the constraints and explictly denies two packages
 	// of the same version installed.
 	forced := l.Options.Force
-
+	nodeps := l.Options.NoDeps
 	l.Options.Force = true
+	l.Options.NoDeps = true
+
+	// First check what would have been done
+	installedtmp := pkg.NewInMemoryDatabase(false)
+
+	for _, i := range s.Database.World() {
+		_, err := installedtmp.CreatePackage(i)
+		if err != nil {
+			return errors.Wrap(err, "Failed create temporary in-memory db")
+		}
+	}
+	systemAfterChanges := &System{Database: installedtmp}
+
+	for _, u := range toRemove {
+		packs, err := l.computeUninstall(u, systemAfterChanges)
+		if err != nil && !l.Options.Force {
+			Error("Failed computing uninstall for ", u.HumanReadableString())
+			return errors.Wrap(err, "computing uninstall "+u.HumanReadableString())
+		}
+		for _, p := range packs {
+			err = systemAfterChanges.Database.RemovePackage(p)
+			if err != nil {
+				return errors.Wrap(err, "Failed removing package from database")
+			}
+		}
+	}
+
+	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, toInstall, systemAfterChanges)
+	if err != nil {
+		return errors.Wrap(err, "computing installation")
+	}
+
+	if err := l.download(syncedRepos, match); err != nil {
+		return errors.Wrap(err, "Pre-downloading packages")
+	}
 
 	for _, u := range toRemove {
 		err := l.Uninstall(u, s)
@@ -263,13 +294,9 @@ func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, to
 			return errors.Wrap(err, "uninstalling "+u.HumanReadableString())
 		}
 	}
+
 	l.Options.Force = forced
-
-	match, packages, assertions, allRepos, err := l.computeInstall(syncedRepos, toInstall, s)
-	if err != nil {
-		return errors.Wrap(err, "computing installation")
-	}
-
+	l.Options.NoDeps = nodeps
 	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
 }
 
@@ -325,32 +352,8 @@ func (l *LuetInstaller) Install(cp pkg.Packages, s *System) error {
 	return l.install(syncedRepos, match, packages, assertions, allRepos, s)
 }
 
-func (l *LuetInstaller) download(syncedRepos Repositories, cp pkg.Packages) error {
-	toDownload := map[string]ArtifactMatch{}
+func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string]ArtifactMatch) error {
 
-	// FIXME: This can be optimized. We don't need to re-match this to the repository
-	// But we could just do it once
-
-	// Gathers things to download
-	for _, currentPack := range cp {
-		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
-		if len(matches) == 0 {
-			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
-		}
-	A:
-		for _, artefact := range matches[0].Repo.GetIndex() {
-			if artefact.GetCompileSpec().GetPackage() == nil {
-				return errors.New("Package in compilespec empty")
-
-			}
-			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
-
-				toDownload[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
-
-				break A
-			}
-		}
-	}
 	// Download packages into cache in parallel.
 	all := make(chan ArtifactMatch)
 
@@ -508,25 +511,13 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages
 
 func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]ArtifactMatch, p pkg.Packages, solution solver.PackagesAssertions, allRepos pkg.PackageDatabase, s *System) error {
 	// Install packages into rootfs in parallel.
+	if err := l.download(syncedRepos, toInstall); err != nil {
+		return errors.Wrap(err, "Downloading packages")
+	}
+
 	all := make(chan ArtifactMatch)
 
-	var wg = new(sync.WaitGroup)
-
-	// Download first
-	for i := 0; i < l.Options.Concurrency; i++ {
-		wg.Add(1)
-		go l.downloadWorker(i, wg, all)
-	}
-
-	for _, c := range toInstall {
-		all <- c
-	}
-	close(all)
-	wg.Wait()
-
-	all = make(chan ArtifactMatch)
-
-	wg = new(sync.WaitGroup)
+	wg := new(sync.WaitGroup)
 
 	// Do the real install
 	for i := 0; i < l.Options.Concurrency; i++ {

pkg/solver/parallel.go

@@ -76,7 +76,7 @@ func (s *Parallel) noRulesInstalled() bool {
 	return true
 }
 
-func (s *Parallel) buildParallelFormula(formulas []bf.Formula, packages pkg.Packages) (bf.Formula, error) {
+func (s *Parallel) buildParallelFormula(db pkg.PackageDatabase, formulas []bf.Formula, packages pkg.Packages) (bf.Formula, error) {
 	var wg = new(sync.WaitGroup)
 	var wg2 = new(sync.WaitGroup)
 
@@ -87,7 +87,7 @@ func (s *Parallel) buildParallelFormula(formulas []bf.Formula, packages pkg.Pack
 		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
 			defer wg.Done()
 			for p := range c {
-				solvable, err := p.BuildFormula(s.DefinitionDatabase, s.ParallelDatabase)
+				solvable, err := p.BuildFormula(db, s.ParallelDatabase)
 				if err != nil {
 					panic(err)
 				}
@@ -126,13 +126,13 @@ func (s *Parallel) BuildInstalled() (bf.Formula, error) {
 	var packages pkg.Packages
 	for _, p := range s.Installed() {
 		packages = append(packages, p)
-		for _, dep := range p.Related(s.DefinitionDatabase) {
+		for _, dep := range p.Related(s.InstalledDatabase) {
 			packages = append(packages, dep)
 		}
 
 	}
 
-	return s.buildParallelFormula(formulas, packages)
+	return s.buildParallelFormula(s.InstalledDatabase, formulas, packages)
 }
 
 // BuildWorld builds the formula which olds the requirements from the package definitions
@@ -148,7 +148,7 @@ func (s *Parallel) BuildWorld(includeInstalled bool) (bf.Formula, error) {
 		//f = bf.And(f, solvable)
 		formulas = append(formulas, solvable)
 	}
-	return s.buildParallelFormula(formulas, s.World())
+	return s.buildParallelFormula(s.DefinitionDatabase, formulas, s.World())
 }
 
 // BuildWorld builds the formula which olds the requirements from the package definitions
@@ -200,7 +200,7 @@ func (s *Parallel) BuildPartialWorld(includeInstalled bool) (bf.Formula, error)
 	close(results)
 	wg2.Wait()
 
-	return s.buildParallelFormula(formulas, packages)
+	return s.buildParallelFormula(s.DefinitionDatabase, formulas, packages)
 
 	//return s.buildParallelFormula(formulas, s.World())
 }
@@ -471,23 +471,22 @@ func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAsse
 		go func(wg *sync.WaitGroup, c <-chan pkg.Package) {
 			defer wg.Done()
 			for p := range c {
-				available, err := universe.FindPackageVersions(p)
-				if err != nil {
-					removed = append(removed, p) /// FIXME: Racy
-				}
-				if len(available) == 0 {
+				available, err := s.DefinitionDatabase.FindPackageVersions(p)
+				if len(available) == 0 || err != nil {
+					removed = append(removed, p)
 					continue
 				}
 
 				bestmatch := available.Best(nil)
 				// Found a better version available
 				if !bestmatch.Matches(p) {
-					encodedP, _ := p.Encode(universe)
-					P := bf.Var(encodedP)
-					results <- bf.And(bf.Not(P), r)
-					encodedP, _ = bestmatch.Encode(universe)
-					P = bf.Var(encodedP)
-					results <- bf.And(P, r)
+					oldP, _ := p.Encode(universe)
+					toreplaceP := bf.Var(oldP)
+					best, _ := bestmatch.Encode(universe)
+					toUpgrade := bf.Var(best)
+
+					solvablenew, _ := bestmatch.BuildFormula(s.DefinitionDatabase, s.ParallelDatabase)
+					results <- bf.And(bf.Not(toreplaceP), bf.And(append(solvablenew, toUpgrade)...))
 				}
 			}
 		}(wg, all)
@@ -514,8 +513,7 @@ func (s *Parallel) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAsse
 	// Treat removed packages from universe as marked for deletion
 	if dropremoved {
 
-		// SAT encode the clauses against the world
-		for _, p := range removed {
+		for _, p := range removed.Unique() {
 			encodedP, err := p.Encode(universe)
 			if err != nil {
 				return nil, nil, errors.Wrap(err, "couldn't encode package")
@@ -699,7 +697,7 @@ func (s *Parallel) Uninstall(checkconflicts, full bool, packs ...pkg.Package) (p
 		}
 	}
 
-	s2 := &Parallel{Concurrency: s.Concurrency, InstalledDatabase: pkg.NewInMemoryDatabase(false), DefinitionDatabase: s.DefinitionDatabase, ParallelDatabase: pkg.NewInMemoryDatabase(false)}
+	s2 := &Parallel{Concurrency: s.Concurrency, InstalledDatabase: pkg.NewInMemoryDatabase(false), DefinitionDatabase: s.InstalledDatabase, ParallelDatabase: pkg.NewInMemoryDatabase(false)}
 	s2.SetResolver(s.Resolver)
 	// Get the requirements to install the candidate
 	asserts, err := s2.Install(toRemove)

pkg/solver/parallel_test.go

@@ -671,8 +671,9 @@ var _ = Describe("Parallel", func() {
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
 
-				Expect(len(solution)).To(Equal(1))
+				Expect(len(solution)).To(Equal(2))
 			})
 
 			It("Uninstalls complex packages correctly, even if shared deps are required by system packages", func() {
@@ -1285,12 +1286,172 @@ var _ = Describe("Parallel", func() {
 			Expect(uninstall[0].GetName()).To(Equal("a"))
 			Expect(uninstall[0].GetVersion()).To(Equal("1.1"))
 
-			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: C, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
 
-			Expect(len(solution)).To(Equal(4))
+			Expect(len(solution)).To(Equal(3))
+		})
+
+		It("UpgradeUniverse upgrades correctly", func() {
+
+			D := pkg.NewPackage("d", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D.SetCategory("test")
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{
+				&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"},
+				&pkg.DefaultPackage{Name: "d", Version: ">=1.0", Category: "test"},
+			}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			C1 := pkg.NewPackage("c", "1.6", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			C1.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+
+			B1 := pkg.NewPackage("b", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "c", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			B1.SetCategory("test")
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+			A1 = pkg.NewPackage("a", "1.2", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A1.SetCategory("test")
+
+			for _, p := range []pkg.Package{A1, B, B1, C, C1, D} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B, D} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.UpgradeUniverse(false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(2))
+			Expect(uninstall).To(ContainElement(A))
+			Expect(uninstall).To(ContainElement(B))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: C1, Value: true}))
+
+			Expect(len(solution)).To(Equal(6))
+		})
+
+		It("UpgradeUniverse upgrades correctly", func() {
+
+			D := pkg.NewPackage("d", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D.SetCategory("test")
+			D1 := pkg.NewPackage("d", "1.6", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D1.SetCategory("test")
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{
+				&pkg.DefaultPackage{Name: "c", Version: ">=1.0", Category: "test"},
+			}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+
+			for _, p := range []pkg.Package{A, B, C, D, D1} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B, C, D} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.UpgradeUniverse(false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(1))
+			Expect(uninstall[0].GetName()).To(Equal("d"))
+			Expect(uninstall[0].GetVersion()).To(Equal("1.5"))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: true}))
+
+			Expect(len(solution)).To(Equal(3))
+		})
+
+		It("Upgrade upgrades correctly", func() {
+
+			D := pkg.NewPackage("d", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D.SetCategory("test")
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{
+				&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"},
+				&pkg.DefaultPackage{Name: "d", Version: ">=1.0", Category: "test"},
+			}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			C1 := pkg.NewPackage("c", "1.6", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			C1.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+
+			B1 := pkg.NewPackage("b", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "c", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			B1.SetCategory("test")
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+			A1 = pkg.NewPackage("a", "1.2", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A1.SetCategory("test")
+
+			for _, p := range []pkg.Package{A1, B, B1, C, C1, D} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B, D} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.Upgrade(false, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(2))
+			Expect(uninstall).To(ContainElement(A))
+			Expect(uninstall).To(ContainElement(B))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: C1, Value: true}))
+
+			Expect(len(solution)).To(Equal(6))
+		})
+
+		It("Upgrade upgrades correctly", func() {
+
+			D := pkg.NewPackage("d", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D.SetCategory("test")
+			D1 := pkg.NewPackage("d", "1.6", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			D1.SetCategory("test")
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{
+				&pkg.DefaultPackage{Name: "c", Version: ">=1.0", Category: "test"},
+			}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+
+			for _, p := range []pkg.Package{A, B, C, D, D1} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B, C, D} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.Upgrade(false, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(1))
+			Expect(uninstall[0].GetName()).To(Equal("d"))
+			Expect(uninstall[0].GetVersion()).To(Equal("1.5"))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: true}))
+
+			Expect(len(solution)).To(Equal(5))
 		})
 	})
 })

pkg/solver/solver.go

@@ -134,13 +134,13 @@ func (s *Solver) BuildInstalled() (bf.Formula, error) {
 	var packages pkg.Packages
 	for _, p := range s.Installed() {
 		packages = append(packages, p)
-		for _, dep := range p.Related(s.DefinitionDatabase) {
+		for _, dep := range p.Related(s.InstalledDatabase) {
 			packages = append(packages, dep)
 		}
 	}
 
 	for _, p := range packages {
-		solvable, err := p.BuildFormula(s.DefinitionDatabase, s.SolverDatabase)
+		solvable, err := p.BuildFormula(s.InstalledDatabase, s.SolverDatabase)
 		if err != nil {
 			return nil, err
 		}
@@ -399,6 +399,7 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert
 	notUptodate := pkg.Packages{}
 	removed := pkg.Packages{}
 	toUpgrade := pkg.Packages{}
+	replacements := map[pkg.Package]pkg.Package{}
 
 	// TODO: this is memory expensive, we need to optimize this
 	universe := pkg.NewInMemoryDatabase(false)
@@ -411,11 +412,9 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert
 
 	// Grab all the installed ones, see if they are eligible for update
 	for _, p := range s.Installed() {
-		available, err := universe.FindPackageVersions(p)
-		if err != nil {
+		available, err := s.DefinitionDatabase.FindPackageVersions(p)
+		if len(available) == 0 || err != nil {
 			removed = append(removed, p)
-		}
-		if len(available) == 0 {
 			continue
 		}
 
@@ -424,6 +423,7 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert
 		if !bestmatch.Matches(p) {
 			notUptodate = append(notUptodate, p)
 			toUpgrade = append(toUpgrade, bestmatch)
+			replacements[p] = bestmatch
 		}
 	}
 
@@ -437,28 +437,37 @@ func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssert
 
 	// Treat removed packages from universe as marked for deletion
 	if dropremoved {
-		notUptodate = append(notUptodate, removed...)
+		// SAT encode the clauses against the world
+		for _, p := range removed.Unique() {
+			encodedP, err := p.Encode(universe)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "couldn't encode package")
+			}
+			P := bf.Var(encodedP)
+			formulas = append(formulas, bf.And(bf.Not(P), r))
+		}
 	}
 
-	// SAT encode the clauses against the world
-	for _, p := range notUptodate.Unique() {
-		encodedP, err := p.Encode(universe)
+	for old, new := range replacements {
+		oldP, err := old.Encode(universe)
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "couldn't encode package")
 		}
-		P := bf.Var(encodedP)
-		formulas = append(formulas, bf.And(bf.Not(P), r))
-	}
-
-	for _, p := range toUpgrade {
-		encodedP, err := p.Encode(universe)
+		oldencodedP := bf.Var(oldP)
+		newP, err := new.Encode(universe)
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "couldn't encode package")
 		}
-		P := bf.Var(encodedP)
-		formulas = append(formulas, bf.And(P, r))
+		newEncodedP := bf.Var(newP)
+
+		//solvable, err := old.BuildFormula(s.DefinitionDatabase, s.SolverDatabase)
+		solvablenew, err := new.BuildFormula(s.DefinitionDatabase, s.SolverDatabase)
+
+		formulas = append(formulas, bf.And(bf.Not(oldencodedP), bf.And(append(solvablenew, newEncodedP)...)))
 	}
 
+	//formulas = append(formulas, r)
+
 	markedForRemoval := pkg.Packages{}
 
 	if len(formulas) == 0 {
@@ -607,7 +616,7 @@ func (s *Solver) Uninstall(checkconflicts, full bool, packs ...pkg.Package) (pkg
 		}
 	}
 
-	s2 := NewSolver(Options{Type: SingleCoreSimple}, pkg.NewInMemoryDatabase(false), s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+	s2 := NewSolver(Options{Type: SingleCoreSimple}, pkg.NewInMemoryDatabase(false), s.InstalledDatabase, pkg.NewInMemoryDatabase(false))
 	s2.SetResolver(s.Resolver)
 
 	// Get the requirements to install the candidate

pkg/solver/solver_test.go

@@ -671,8 +671,9 @@ var _ = Describe("Solver", func() {
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
 
-				Expect(len(solution)).To(Equal(1))
+				Expect(len(solution)).To(Equal(2))
 			})
 
 			It("Uninstalls complex packages correctly, even if shared deps are required by system packages", func() {
@@ -1209,6 +1210,8 @@ var _ = Describe("Solver", func() {
 		})
 	})
 	Context("Upgrades", func() {
+		E := pkg.NewPackage("e", "1.5", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		E.SetCategory("test")
 		C := pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
 		C.SetCategory("test")
 		B := pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
@@ -1296,10 +1299,37 @@ var _ = Describe("Solver", func() {
 
 			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
-			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
 
-			Expect(len(solution)).To(Equal(4))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: C, Value: true}))
+			Expect(solution).ToNot(ContainElement(PackageAssert{Package: A, Value: true}))
+
+			Expect(len(solution)).To(Equal(3))
+		})
+
+		It("Suggests to remove untracked packages", func() {
+			for _, p := range []pkg.Package{E} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B, C, E} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.UpgradeUniverse(true)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(uninstall)).To(Equal(3))
+
+			Expect(uninstall).To(ContainElement(B))
+			Expect(uninstall).To(ContainElement(A))
+			Expect(uninstall).To(ContainElement(C))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+
+			Expect(len(solution)).To(Equal(3))
 		})
 	})
 })

tests/fixtures/retrieve-integration/b/build.yaml

@@ -1,6 +1,6 @@
 
 steps:
-- tar xvf a-test-1.0.package.* -C ./
+- ls -liah /a
 - mv a /b
 requires:
 - name: "a"

tests/integration/07_finalizer.sh

@@ -12,7 +12,7 @@ oneTimeTearDown() {
 
 testBuild() {
     mkdir $tmpdir/testbuild
-    luet build --tree "$ROOT_DIR/tests/fixtures/finalizers" --destination $tmpdir/testbuild --compression gzip --all > /dev/null
+    luet build --tree "$ROOT_DIR/tests/fixtures/finalizers" --destination $tmpdir/testbuild --compression gzip --all
     buildst=$?
     assertEquals 'builds successfully' "$buildst" "0"
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/alpine-seed-1.0.package.tar.gz' ]"

tests/integration/11_upgrade_universe.sh

@@ -119,8 +119,8 @@ testUpgrade() {
     assertTrue 'package installed new A' "[ -e '$tmpdir/testrootfs/testlatest' ]"
 
     # It does remove C as well, no other package depends on it.
-    assertContains 'does contain test/c-1.0' "$upgrade" 'test/c-1.0'
-    assertNotContains 'does not attempt to download test/c-1.0' "$upgrade" 'test/c-1.0 downloaded'
+    #assertContains 'does contain test/c-1.0' "$upgrade" 'test/c-1.0'
+    #assertNotContains 'does not attempt to download test/c-1.0' "$upgrade" 'test/c-1.0 downloaded'
 }
 
 # Load shUnit2.