Prepare for 0.6.4 tag

This enables smoother upgrades, and uses cache to pre-download
packages before uninstall/install

Allows also from cli to download deps only

cmd/install.go

@@ -41,6 +41,7 @@ var installCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
 		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("download-only", cmd.Flags().Lookup("download-only"))
 
 	},
 	Long: `Install packages in parallel`,
@@ -73,7 +74,7 @@ var installCmd = &cobra.Command{
 		force := LuetCfg.Viper.GetBool("force")
 		nodeps := LuetCfg.Viper.GetBool("nodeps")
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
-
+		downloadOnly := LuetCfg.Viper.GetBool("download-only")
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
@@ -87,6 +88,7 @@ var installCmd = &cobra.Command{
 			NoDeps:                      nodeps,
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
 		})
 		inst.Repositories(repos)
 
@@ -97,7 +99,7 @@ var installCmd = &cobra.Command{
 			systemDB = pkg.NewInMemoryDatabase(true)
 		}
 		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
-		err := inst.Install(toInstall, system)
+		err := inst.Install(toInstall, system, downloadOnly)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
@@ -118,6 +120,7 @@ func init() {
 	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
 	installCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
 	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	installCmd.Flags().Bool("download-only", false, "Download dependencies only")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/root.go

@@ -37,7 +37,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.6.3"
+	LuetCLIVersion = "0.6.4"
 	LuetEnvPrefix  = "LUET"
 )
 

pkg/installer/installer.go

@@ -21,6 +21,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 
 	"github.com/ghodss/yaml"
@@ -41,6 +42,7 @@ type LuetInstallerOptions struct {
 	NoDeps                      bool
 	OnlyDeps                    bool
 	Force                       bool
+	PreserveSystemEssentialData bool
 }
 
 type LuetInstaller struct {
@@ -115,31 +117,6 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 		return errors.Wrap(err, "Failed solving solution for upgrade")
 	}
 
-	// We don't want any conflict with the installed to raise during the upgrade.
-	// In this way we both force uninstalls and we avoid to check with conflicts
-	// against the current system state which is pending to deletion
-	// E.g. you can't check for conflicts for an upgrade of a new version of A
-	// if the old A results installed in the system. This is due to the fact that
-	// now the solver enforces the constraints and explictly denies two packages
-	// of the same version installed.
-	forced := false
-	if l.Options.Force {
-		forced = true
-	}
-	l.Options.Force = true
-
-	for _, u := range uninstall {
-		Info(":package: Marked for deletion", u.HumanReadableString())
-
-		err := l.Uninstall(u, s)
-		if err != nil && !l.Options.Force {
-			Error("Failed uninstall for ", u.HumanReadableString())
-			return errors.Wrap(err, "uninstalling "+u.HumanReadableString())
-		}
-
-	}
-	l.Options.Force = forced
-
 	toInstall := []pkg.Package{}
 	for _, assertion := range solution {
 		if assertion.Value {
@@ -147,7 +124,7 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 		}
 	}
 
-	return l.Install(toInstall, s)
+	return l.swap(syncedRepos, uninstall, toInstall, s)
 }
 
 func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
@@ -172,7 +149,62 @@ func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
 	return syncedRepos, nil
 }
 
-func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
+func (l *LuetInstaller) Swap(toRemove []pkg.Package, toInstall []pkg.Package, s *System) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+	return l.swap(syncedRepos, toRemove, toInstall, s)
+}
+
+func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove []pkg.Package, toInstall []pkg.Package, s *System) error {
+
+	// First match packages against repositories by priority
+	allRepos := pkg.NewInMemoryDatabase(false)
+	syncedRepos.SyncDatabase(allRepos)
+	toInstall = syncedRepos.ResolveSelectors(toInstall)
+
+	if err := l.install(syncedRepos, toInstall, s, true); err != nil {
+		return errors.Wrap(err, "Pre-downloading packages")
+	}
+
+	// We don't want any conflict with the installed to raise during the upgrade.
+	// In this way we both force uninstalls and we avoid to check with conflicts
+	// against the current system state which is pending to deletion
+	// E.g. you can't check for conflicts for an upgrade of a new version of A
+	// if the old A results installed in the system. This is due to the fact that
+	// now the solver enforces the constraints and explictly denies two packages
+	// of the same version installed.
+	forced := false
+	if l.Options.Force {
+		forced = true
+	}
+	l.Options.Force = true
+
+	for _, u := range toRemove {
+		Info(":package: Marked for deletion", u.HumanReadableString())
+
+		err := l.Uninstall(u, s)
+		if err != nil && !l.Options.Force {
+			Error("Failed uninstall for ", u.HumanReadableString())
+			return errors.Wrap(err, "uninstalling "+u.HumanReadableString())
+		}
+
+	}
+	l.Options.Force = forced
+
+	return l.install(syncedRepos, toInstall, s, false)
+}
+
+func (l *LuetInstaller) Install(cp []pkg.Package, s *System, downloadOnly bool) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+	return l.install(syncedRepos, cp, s, downloadOnly)
+}
+
+func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *System, downloadOnly bool) error {
 	var p []pkg.Package
 
 	// Check if the package is installed first
@@ -195,10 +227,6 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 	}
 	// First get metas from all repos (and decodes trees)
 
-	syncedRepos, err := l.SyncRepositories(true)
-	if err != nil {
-		return err
-	}
 	// First match packages against repositories by priority
 	//	matches := syncedRepos.PackageMatches(p)
 
@@ -208,7 +236,7 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 	p = syncedRepos.ResolveSelectors(p)
 	toInstall := map[string]ArtifactMatch{}
 	var packagesToInstall []pkg.Package
-
+	var err error
 	var solution solver.PackagesAssertions
 
 	if !l.Options.NoDeps {
@@ -254,9 +282,12 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 	all := make(chan ArtifactMatch)
 
 	var wg = new(sync.WaitGroup)
+
+	if !downloadOnly {
+		// Download first
 		for i := 0; i < l.Options.Concurrency; i++ {
 			wg.Add(1)
-		go l.installerWorker(i, wg, all, s)
+			go l.installerWorker(i, wg, all, s, true)
 		}
 
 		for _, c := range toInstall {
@@ -265,6 +296,38 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 		close(all)
 		wg.Wait()
 
+		all = make(chan ArtifactMatch)
+
+		wg = new(sync.WaitGroup)
+
+		// Do the real install
+		for i := 0; i < l.Options.Concurrency; i++ {
+			wg.Add(1)
+			go l.installerWorker(i, wg, all, s, false)
+		}
+
+		for _, c := range toInstall {
+			all <- c
+		}
+		close(all)
+		wg.Wait()
+	} else {
+		for i := 0; i < l.Options.Concurrency; i++ {
+			wg.Add(1)
+			go l.installerWorker(i, wg, all, s, downloadOnly)
+		}
+
+		for _, c := range toInstall {
+			all <- c
+		}
+		close(all)
+		wg.Wait()
+	}
+
+	if downloadOnly {
+		return nil
+	}
+
 	for _, c := range toInstall {
 		// Annotate to the system that the package was installed
 		_, err := s.Database.CreatePackage(c.Package)
@@ -318,7 +381,7 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 
 }
 
-func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
+func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System, downloadOnly bool) error {
 
 	artifact, err := a.Repository.Client().DownloadArtifact(a.Artifact)
 	if err != nil {
@@ -329,6 +392,9 @@ func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
 	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Artifact integrity check failure")
 	}
+	if downloadOnly {
+		return nil
+	}
 
 	files, err := artifact.FileList()
 	if err != nil && !l.Options.Force {
@@ -345,18 +411,20 @@ func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
 	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.Package.GetFingerPrint(), Files: files})
 }
 
-func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System) error {
+func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System, downloadOnly bool) error {
 	defer wg.Done()
 
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
-		err := l.installPackage(p, s)
+		err := l.installPackage(p, s, downloadOnly)
 		if err != nil && !l.Options.Force {
 			//TODO: Uninstall, rollback.
 			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
 			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
 		}
-		if err == nil {
+		if err == nil && downloadOnly {
+			Info(":package: ", p.Package.HumanReadableString(), "downloaded")
+		} else if err == nil {
 			Info(":package: ", p.Package.HumanReadableString(), "installed")
 		} else if err != nil && l.Options.Force {
 			Info(":package: ", p.Package.HumanReadableString(), "installed with failures (force install)")
@@ -375,7 +443,15 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 	// Remove from target
 	for _, f := range files {
 		target := filepath.Join(s.Target, f)
-		Info("Removing", target)
+		Debug("Removing", target)
+
+		if l.Options.PreserveSystemEssentialData &&
+			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) ||
+			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath()) {
+			Warning("Preserve ", f, " which is required by luet ( you have to delete it manually if you really need to)")
+			continue
+		}
+
 		err := os.Remove(target)
 		if err != nil {
 			Warning("Failed removing file (not present in the system target ?)", target)

pkg/installer/installer_test.go

@@ -113,7 +113,7 @@ urls:
 			Expect(repo.GetType()).To(Equal("disk"))
 			systemDB := pkg.NewInMemoryDatabase(false)
 			system := &System{Database: systemDB, Target: fakeroot}
-			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
@@ -231,7 +231,7 @@ urls:
 
 			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
 			system := &System{Database: systemDB, Target: fakeroot}
-			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
@@ -339,7 +339,7 @@ urls:
 
 			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
 			system := &System{Database: systemDB, Target: fakeroot}
-			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
@@ -454,7 +454,7 @@ urls:
 
 			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
 			system := &System{Database: systemDB, Target: fakeroot}
-			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())

pkg/installer/interface.go

@@ -23,11 +23,12 @@ import (
 )
 
 type Installer interface {
-	Install([]pkg.Package, *System) error
+	Install([]pkg.Package, *System, bool) error
 	Uninstall(pkg.Package, *System) error
 	Upgrade(s *System) error
 	Repositories([]Repository)
 	SyncRepositories(bool) (Repositories, error)
+	Swap([]pkg.Package, []pkg.Package, *System) error
 }
 
 type Client interface {