Prepare for 0.6.4 tag

This enables smoother upgrades, and uses cache to pre-download
packages before uninstall/install

Allows also from cli to download deps only

cmd/build.go

@@ -15,18 +15,17 @@
 package cmd
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/config"
+	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	tree "github.com/mudler/luet/pkg/tree"
 
-	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -45,6 +44,15 @@ var buildCmd = &cobra.Command{
 		viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
 		viper.BindPFlag("all", cmd.Flags().Lookup("all"))
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+
+		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
+		viper.BindPFlag("push", cmd.Flags().Lookup("push"))
+		viper.BindPFlag("pull", cmd.Flags().Lookup("pull"))
+		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
+
+		LuetCfg.Viper.BindPFlag("keep-exported-images", cmd.Flags().Lookup("keep-exported-images"))
 
 		LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
 		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
@@ -63,6 +71,13 @@ var buildCmd = &cobra.Command{
 		all := viper.GetBool("all")
 		databaseType := viper.GetString("database")
 		compressionType := viper.GetString("compression")
+		imageRepository := viper.GetString("image-repository")
+		push := viper.GetBool("push")
+		pull := viper.GetBool("pull")
+		keepImages := viper.GetBool("keep-images")
+		nodeps := viper.GetBool("nodeps")
+		onlydeps := viper.GetBool("onlydeps")
+		keepExportedImages := viper.GetBool("keep-exported-images")
 
 		compilerSpecs := compiler.NewLuetCompilationspecs()
 		var compilerBackend compiler.CompilerBackend
@@ -112,33 +127,26 @@ var buildCmd = &cobra.Command{
 
 		opts := compiler.NewDefaultCompilerOptions()
 		opts.SolverOptions = *LuetCfg.GetSolverOptions()
-
+		opts.ImageRepository = imageRepository
 		opts.Clean = clean
+		opts.PullFirst = pull
+		opts.KeepImg = keepImages
+		opts.Push = push
+		opts.OnlyDeps = onlydeps
+		opts.NoDeps = nodeps
+		opts.KeepImageExport = keepExportedImages
+
 		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts)
 		luetCompiler.SetConcurrency(concurrency)
 		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
 		if !all {
 			for _, a := range args {
-				gp, err := _gentoo.ParsePackageStr(a)
+
+				pack, err := helpers.ParsePackageStr(a)
 				if err != nil {
 					Fatal("Invalid package string ", a, ": ", err.Error())
 				}
 
-				if gp.Version == "" {
-					gp.Version = "0"
-					gp.Condition = _gentoo.PkgCondGreaterEqual
-				}
-
-				pack := &pkg.DefaultPackage{
-					Name: gp.Name,
-					Version: fmt.Sprintf("%s%s%s",
-						pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
-						gp.Version,
-						gp.VersionSuffix,
-					),
-					Category: gp.Category,
-					Uri:      make([]string, 0),
-				}
 				spec, err := luetCompiler.FromPackage(pack)
 				if err != nil {
 					Fatal("Error: " + err.Error())
@@ -196,6 +204,13 @@ func init() {
 	buildCmd.Flags().Bool("all", false, "Build all packages in the tree")
 	buildCmd.Flags().String("destination", path, "Destination folder")
 	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
+	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+	buildCmd.Flags().Bool("push", false, "Push images to a hub")
+	buildCmd.Flags().Bool("pull", false, "Pull images from a hub")
+	buildCmd.Flags().Bool("keep-images", true, "Keep built docker images in the host")
+	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
+	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
+	buildCmd.Flags().Bool("keep-exported-images", false, "Keep exported images used during building")
 
 	buildCmd.Flags().String("solver-type", "", "Solver strategy")
 	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")

cmd/install.go

@@ -15,17 +15,16 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 
 	installer "github.com/mudler/luet/pkg/installer"
 
 	. "github.com/mudler/luet/pkg/config"
+	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
-	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/spf13/cobra"
 )
 
@@ -39,6 +38,11 @@ var installCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
 		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		LuetCfg.Viper.BindPFlag("download-only", cmd.Flags().Lookup("download-only"))
+
 	},
 	Long: `Install packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
@@ -46,26 +50,10 @@ var installCmd = &cobra.Command{
 		var systemDB pkg.PackageDatabase
 
 		for _, a := range args {
-			gp, err := _gentoo.ParsePackageStr(a)
+			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
 				Fatal("Invalid package string ", a, ": ", err.Error())
 			}
-
-			if gp.Version == "" {
-				gp.Version = "0"
-				gp.Condition = _gentoo.PkgCondGreaterEqual
-			}
-
-			pack := &pkg.DefaultPackage{
-				Name: gp.Name,
-				Version: fmt.Sprintf("%s%s%s",
-					pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
-					gp.Version,
-					gp.VersionSuffix,
-				),
-				Category: gp.Category,
-				Uri:      make([]string, 0),
-			}
 			toInstall = append(toInstall, pack)
 		}
 
@@ -83,7 +71,10 @@ var installCmd = &cobra.Command{
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
 		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
-
+		force := LuetCfg.Viper.GetBool("force")
+		nodeps := LuetCfg.Viper.GetBool("nodeps")
+		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
+		downloadOnly := LuetCfg.Viper.GetBool("download-only")
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
 		LuetCfg.GetSolverOptions().Discount = float32(discount)
@@ -91,7 +82,14 @@ var installCmd = &cobra.Command{
 
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
-		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+		})
 		inst.Repositories(repos)
 
 		if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
@@ -101,7 +99,7 @@ var installCmd = &cobra.Command{
 			systemDB = pkg.NewInMemoryDatabase(true)
 		}
 		system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs}
-		err := inst.Install(toInstall, system)
+		err := inst.Install(toInstall, system, downloadOnly)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
@@ -119,6 +117,10 @@ func init() {
 	installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	installCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	installCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	installCmd.Flags().Bool("download-only", false, "Download dependencies only")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/root.go

@@ -16,6 +16,7 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -24,6 +25,7 @@ import (
 
 	"github.com/marcsauter/single"
 	config "github.com/mudler/luet/pkg/config"
+	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	repo "github.com/mudler/luet/pkg/repository"
 
@@ -35,7 +37,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.5.1"
+	LuetCLIVersion = "0.6.4"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -51,12 +53,13 @@ var RootCmd = &cobra.Command{
 			Fatal("failed to load configuration:", err.Error())
 		}
 	},
+	SilenceErrors: true,
 }
 
 func LoadConfig(c *config.LuetConfig) error {
 	// If a config file is found, read it in.
 	if err := c.Viper.ReadInConfig(); err != nil {
-		Warning(err)
+		Debug(err)
 	}
 
 	err := c.Viper.Unmarshal(&config.LuetCfg)
@@ -99,8 +102,18 @@ func Execute() {
 		}
 		defer s.TryUnlock()
 	}
+
 	if err := RootCmd.Execute(); err != nil {
-		Error(err)
+		if len(os.Args) > 0 {
+			for _, c := range RootCmd.Commands() {
+				if c.Name() == os.Args[1] {
+					os.Exit(-1) // Something failed
+				}
+			}
+			// Try to load a bin from path.
+			helpers.Exec("luet-"+os.Args[1], os.Args[1:], os.Environ())
+		}
+		fmt.Println(err)
 		os.Exit(-1)
 	}
 }

cmd/uninstall.go

@@ -15,16 +15,15 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 
 	. "github.com/mudler/luet/pkg/config"
+	helpers "github.com/mudler/luet/pkg/helpers"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
-	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/spf13/cobra"
 )
 
@@ -39,34 +38,25 @@ var uninstallCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
 		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var systemDB pkg.PackageDatabase
 
 		for _, a := range args {
-			gp, err := _gentoo.ParsePackageStr(a)
+
+			pack, err := helpers.ParsePackageStr(a)
 			if err != nil {
 				Fatal("Invalid package string ", a, ": ", err.Error())
 			}
-			if gp.Version == "" {
-				gp.Version = "0"
-				gp.Condition = _gentoo.PkgCondGreaterEqual
-			}
-			pack := &pkg.DefaultPackage{
-				Name: gp.Name,
-				Version: fmt.Sprintf("%s%s%s",
-					pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
-					gp.Version,
-					gp.VersionSuffix,
-				),
-				Category: gp.Category,
-				Uri:      make([]string, 0),
-			}
 
 			stype := LuetCfg.Viper.GetString("solver.type")
 			discount := LuetCfg.Viper.GetFloat64("solver.discount")
 			rate := LuetCfg.Viper.GetFloat64("solver.rate")
 			attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+			force := LuetCfg.Viper.GetBool("force")
+			nodeps := LuetCfg.Viper.GetBool("nodeps")
 
 			LuetCfg.GetSolverOptions().Type = stype
 			LuetCfg.GetSolverOptions().LearnRate = float32(rate)
@@ -75,7 +65,12 @@ var uninstallCmd = &cobra.Command{
 
 			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
-			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+				Concurrency:   LuetCfg.GetGeneral().Concurrency,
+				SolverOptions: *LuetCfg.GetSolverOptions(),
+				NoDeps:        nodeps,
+				Force:         force,
+			})
 
 			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
 				systemDB = pkg.NewBoltDatabase(
@@ -103,5 +98,8 @@ func init() {
 	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	uninstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	uninstallCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	uninstallCmd.Flags().Bool("force", false, "Force uninstall")
+
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -36,6 +36,7 @@ var upgradeCmd = &cobra.Command{
 		LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
 		LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
 		LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+		LuetCfg.Viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 	},
 	Long: `Upgrades packages in parallel`,
 	Run: func(cmd *cobra.Command, args []string) {
@@ -55,6 +56,7 @@ var upgradeCmd = &cobra.Command{
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
 		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
+		force := LuetCfg.Viper.GetBool("force")
 
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
@@ -63,7 +65,11 @@ var upgradeCmd = &cobra.Command{
 
 		Debug("Solver", LuetCfg.GetSolverOptions().String())
 
-		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:   LuetCfg.GetGeneral().Concurrency,
+			SolverOptions: *LuetCfg.GetSolverOptions(),
+			Force:         force,
+		})
 		inst.Repositories(repos)
 		_, err := inst.SyncRepositories(false)
 		if err != nil {
@@ -95,5 +101,7 @@ func init() {
 	upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
+
 	RootCmd.AddCommand(upgradeCmd)
 }

go.mod

@@ -4,7 +4,8 @@ go 1.12
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7
+	github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56 // indirect
+	github.com/Sabayon/pkgs-checker v0.5.1-0.20200221202320-073693f2c657
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/briandowns/spinner v1.7.0
 	github.com/cavaliercoder/grab v2.0.0+incompatible
@@ -12,6 +13,7 @@ require (
 	github.com/docker/docker v0.7.3-0.20180827131323-0c5f8d2b9b23
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
 	github.com/ghodss/yaml v1.0.0
+	github.com/go-yaml/yaml v2.1.0+incompatible // indirect
 	github.com/hashicorp/go-version v1.2.0
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
 	github.com/klauspost/pgzip v1.2.1

go.sum

@@ -9,11 +9,16 @@ github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
+github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56 h1:XCZM9J5KqLsr5NqtrZuXiD3X5fe5IfgU7IIUZzpeFBk=
+github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56/go.mod h1:+Gbv6dg6TPHWq4oDjZY1vn978PLCEZ2hOu8kvn+S7t4=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7 h1:Vf80sSLu1ZWjjMmUKhw0FqM43lEOvT8O5B22NaHB6AQ=
 github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
+github.com/Sabayon/pkgs-checker v0.5.0 h1:VRyyAxo6ox41Dytyl+K+QC+Tps0IxvqYbidu+AH+HUQ=
+github.com/Sabayon/pkgs-checker v0.5.1-0.20200221202320-073693f2c657 h1:VK5S2Gh9kPUxX81zCFUgKVQn+hFy6VgyZMD3QLm76u8=
+github.com/Sabayon/pkgs-checker v0.5.1-0.20200221202320-073693f2c657/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
 github.com/Sereal/Sereal v0.0.0-20181211220259-509a78ddbda3 h1:Xu7z47ZiE/J+sKXHZMGxEor/oY2q6dq51fkO0JqdSwY=
 github.com/Sereal/Sereal v0.0.0-20181211220259-509a78ddbda3/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -91,6 +96,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
+github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=

pkg/compiler/backend/simpledocker.go

@@ -101,6 +101,18 @@ func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
+func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
+	name := opts.ImageName
+	pusharg := []string{"push", name}
+	out, err := exec.Command("docker", pusharg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed pushing image: "+string(out))
+	}
+	Info(":whale: Pushed image:", name)
+	//Info(string(out))
+	return nil
+}
+
 func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")

pkg/compiler/backend/simpleimg.go

@@ -145,3 +145,15 @@ func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms
 func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
 	return NewSimpleDockerBackend().Changes(fromImage, toImage)
 }
+
+func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {
+	name := opts.ImageName
+	pusharg := []string{"push", name}
+	out, err := exec.Command("img", pusharg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed pushing image: "+string(out))
+	}
+	Info(":tea: Pushed image:", name)
+	//Info(string(out))
+	return nil
+}

pkg/compiler/compiler.go

@@ -256,17 +256,21 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		return nil, errors.Wrap(err, "Could not copy package sources")
 
 	}
-	if buildertaggedImage == "" {
-		buildertaggedImage = cs.ImageRepository + "-" + p.GetPackage().GetFingerPrint() + "-builder"
-	}
-	if packageImage == "" {
-		packageImage = cs.ImageRepository + "-" + p.GetPackage().GetFingerPrint()
+
+	// Copy file into the build context, the compilespec might have requested to do so.
+	if len(p.GetRetrieve()) > 0 {
+		err := p.CopyRetrieves(buildDir)
+		if err != nil {
+			Warning("Failed copying retrieves", err.Error())
+		}
 	}
 
-	if cs.PullFirst {
-		//Best effort pull
-		cs.Backend.DownloadImage(CompilerBackendOptions{ImageName: buildertaggedImage})
-		cs.Backend.DownloadImage(CompilerBackendOptions{ImageName: packageImage})
+	fp := p.GetPackage().HashFingerprint()
+	if buildertaggedImage == "" {
+		buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder"
+	}
+	if packageImage == "" {
+		packageImage = cs.ImageRepository + "-" + fp
 	}
 
 	Info(pkgTag, "Generating :whale: definition for builder image from", image)
@@ -280,16 +284,32 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 	}
 
-	err = cs.Backend.BuildImage(builderOpts)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
+	buildBuilderImage := true
+	if cs.Options.PullFirst {
+		if err := cs.Backend.DownloadImage(builderOpts); err == nil {
+			buildBuilderImage = false
+		}
 	}
 
-	err = cs.Backend.ExportImage(builderOpts)
-	if err != nil {
+	if buildBuilderImage {
+		if err = cs.Backend.BuildImage(builderOpts); err != nil {
+			return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
+
+	if err = cs.Backend.ExportImage(builderOpts); err != nil {
 		return nil, errors.Wrap(err, "Could not export image")
 	}
 
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(builderOpts.Destination)
+	}
+
+	if cs.Options.Push && buildBuilderImage {
+		if err = cs.Backend.Push(builderOpts); err != nil {
+			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
 	// Then we write the step image, which uses the builder one
 	p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
 	runnerOpts := CompilerBackendOptions{
@@ -305,12 +325,34 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	// 		return nil, errors.Wrap(err, "Could not export image to tar")
 	// 	}
 	// } else {
-	if err := cs.Backend.BuildImage(runnerOpts); err != nil {
-		return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
+	buildPackageImage := true
+	if cs.Options.PullFirst {
+		//Best effort pull
+		if err := cs.Backend.DownloadImage(runnerOpts); err == nil {
+			buildPackageImage = false
+		}
 	}
+
+	if buildPackageImage {
+		if err := cs.Backend.BuildImage(runnerOpts); err != nil {
+			return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
+		}
+	}
+
 	if err := cs.Backend.ExportImage(runnerOpts); err != nil {
 		return nil, errors.Wrap(err, "Failed exporting image")
 	}
+
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(runnerOpts.Destination)
+	}
+
+	if cs.Options.Push && buildPackageImage {
+		err = cs.Backend.Push(runnerOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
+		}
+	}
 	//	}
 
 	var diffs []ArtifactLayer
@@ -394,7 +436,7 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 			return art, err
 		}
 	}
-	pkgTag := ":package:  " + p.GetPackage().GetName()
+	pkgTag := ":package:  " + p.GetPackage().HumanReadableString()
 
 	Info(pkgTag, "   🍩 Build starts 🔨 🔨 🔨 ")
 
@@ -418,6 +460,10 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 		return nil, errors.Wrap(err, "Could not export image")
 	}
 
+	if !cs.Options.KeepImageExport {
+		defer os.Remove(builderOpts.Destination)
+	}
+
 	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
@@ -450,7 +496,7 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 		}
 	}
 
-	Info(pkgTag, "   :white_check_mark: Done")
+	Info(pkgTag, " :white_check_mark: Done")
 
 	err = artifact.WriteYaml(p.GetOutputPath())
 	if err != nil {
@@ -465,7 +511,7 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
 
 	solution, err := s.Install([]pkg.Package{p.GetPackage()})
 	if err != nil {
-		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
 	}
 
 	dependencies := solution.Order(cs.Database, p.GetPackage().GetFingerPrint())
@@ -497,7 +543,7 @@ func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifa
 }
 
 func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
-	Info(":package: Compiling", p.GetPackage().GetName(), "version", p.GetPackage().GetVersion(), ".... :coffee:")
+	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
 	if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" {
 		Error("Package with no deps and no seed image supplied, bailing out")
@@ -525,74 +571,82 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	depsN := 0
 	currentN := 0
 
-	Info(":deciduous_tree: Build dependencies for " + p.GetPackage().GetName())
-	for _, assertion := range dependencies { //highly dependent on the order
-		depsN++
-		Info(" :arrow_right_hook:", assertion.Package.GetName(), ":leaves:", assertion.Package.GetVersion(), "(", assertion.Package.GetCategory(), ")")
-
-	}
-
-	for _, assertion := range dependencies { //highly dependent on the order
-		currentN++
-		pkgTag := fmt.Sprintf(":package:  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().GetName(), assertion.Package.GetName())
-		Info(pkgTag, "   :zap:  Building dependency")
-		compileSpec, err := cs.FromPackage(assertion.Package)
-		if err != nil {
-			return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+	if !cs.Options.NoDeps {
+		Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
+		for _, assertion := range dependencies { //highly dependent on the order
+			depsN++
+			Info(" :arrow_right_hook:", assertion.Package.HumanReadableString(), ":leaves:")
 		}
-		compileSpec.SetOutputPath(p.GetOutputPath())
 
-		buildImageHash := cs.ImageRepository + ":" + assertion.Hash.BuildHash
-		currentPackageImageHash := cs.ImageRepository + ":" + assertion.Hash.PackageHash
-		Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from", buildImageHash)
-		Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
+		for _, assertion := range dependencies { //highly dependent on the order
+			currentN++
+			pkgTag := fmt.Sprintf(":package:  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().HumanReadableString(), assertion.Package.HumanReadableString())
+			Info(pkgTag, "   :zap:  Building dependency")
+			compileSpec, err := cs.FromPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+			}
+			compileSpec.SetOutputPath(p.GetOutputPath())
 
-		lastHash = currentPackageImageHash
-		if compileSpec.GetImage() != "" {
-			// TODO: Refactor this
-			if compileSpec.ImageUnpack() { // If it is just an entire image, create a package from it
-				if compileSpec.GetImage() == "" {
-					return nil, errors.New("No image defined for package: " + assertion.Package.GetName())
+			buildImageHash := cs.ImageRepository + ":" + assertion.Hash.BuildHash
+			currentPackageImageHash := cs.ImageRepository + ":" + assertion.Hash.PackageHash
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Builder image from", buildImageHash)
+			Debug(pkgTag, "    :arrow_right_hook: :whale: Package image name", currentPackageImageHash)
+
+			lastHash = currentPackageImageHash
+			if compileSpec.GetImage() != "" {
+				// TODO: Refactor this
+				if compileSpec.ImageUnpack() { // If it is just an entire image, create a package from it
+					if compileSpec.GetImage() == "" {
+						return nil, errors.New("No image defined for package: " + assertion.Package.HumanReadableString())
+					}
+					Info(pkgTag, ":whale: Sourcing package from image", compileSpec.GetImage())
+					artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions, cs.KeepImg, concurrency)
+					if err != nil {
+						return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
+					}
+					departifacts = append(departifacts, artifact)
+					continue
 				}
-				Info(pkgTag, ":whale: Sourcing package from image", compileSpec.GetImage())
-				artifact, err := cs.packageFromImage(compileSpec, currentPackageImageHash, keepPermissions, cs.KeepImg, concurrency)
+
+				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
+				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
 				if err != nil {
-					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
+					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				}
 				departifacts = append(departifacts, artifact)
+				Info(pkgTag, ":white_check_mark: Done")
 				continue
 			}
 
-			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().GetFingerPrint()+" from image")
-			artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
+			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
 			if err != nil {
-				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
+				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
+				//	deperrs = append(deperrs, err)
+				//		break // stop at first error
 			}
 			departifacts = append(departifacts, artifact)
-			Info(pkgTag, ":white_check_mark: Done")
-			continue
+			Info(pkgTag, ":collision: Done")
 		}
 
-		Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().GetFingerPrint()+" from tree")
-		artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+	} else if len(dependencies) > 0 {
+		lastHash = cs.ImageRepository + ":" + dependencies[len(dependencies)-1].Hash.PackageHash
+	}
+
+	if !cs.Options.OnlyDeps {
+		Info(":package:", p.GetPackage().HumanReadableString(), ":cyclone:  Building package target from:", lastHash)
+		artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, cs.KeepImg, p)
 		if err != nil {
-			return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().GetName())
-			//	deperrs = append(deperrs, err)
-			//		break // stop at first error
+			return artifact, err
 		}
-		departifacts = append(departifacts, artifact)
-		Info(pkgTag, ":collision: Done")
-	}
+		artifact.SetDependencies(departifacts)
+		artifact.SetSourceAssertion(p.GetSourceAssertion())
 
-	Info(":package:", p.GetPackage().GetName(), ":cyclone:  Building package target from:", lastHash)
-	artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, cs.KeepImg, p)
-	if err != nil {
 		return artifact, err
+	} else {
+		return departifacts[len(departifacts)-1], nil
 	}
-	artifact.SetDependencies(departifacts)
-	artifact.SetSourceAssertion(p.GetSourceAssertion())
-
-	return artifact, err
 }
 
 func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {

pkg/compiler/interface.go

@@ -44,23 +44,29 @@ type CompilerBackendOptions struct {
 }
 
 type CompilerOptions struct {
-	ImageRepository    string
-	PullFirst, KeepImg bool
-	Concurrency        int
-	CompressionType    CompressionImplementation
-	Clean              bool
+	ImageRepository          string
+	PullFirst, KeepImg, Push bool
+	Concurrency              int
+	CompressionType          CompressionImplementation
+	Clean                    bool
+	KeepImageExport          bool
 
+	OnlyDeps      bool
+	NoDeps        bool
 	SolverOptions config.LuetSolverOptions
 }
 
 func NewDefaultCompilerOptions() *CompilerOptions {
 	return &CompilerOptions{
 		ImageRepository: "luet/cache",
-		PullFirst:       true,
+		PullFirst:       false,
+		Push:            false,
 		CompressionType: None,
 		KeepImg:         true,
 		Concurrency:     runtime.NumCPU(),
 		Clean:           true,
+		OnlyDeps:        false,
+		NoDeps:          false,
 	}
 }
 
@@ -74,6 +80,8 @@ type CompilerBackend interface {
 
 	CopyImage(string, string) error
 	DownloadImage(opts CompilerBackendOptions) error
+
+	Push(opts CompilerBackendOptions) error
 }
 
 type Artifact interface {
@@ -154,6 +162,9 @@ type CompilationSpec interface {
 
 	GetSourceAssertion() solver.PackagesAssertions
 	SetSourceAssertion(as solver.PackagesAssertions)
+
+	GetRetrieve() []string
+	CopyRetrieves(dest string) error
 }
 
 type CompilationSpecs interface {

pkg/compiler/spec.go

@@ -21,6 +21,7 @@ import (
 
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
+	"github.com/otiai10/copy"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -95,6 +96,8 @@ type LuetCompilationSpec struct {
 	Package         *pkg.DefaultPackage       `json:"package"`
 	SourceAssertion solver.PackagesAssertions `json:"-"`
 
+	Retrieve []string `json:"retrieve"`
+
 	OutputPath string   `json:"-"` // Where the build processfiles go
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
@@ -136,6 +139,10 @@ func (cs *LuetCompilationSpec) GetIncludes() []string {
 	return cs.Includes
 }
 
+func (cs *LuetCompilationSpec) GetRetrieve() []string {
+	return cs.Retrieve
+}
+
 func (cs *LuetCompilationSpec) GetSeedImage() string {
 	return cs.Seed
 }
@@ -164,6 +171,24 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
 	cs.Seed = s
 }
 
+func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
+	var err error
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			matches, err := filepath.Glob(cs.Rel(s))
+
+			if err != nil {
+				continue
+			}
+
+			for _, m := range matches {
+				err = copy.Copy(m, filepath.Join(dest, filepath.Base(m)))
+			}
+		}
+	}
+	return err
+}
+
 // TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
 func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
 	spec := `
@@ -174,6 +199,19 @@ ENV PACKAGE_NAME=` + cs.Package.GetName() + `
 ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
 ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
 
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			//var file string
+			// if helpers.IsValidUrl(s) {
+			// 	file = s
+			// } else {
+			// 	file = cs.Rel(s)
+			// }
+			spec = spec + `
+ADD ` + s + ` /luetbuild/`
+		}
+	}
+
 	for _, s := range cs.Env {
 		spec = spec + `
 ENV ` + s

pkg/compiler/spec_test.go

@@ -105,4 +105,74 @@ RUN echo bar > /test2`))
 		})
 
 	})
+
+	It("Renders retrieve and env fields", func() {
+		generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
+
+		err := generalRecipe.Load("../../tests/fixtures/retrieve")
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+		spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
+		Expect(err).ToNot(HaveOccurred())
+
+		lspec, ok := spec.(*LuetCompilationSpec)
+		Expect(ok).To(BeTrue())
+
+		Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
+		Expect(lspec.Image).To(Equal("luet/base"))
+		Expect(lspec.Seed).To(Equal("alpine"))
+		tmpdir, err := ioutil.TempDir("", "tree")
+		Expect(err).ToNot(HaveOccurred())
+		defer os.RemoveAll(tmpdir) // clean up
+
+		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1`))
+
+		lspec.SetOutputPath("/foo/bar")
+
+		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1`))
+
+		err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(dockerfile).To(Equal(`
+FROM luet/base
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ENV test=1
+RUN echo foo > /test
+RUN echo bar > /test2`))
+
+	})
 })

pkg/helpers/cli.go

@@ -0,0 +1,60 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"fmt"
+
+	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
+	pkg "github.com/mudler/luet/pkg/package"
+)
+
+func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
+	gp, err := _gentoo.ParsePackageStr(p)
+	if err != nil {
+		return nil, err
+	}
+	if gp.Version == "" {
+		gp.Version = "0"
+		gp.Condition = _gentoo.PkgCondGreaterEqual
+	}
+
+	pkgVersion := ""
+	if gp.VersionBuild != "" {
+		pkgVersion = fmt.Sprintf("%s%s%s+%s",
+			pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+			gp.Version,
+			gp.VersionSuffix,
+			gp.VersionBuild,
+		)
+	} else {
+		pkgVersion = fmt.Sprintf("%s%s%s",
+			pkg.PkgSelectorConditionFromInt(gp.Condition.Int()).String(),
+			gp.Version,
+			gp.VersionSuffix,
+		)
+	}
+
+	pack := &pkg.DefaultPackage{
+		Name:     gp.Name,
+		Category: gp.Category,
+		Version:  pkgVersion,
+		Uri:      make([]string, 0),
+	}
+
+	return pack, nil
+}

pkg/helpers/sys.go

@@ -0,0 +1,32 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"os/exec"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+// This allows a multi-platform switch in the future
+func Exec(cmd string, args []string, env []string) error {
+	path, err := exec.LookPath(cmd)
+	if err != nil {
+		return errors.Wrap(err, "Could not find binary in path: "+cmd)
+	}
+	return syscall.Exec(path, args, env)
+}

pkg/installer/installer.go

@@ -21,6 +21,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 
 	"github.com/ghodss/yaml"
@@ -36,8 +37,12 @@ import (
 )
 
 type LuetInstallerOptions struct {
-	SolverOptions config.LuetSolverOptions
-	Concurrency   int
+	SolverOptions               config.LuetSolverOptions
+	Concurrency                 int
+	NoDeps                      bool
+	OnlyDeps                    bool
+	Force                       bool
+	PreserveSystemEssentialData bool
 }
 
 type LuetInstaller struct {
@@ -103,23 +108,15 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 		return err
 	}
 	// First match packages against repositories by priority
-	//	matches := syncedRepos.PackageMatches(p)
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	// compute a "big" world
 	solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
-	uninstall, solution, err := solv.Upgrade()
+	uninstall, solution, err := solv.Upgrade(false)
 	if err != nil {
 		return errors.Wrap(err, "Failed solving solution for upgrade")
 	}
 
-	for _, u := range uninstall {
-		err := l.Uninstall(u, s)
-		if err != nil {
-			Warning("Failed uninstall for ", u.GetFingerPrint())
-		}
-	}
-
 	toInstall := []pkg.Package{}
 	for _, assertion := range solution {
 		if assertion.Value {
@@ -127,7 +124,7 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 		}
 	}
 
-	return l.Install(toInstall, s)
+	return l.swap(syncedRepos, uninstall, toInstall, s)
 }
 
 func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
@@ -152,7 +149,62 @@ func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) {
 	return syncedRepos, nil
 }
 
-func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
+func (l *LuetInstaller) Swap(toRemove []pkg.Package, toInstall []pkg.Package, s *System) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+	return l.swap(syncedRepos, toRemove, toInstall, s)
+}
+
+func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove []pkg.Package, toInstall []pkg.Package, s *System) error {
+
+	// First match packages against repositories by priority
+	allRepos := pkg.NewInMemoryDatabase(false)
+	syncedRepos.SyncDatabase(allRepos)
+	toInstall = syncedRepos.ResolveSelectors(toInstall)
+
+	if err := l.install(syncedRepos, toInstall, s, true); err != nil {
+		return errors.Wrap(err, "Pre-downloading packages")
+	}
+
+	// We don't want any conflict with the installed to raise during the upgrade.
+	// In this way we both force uninstalls and we avoid to check with conflicts
+	// against the current system state which is pending to deletion
+	// E.g. you can't check for conflicts for an upgrade of a new version of A
+	// if the old A results installed in the system. This is due to the fact that
+	// now the solver enforces the constraints and explictly denies two packages
+	// of the same version installed.
+	forced := false
+	if l.Options.Force {
+		forced = true
+	}
+	l.Options.Force = true
+
+	for _, u := range toRemove {
+		Info(":package: Marked for deletion", u.HumanReadableString())
+
+		err := l.Uninstall(u, s)
+		if err != nil && !l.Options.Force {
+			Error("Failed uninstall for ", u.HumanReadableString())
+			return errors.Wrap(err, "uninstalling "+u.HumanReadableString())
+		}
+
+	}
+	l.Options.Force = forced
+
+	return l.install(syncedRepos, toInstall, s, false)
+}
+
+func (l *LuetInstaller) Install(cp []pkg.Package, s *System, downloadOnly bool) error {
+	syncedRepos, err := l.SyncRepositories(true)
+	if err != nil {
+		return err
+	}
+	return l.install(syncedRepos, cp, s, downloadOnly)
+}
+
+func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *System, downloadOnly bool) error {
 	var p []pkg.Package
 
 	// Check if the package is installed first
@@ -161,7 +213,7 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 		vers, _ := s.Database.FindPackageVersions(pi)
 
 		if len(vers) >= 1 {
-			Warning("Filtering out package " + pi.GetFingerPrint() + ", it has other versions already installed. Uninstall one of them first ")
+			Warning("Filtering out package " + pi.HumanReadableString() + ", it has other versions already installed. Uninstall one of them first ")
 			continue
 			//return errors.New("Package " + pi.GetFingerPrint() + " has other versions already installed. Uninstall one of them first: " + strings.Join(vers, " "))
 
@@ -175,10 +227,6 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 	}
 	// First get metas from all repos (and decodes trees)
 
-	syncedRepos, err := l.SyncRepositories(true)
-	if err != nil {
-		return err
-	}
 	// First match packages against repositories by priority
 	//	matches := syncedRepos.PackageMatches(p)
 
@@ -186,96 +234,140 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	p = syncedRepos.ResolveSelectors(p)
-
-	solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
-	solution, err := solv.Install(p)
-	if err != nil {
-		return errors.Wrap(err, "Failed solving solution for package")
-	}
-
-	// Gathers things to install
 	toInstall := map[string]ArtifactMatch{}
-	for _, assertion := range solution {
-		if assertion.Value {
-			matches := syncedRepos.PackageMatches([]pkg.Package{assertion.Package})
-			if len(matches) == 0 {
-				return errors.New("Failed matching solutions against repository - where are definitions coming from?!")
-			}
-		A:
-			for _, artefact := range matches[0].Repo.GetIndex() {
-				if artefact.GetCompileSpec().GetPackage() == nil {
-					return errors.New("Package in compilespec empty")
+	var packagesToInstall []pkg.Package
+	var err error
+	var solution solver.PackagesAssertions
 
-				}
-				if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
-					// Filter out already installed
-					if _, err := s.Database.FindPackage(assertion.Package); err != nil {
-						toInstall[assertion.Package.GetFingerPrint()] = ArtifactMatch{Package: assertion.Package, Artifact: artefact, Repository: matches[0].Repo}
-					}
-					break A
-				}
+	if !l.Options.NoDeps {
+		solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		solution, err = solv.Install(p)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed solving solution for package")
+		}
+		// Gathers things to install
+		for _, assertion := range solution {
+			if assertion.Value {
+				packagesToInstall = append(packagesToInstall, assertion.Package)
 			}
 		}
+	} else if !l.Options.OnlyDeps {
+		for _, currentPack := range p {
+			packagesToInstall = append(packagesToInstall, currentPack)
+		}
 	}
 
+	// Gathers things to install
+	for _, currentPack := range packagesToInstall {
+		matches := syncedRepos.PackageMatches([]pkg.Package{currentPack})
+		if len(matches) == 0 {
+			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
+		}
+	A:
+		for _, artefact := range matches[0].Repo.GetIndex() {
+			if artefact.GetCompileSpec().GetPackage() == nil {
+				return errors.New("Package in compilespec empty")
+
+			}
+			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+				// Filter out already installed
+				if _, err := s.Database.FindPackage(currentPack); err != nil {
+					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
+				}
+				break A
+			}
+		}
+	}
 	// Install packages into rootfs in parallel.
 	all := make(chan ArtifactMatch)
 
 	var wg = new(sync.WaitGroup)
-	for i := 0; i < l.Options.Concurrency; i++ {
-		wg.Add(1)
-		go l.installerWorker(i, wg, all, s)
+
+	if !downloadOnly {
+		// Download first
+		for i := 0; i < l.Options.Concurrency; i++ {
+			wg.Add(1)
+			go l.installerWorker(i, wg, all, s, true)
+		}
+
+		for _, c := range toInstall {
+			all <- c
+		}
+		close(all)
+		wg.Wait()
+
+		all = make(chan ArtifactMatch)
+
+		wg = new(sync.WaitGroup)
+
+		// Do the real install
+		for i := 0; i < l.Options.Concurrency; i++ {
+			wg.Add(1)
+			go l.installerWorker(i, wg, all, s, false)
+		}
+
+		for _, c := range toInstall {
+			all <- c
+		}
+		close(all)
+		wg.Wait()
+	} else {
+		for i := 0; i < l.Options.Concurrency; i++ {
+			wg.Add(1)
+			go l.installerWorker(i, wg, all, s, downloadOnly)
+		}
+
+		for _, c := range toInstall {
+			all <- c
+		}
+		close(all)
+		wg.Wait()
+	}
+
+	if downloadOnly {
+		return nil
 	}
 
 	for _, c := range toInstall {
-		all <- c
+		// Annotate to the system that the package was installed
+		_, err := s.Database.CreatePackage(c.Package)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Failed creating package")
+		}
 	}
-	close(all)
-	wg.Wait()
-
 	executedFinalizer := map[string]bool{}
 
 	// TODO: Lower those errors as warning
 	for _, w := range p {
 		// Finalizers needs to run in order and in sequence.
 		ordered := solution.Order(allRepos, w.GetFingerPrint())
+	ORDER:
 		for _, ass := range ordered {
 			if ass.Value {
-				// Annotate to the system that the package was installed
-				// TODO: Annotate also files that belong to the package, somewhere to uninstall
-				if _, err := s.Database.FindPackage(ass.Package); err == nil {
-					err := s.Database.UpdatePackage(ass.Package)
-					if err != nil {
-						return errors.Wrap(err, "Failed updating package")
-					}
-				} else {
-					_, err := s.Database.CreatePackage(ass.Package)
-					if err != nil {
-						return errors.Wrap(err, "Failed creating package")
-					}
-				}
+
 				installed, ok := toInstall[ass.Package.GetFingerPrint()]
 				if !ok {
-					return errors.New("Couldn't find ArtifactMatch for " + ass.Package.GetFingerPrint())
+					// It was a dep already installed in the system, so we can skip it safely
+					continue ORDER
 				}
 
 				treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
 				if err != nil {
-					return errors.Wrap(err, "Error getting package "+ass.Package.GetFingerPrint())
+					return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
 				}
 				if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
-					Info("Executing finalizer for " + ass.Package.GetName())
+					Info("Executing finalizer for " + ass.Package.HumanReadableString())
 					finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
-					if err != nil {
+					if err != nil && !l.Options.Force {
 						return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
 					}
 					if _, exists := executedFinalizer[ass.Package.GetFingerPrint()]; !exists {
 						finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
-						if err != nil {
+						if err != nil && !l.Options.Force {
 							return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
 						}
 						err = finalizer.RunInstall()
-						if err != nil {
+						if err != nil && !l.Options.Force {
 							return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
 						}
 						executedFinalizer[ass.Package.GetFingerPrint()] = true
@@ -285,12 +377,11 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
 		}
 
 	}
-
 	return nil
 
 }
 
-func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
+func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System, downloadOnly bool) error {
 
 	artifact, err := a.Repository.Client().DownloadArtifact(a.Artifact)
 	if err != nil {
@@ -298,17 +389,20 @@ func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
 	}
 
 	err = artifact.Verify()
-	if err != nil {
+	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Artifact integrity check failure")
 	}
+	if downloadOnly {
+		return nil
+	}
 
 	files, err := artifact.FileList()
-	if err != nil {
+	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Could not open package archive")
 	}
 
 	err = artifact.Unpack(s.Target, true)
-	if err != nil {
+	if err != nil && !l.Options.Force {
 		return errors.Wrap(err, "Error met while unpacking rootfs")
 	}
 
@@ -317,17 +411,24 @@ func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
 	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.Package.GetFingerPrint(), Files: files})
 }
 
-func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System) error {
+func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System, downloadOnly bool) error {
 	defer wg.Done()
 
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
-		err := l.installPackage(p, s)
-		if err != nil {
+		err := l.installPackage(p, s, downloadOnly)
+		if err != nil && !l.Options.Force {
 			//TODO: Uninstall, rollback.
 			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
 			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
 		}
+		if err == nil && downloadOnly {
+			Info(":package: ", p.Package.HumanReadableString(), "downloaded")
+		} else if err == nil {
+			Info(":package: ", p.Package.HumanReadableString(), "installed")
+		} else if err != nil && l.Options.Force {
+			Info(":package: ", p.Package.HumanReadableString(), "installed with failures (force install)")
+		}
 	}
 
 	return nil
@@ -342,7 +443,15 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 	// Remove from target
 	for _, f := range files {
 		target := filepath.Join(s.Target, f)
-		Info("Removing", target)
+		Debug("Removing", target)
+
+		if l.Options.PreserveSystemEssentialData &&
+			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) ||
+			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath()) {
+			Warning("Preserve ", f, " which is required by luet ( you have to delete it manually if you really need to)")
+			continue
+		}
+
 		err := os.Remove(target)
 		if err != nil {
 			Warning("Failed removing file (not present in the system target ?)", target)
@@ -362,19 +471,34 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 }
 
 func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
-	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) - mark the uninstallation in db
+	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) TODO - mark the uninstallation in db
 	// Get installed definition
-	solv := solver.NewResolver(s.Database, s.Database, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
-	solution, err := solv.Uninstall(p)
-	if err != nil {
-		return errors.Wrap(err, "Uninstall failed")
+
+	checkConflicts := true
+	if l.Options.Force == true {
+		checkConflicts = false
 	}
-	for _, p := range solution {
-		Info("Uninstalling", p.GetFingerPrint())
+
+	if !l.Options.NoDeps {
+		solv := solver.NewResolver(s.Database, s.Database, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
+		solution, err := solv.Uninstall(p, checkConflicts)
+		if err != nil && !l.Options.Force {
+			return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+		}
+		for _, p := range solution {
+			Info("Uninstalling", p.HumanReadableString())
+			err := l.uninstall(p, s)
+			if err != nil && !l.Options.Force {
+				return errors.Wrap(err, "Uninstall failed")
+			}
+		}
+	} else {
+		Info("Uninstalling", p.HumanReadableString(), "without deps")
 		err := l.uninstall(p, s)
-		if err != nil {
+		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Uninstall failed")
 		}
+		Info(":package: ", p.HumanReadableString(), "uninstalled")
 	}
 	return nil
 

pkg/installer/installer_test.go

@@ -113,7 +113,7 @@ urls:
 			Expect(repo.GetType()).To(Equal("disk"))
 			systemDB := pkg.NewInMemoryDatabase(false)
 			system := &System{Database: systemDB, Target: fakeroot}
-			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
@@ -231,7 +231,7 @@ urls:
 
 			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
 			system := &System{Database: systemDB, Target: fakeroot}
-			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
@@ -339,7 +339,7 @@ urls:
 
 			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
 			system := &System{Database: systemDB, Target: fakeroot}
-			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
@@ -454,7 +454,7 @@ urls:
 
 			systemDB := pkg.NewBoltDatabase(filepath.Join(bolt, "db.db"))
 			system := &System{Database: systemDB, Target: fakeroot}
-			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system)
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())

pkg/installer/interface.go

@@ -23,11 +23,12 @@ import (
 )
 
 type Installer interface {
-	Install([]pkg.Package, *System) error
+	Install([]pkg.Package, *System, bool) error
 	Uninstall(pkg.Package, *System) error
 	Upgrade(s *System) error
 	Repositories([]Repository)
 	SyncRepositories(bool) (Repositories, error)
+	Swap([]pkg.Package, []pkg.Package, *System) error
 }
 
 type Client interface {

pkg/installer/repository.go

@@ -530,14 +530,21 @@ func (re Repositories) ResolveSelectors(p []pkg.Package) []pkg.Package {
 	var matches []pkg.Package
 PACKAGE:
 	for _, pack := range p {
+	REPOSITORY:
 		for _, r := range re {
 			if pack.IsSelector() {
 				c, err := r.GetTree().GetDatabase().FindPackageCandidate(pack)
+				// If FindPackageCandidate returns the same package, it means it couldn't find one.
+				// Skip this repository and keep looking.
+				if c.String() == pack.String() {
+					continue REPOSITORY
+				}
 				if err == nil {
 					matches = append(matches, c)
 					continue PACKAGE
 				}
 			} else {
+				// If it's not a selector, just append it
 				matches = append(matches, pack)
 			}
 		}

pkg/package/package.go

@@ -17,8 +17,10 @@ package pkg
 
 import (
 	"bytes"
+	"crypto/md5"
 	"encoding/json"
 	"fmt"
+	"io"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -87,6 +89,8 @@ type Package interface {
 	SelectorMatchVersion(string) (bool, error)
 
 	String() string
+	HumanReadableString() string
+	HashFingerprint() string
 }
 
 type Tree interface {
@@ -173,6 +177,16 @@ func (p *DefaultPackage) GetFingerPrint() string {
 	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version)
 }
 
+func (p *DefaultPackage) HashFingerprint() string {
+	h := md5.New()
+	io.WriteString(h, p.GetFingerPrint())
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func (p *DefaultPackage) HumanReadableString() string {
+	return fmt.Sprintf("%s/%s-%s", p.Category, p.Name, p.Version)
+}
+
 func FromString(s string) Package {
 	var unescaped DefaultPackage
 
@@ -454,6 +468,22 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 	A := bf.Var(encodedA)
 
 	var formulas []bf.Formula
+
+	// Do conflict with other packages versions (if A is selected, then conflict with other versions of A)
+	packages, _ := definitiondb.FindPackageVersions(p)
+	if len(packages) > 0 {
+		for _, cp := range packages {
+			encodedB, err := cp.Encode(db)
+			if err != nil {
+				return nil, err
+			}
+			B := bf.Var(encodedB)
+			if !p.Matches(cp) {
+				formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Not(A), bf.Not(B))))
+			}
+		}
+	}
+
 	for _, requiredDef := range p.GetRequires() {
 		required, err := definitiondb.FindPackage(requiredDef)
 		if err != nil || requiredDef.IsSelector() {
@@ -465,62 +495,60 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 			if err != nil || len(packages) == 0 {
 				required = requiredDef
 			} else {
-				if len(packages) == 1 {
-					required = packages[0]
-				} else {
-					var ALO, priorityConstraints, priorityALO []bf.Formula
 
-					// Try to prio best match
-					// Force the solver to consider first our candidate (if does exists).
-					// Then builds ALO and AMO for the requires.
-					c, candidateErr := definitiondb.FindPackageCandidate(requiredDef)
-					var C bf.Formula
-					if candidateErr == nil {
-						// We have a desired candidate, try to look a solution with that included first
-						for _, o := range packages {
-							encodedB, err := o.Encode(db)
-							if err != nil {
-								return nil, err
-							}
-							B := bf.Var(encodedB)
-							if !o.Matches(c) {
-								priorityConstraints = append(priorityConstraints, bf.Not(B))
-								priorityALO = append(priorityALO, B)
-							}
-						}
-						encodedC, err := c.Encode(db)
-						if err != nil {
-							return nil, err
-						}
-						C = bf.Var(encodedC)
-						// Or the Candidate is true, or all the others might be not true
-						// This forces the CDCL sat implementation to look first at a solution with C=true
-						formulas = append(formulas, bf.Or(bf.Or(C, bf.Or(priorityConstraints...)), bf.Or(bf.Not(C), bf.Or(priorityALO...))))
-					}
+				var ALO, priorityConstraints, priorityALO []bf.Formula
 
-					// AMO - At most one
+				// Try to prio best match
+				// Force the solver to consider first our candidate (if does exists).
+				// Then builds ALO and AMO for the requires.
+				c, candidateErr := definitiondb.FindPackageCandidate(requiredDef)
+				var C bf.Formula
+				if candidateErr == nil {
+					// We have a desired candidate, try to look a solution with that included first
 					for _, o := range packages {
 						encodedB, err := o.Encode(db)
 						if err != nil {
 							return nil, err
 						}
 						B := bf.Var(encodedB)
-						ALO = append(ALO, B)
-						for _, i := range packages {
-							encodedI, err := i.Encode(db)
-							if err != nil {
-								return nil, err
-							}
-							I := bf.Var(encodedI)
-							if !o.Matches(i) {
-								formulas = append(formulas, bf.Or(bf.Not(I), bf.Not(B)))
-							}
+						if !o.Matches(c) {
+							priorityConstraints = append(priorityConstraints, bf.Not(B))
+							priorityALO = append(priorityALO, B)
 						}
 					}
-					formulas = append(formulas, bf.Or(ALO...)) // ALO - At least one
-					continue
+					encodedC, err := c.Encode(db)
+					if err != nil {
+						return nil, err
+					}
+					C = bf.Var(encodedC)
+					// Or the Candidate is true, or all the others might be not true
+					// This forces the CDCL sat implementation to look first at a solution with C=true
+					formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Or(C, bf.Or(priorityConstraints...)), bf.Or(bf.Not(C), bf.Or(priorityALO...)))))
 				}
+
+				// AMO - At most one
+				for _, o := range packages {
+					encodedB, err := o.Encode(db)
+					if err != nil {
+						return nil, err
+					}
+					B := bf.Var(encodedB)
+					ALO = append(ALO, B)
+					for _, i := range packages {
+						encodedI, err := i.Encode(db)
+						if err != nil {
+							return nil, err
+						}
+						I := bf.Var(encodedI)
+						if !o.Matches(i) {
+							formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Not(I), bf.Not(B))))
+						}
+					}
+				}
+				formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(ALO...))) // ALO - At least one
+				continue
 			}
+
 		}
 
 		encodedB, err := required.Encode(db)
@@ -586,6 +614,7 @@ func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db Packag
 		formulas = append(formulas, f...)
 
 	}
+
 	return formulas, nil
 }
 

pkg/package/package_test.go

@@ -25,11 +25,18 @@ var _ = Describe("Package", func() {
 
 	Context("Encoding/Decoding", func() {
 		a := &DefaultPackage{Name: "test", Version: "1", Category: "t"}
+		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+
 		It("Encodes and decodes correctly", func() {
 			Expect(a.String()).ToNot(Equal(""))
 			p := FromString(a.String())
 			Expect(p).To(Equal(a))
 		})
+
+		It("Generates packages fingerprint's hashes", func() {
+			Expect(a.HashFingerprint()).ToNot(Equal(a1.HashFingerprint()))
+			Expect(a.HashFingerprint()).To(Equal("c64caa391b79adb598ad98e261aa79a0"))
+		})
 	})
 
 	Context("Simple package", func() {
@@ -204,7 +211,7 @@ var _ = Describe("Package", func() {
 
 			f, err := a1.BuildFormula(definitions, db)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(len(f)).To(Equal(2))
+			Expect(len(f)).To(Equal(8))
 			//	Expect(f[0].String()).To(Equal("or(not(c31f5842), a4910f77)"))
 			//	Expect(f[1].String()).To(Equal("or(not(c31f5842), not(a97670be))"))
 		})

pkg/package/version.go

@@ -161,6 +161,35 @@ func ParseVersion(v string) (PkgVersionSelector, error) {
 		ans.Condition = PkgCondNot
 	}
 
+	// Check if build number is present
+	buildIdx := strings.Index(v, "+")
+	buildVersion := ""
+	if buildIdx > 0 {
+		// <pre-release> ::= <dot-separated pre-release identifiers>
+		//
+		// <dot-separated pre-release identifiers> ::=
+		//      <pre-release identifier> | <pre-release identifier> "."
+		//      <dot-separated pre-release identifiers>
+		//
+		// <build> ::= <dot-separated build identifiers>
+		//
+		// <dot-separated build identifiers> ::= <build identifier>
+		//      | <build identifier> "." <dot-separated build identifiers>
+		//
+		// <pre-release identifier> ::= <alphanumeric identifier>
+		//                            | <numeric identifier>
+		//
+		// <build identifier> ::= <alphanumeric identifier>
+		//      | <digits>
+		//
+		// <alphanumeric identifier> ::= <non-digit>
+		//      | <non-digit> <identifier characters>
+		//      | <identifier characters> <non-digit>
+		//      | <identifier characters> <non-digit> <identifier characters>
+		buildVersion = v[buildIdx:]
+		v = v[0:buildIdx]
+	}
+
 	regexPkg := regexp.MustCompile(
 		fmt.Sprintf("(%s|%s|%s|%s|%s|%s)((%s|%s|%s|%s|%s|%s|%s)+)*$",
 			// Version regex
@@ -216,6 +245,8 @@ func ParseVersion(v string) (PkgVersionSelector, error) {
 		ans.Condition = PkgCondEqual
 	}
 
+	ans.Version += buildVersion
+
 	// NOTE: Now suffix complex like _alpha_rc1 are not supported.
 	return ans, nil
 }

pkg/package/version_test.go

@@ -136,6 +136,61 @@ var _ = Describe("Versions", func() {
 		})
 	})
 
+	Context("Versions Parser11 - semver", func() {
+		v, err := ParseVersion("0.1.0+0")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.1.0+0"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser12 - semver", func() {
+		v, err := ParseVersion(">=0.1.0_alpha+AB")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.1.0+AB"))
+			Expect(v.VersionSuffix).Should(Equal("_alpha"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser13 - semver", func() {
+		v, err := ParseVersion(">=0.1.0_alpha+0.1.22")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.1.0+0.1.22"))
+			Expect(v.VersionSuffix).Should(Equal("_alpha"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser14 - semver", func() {
+		v, err := ParseVersion(">=0.1.0_alpha+0.1.22")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondGreaterEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.1.0+0.1.22"))
+			Expect(v.VersionSuffix).Should(Equal("_alpha"))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
+	Context("Versions Parser15 - semver", func() {
+		v, err := ParseVersion("<=0.3.222.4.5+AB")
+		It("ParseVersion10", func() {
+			var c PkgSelectorCondition = PkgCondLessEqual
+			Expect(err).Should(BeNil())
+			Expect(v.Version).Should(Equal("0.3.222.4.5+AB"))
+			Expect(v.VersionSuffix).Should(Equal(""))
+			Expect(v.Condition).Should(Equal(c))
+		})
+	})
+
 	Context("Selector1", func() {
 		v1, err := ParseVersion(">=0.0.20190406.4.9.172-r1")
 		v2, err2 := ParseVersion("1.0.111")
@@ -184,6 +239,42 @@ var _ = Describe("Versions", func() {
 		})
 	})
 
+	Context("Selector5", func() {
+		v1, err := ParseVersion(">0.1.0+0.4")
+		v2, err2 := ParseVersion("0.1.0+0.3")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector5", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(false))
+		})
+	})
+
+	Context("Selector6", func() {
+		v1, err := ParseVersion(">=0.1.0+0.4")
+		v2, err2 := ParseVersion("0.1.0+0.5")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector6", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(true))
+		})
+	})
+
+	PContext("Selector7", func() {
+		v1, err := ParseVersion(">0.1.0+0.4")
+		v2, err2 := ParseVersion("0.1.0+0.5")
+		match, err3 := PackageAdmit(v1, v2)
+		It("Selector7", func() {
+			Expect(err).Should(BeNil())
+			Expect(err2).Should(BeNil())
+			Expect(err3).Should(BeNil())
+			Expect(match).Should(Equal(true))
+		})
+	})
+
 	Context("Condition Converter 1", func() {
 		gp, err := gentoo.ParsePackageStr("=layer/build-1.0")
 		var cond gentoo.PackageCond = gentoo.PkgCondEqual

pkg/solver/solver.go

@@ -28,11 +28,11 @@ import (
 type PackageSolver interface {
 	SetDefinitionDatabase(pkg.PackageDatabase)
 	Install(p []pkg.Package) (PackagesAssertions, error)
-	Uninstall(candidate pkg.Package) ([]pkg.Package, error)
+	Uninstall(candidate pkg.Package, checkconflicts bool) ([]pkg.Package, error)
 	ConflictsWithInstalled(p pkg.Package) (bool, error)
 	ConflictsWith(p pkg.Package, ls []pkg.Package) (bool, error)
 	World() []pkg.Package
-	Upgrade() ([]pkg.Package, PackagesAssertions, error)
+	Upgrade(checkconflicts bool) ([]pkg.Package, PackagesAssertions, error)
 
 	SetResolver(PackageResolver)
 
@@ -210,7 +210,7 @@ func (s *Solver) ConflictsWithInstalled(p pkg.Package) (bool, error) {
 	return s.ConflictsWith(p, s.Installed())
 }
 
-func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
+func (s *Solver) Upgrade(checkconflicts bool) ([]pkg.Package, PackagesAssertions, error) {
 
 	// First get candidates that needs to be upgraded..
 
@@ -236,12 +236,11 @@ func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
 			}
 		}
 	}
-
 	s2 := NewSolver(installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
 	s2.SetResolver(s.Resolver)
 	// Then try to uninstall the versions in the system, and store that tree
 	for _, p := range toUninstall {
-		r, err := s.Uninstall(p)
+		r, err := s.Uninstall(p, checkconflicts)
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't uninstall selected candidate "+p.GetFingerPrint())
 		}
@@ -262,7 +261,7 @@ func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
 
 // Uninstall takes a candidate package and return a list of packages that would be removed
 // in order to purge the candidate. Returns error if unsat.
-func (s *Solver) Uninstall(c pkg.Package) ([]pkg.Package, error) {
+func (s *Solver) Uninstall(c pkg.Package, checkconflicts bool) ([]pkg.Package, error) {
 	var res []pkg.Package
 	candidate, err := s.InstalledDatabase.FindPackage(c)
 	if err != nil {
@@ -294,17 +293,19 @@ func (s *Solver) Uninstall(c pkg.Package) ([]pkg.Package, error) {
 		}
 	}
 
+	s2 := NewSolver(pkg.NewInMemoryDatabase(false), s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
+	s2.SetResolver(s.Resolver)
 	// Get the requirements to install the candidate
-	saved := s.InstalledDatabase
-	s.InstalledDatabase = pkg.NewInMemoryDatabase(false)
-	asserts, err := s.Install([]pkg.Package{candidate})
+	asserts, err := s2.Install([]pkg.Package{candidate})
 	if err != nil {
 		return nil, err
 	}
-	s.InstalledDatabase = saved
-
 	for _, a := range asserts {
 		if a.Value {
+			if !checkconflicts {
+				res = append(res, a.Package.IsFlagged(false))
+				continue
+			}
 
 			c, err := s.ConflictsWithInstalled(a.Package)
 			if err != nil {

pkg/solver/solver_test.go

@@ -429,7 +429,7 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(5))
+			Expect(len(solution)).To(Equal(6))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -470,7 +470,7 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(4))
+			Expect(len(solution)).To(Equal(6))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -511,7 +511,7 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: D1, Value: false}))
 			Expect(solution).ToNot(ContainElement(PackageAssert{Package: E, Value: true}))
 
-			Expect(len(solution)).To(Equal(4))
+			Expect(len(solution)).To(Equal(6))
 			Expect(err).ToNot(HaveOccurred())
 		})
 
@@ -533,7 +533,7 @@ var _ = Describe("Solver", func() {
 			}
 			s = NewSolver(dbInstalled, dbDefinitions, db)
 
-			solution, err := s.Uninstall(A)
+			solution, err := s.Uninstall(A, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(solution).To(ContainElement(A.IsFlagged(false)))
@@ -559,7 +559,7 @@ var _ = Describe("Solver", func() {
 			}
 			s = NewSolver(dbInstalled, dbDefinitions, db)
 
-			solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"})
+			solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"}, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(solution).To(ContainElement(A.IsFlagged(false)))
@@ -674,7 +674,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			solution, err := s.Uninstall(A)
+			solution, err := s.Uninstall(A, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(solution).To(ContainElement(A.IsFlagged(false)))
@@ -698,7 +698,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			solution, err := s.Uninstall(A)
+			solution, err := s.Uninstall(A, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(solution).To(ContainElement(A.IsFlagged(false)))
@@ -721,7 +721,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			solution, err := s.Uninstall(A)
+			solution, err := s.Uninstall(A, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(solution).To(ContainElement(A.IsFlagged(false)))
@@ -746,7 +746,7 @@ var _ = Describe("Solver", func() {
 				Expect(err).ToNot(HaveOccurred())
 			}
 
-			solution, err := s.Uninstall(A)
+			solution, err := s.Uninstall(A, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(solution).To(ContainElement(A.IsFlagged(false)))
@@ -772,7 +772,7 @@ var _ = Describe("Solver", func() {
 				Expect(err).ToNot(HaveOccurred())
 			}
 
-			solution, err := s.Uninstall(A)
+			solution, err := s.Uninstall(A, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(solution).To(ContainElement(A.IsFlagged(false)))
@@ -893,7 +893,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			uninstall, solution, err := s.Upgrade()
+			uninstall, solution, err := s.Upgrade(true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(uninstall)).To(Equal(1))

tests/fixtures/retrieve-integration/a/build.yaml

@@ -0,0 +1,3 @@
+image: "alpine"
+steps:
+  - echo a > /a

tests/fixtures/retrieve-integration/a/definition.yaml

@@ -0,0 +1,3 @@
+name: "a"
+version: "1.0"
+category: "test"

tests/fixtures/retrieve-integration/b/build.yaml

@@ -0,0 +1,10 @@
+
+steps:
+- tar xvf a-test-1.0.package.* -C ./
+- mv a /b
+requires:
+- name: "a"
+  version: "1.0"
+  category: "test"
+retrieve:
+- a-test-1.0.package.*

tests/fixtures/retrieve-integration/b/definition.yaml

@@ -0,0 +1,7 @@
+name: "b"
+version: "1.0"
+category: "test"
+requires:
+- name: "a"
+  version: "1.0"
+  category: "test"

tests/fixtures/retrieve/a/build.yaml

@@ -0,0 +1,10 @@
+image: "luet/base"
+seed: "alpine"
+steps:
+  - echo foo > /test
+  - echo bar > /test2
+retrieve:
+- test
+- http://www.google.com
+env:
+- test=1

tests/fixtures/retrieve/a/definition.yaml

@@ -0,0 +1,3 @@
+name: "a"
+version: "1.0"
+category: "test"

tests/fixtures/upgrade_integration/c/build.yaml

@@ -0,0 +1,10 @@
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo c > /c
+  - echo c > /cd
+requires:
+- category: "test"
+  name: "a"
+  version: ">=1.0"

tests/fixtures/upgrade_integration/c/definition.yaml

@@ -0,0 +1,9 @@
+category: "test"
+name: "c"
+version: "1.0"
+# Boom?
+
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0.1"

tests/fixtures/upgrade_integration/cat/a/a/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/upgrade_integration/cat/a/a/definition.yaml

@@ -0,0 +1,8 @@
+category: "test"
+name: "a"
+version: "1.1"
+requires:
+- category: "test"
+  name: "b"
+  version: ">=0.1"
+

tests/fixtures/upgrade_integration/cat/a/a1.0/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /testaa
+  - echo artifact4 > /testaa2
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/upgrade_integration/cat/a/a1.0/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.0"

tests/fixtures/upgrade_integration/cat/a/alatest/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /testlatest
+  - echo artifact4 > /testlatest2
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/upgrade_integration/cat/a/alatest/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/upgrade_integration/cat/b-1.1/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+  - chmod +x generate.sh
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - ./generate.sh

tests/fixtures/upgrade_integration/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/upgrade_integration/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/fixtures/upgrade_integration/cat/b/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+  - chmod +x generate.sh
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6
+  - ./generate.sh

tests/fixtures/upgrade_integration/cat/b/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/fixtures/upgrade_integration/cat/b/generate.sh

@@ -0,0 +1 @@
+echo generated > /artifact42

tests/integration/01_simple.sh

@@ -71,7 +71,7 @@ testReInstall() {
 }
 
 testUnInstall() {
-    luet uninstall --config $tmpdir/luet.yaml test/c-1.0
+    luet uninstall --config $tmpdir/luet.yaml test/c
     installst=$?
     assertEquals 'uninstall test successfully' "$installst" "0"
     assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"

tests/integration/03_qlearning.sh

@@ -12,7 +12,7 @@ oneTimeTearDown() {
 
 testBuild() {
     mkdir $tmpdir/testbuild
-    luet build --all --tree "$ROOT_DIR/tests/fixtures/qlearning" --destination $tmpdir/testbuild --compression gzip
+    luet build --all --concurrency 1 --tree "$ROOT_DIR/tests/fixtures/qlearning" --destination $tmpdir/testbuild --compression gzip
     buildst=$?
     assertEquals 'builds successfully' "$buildst" "0"
     assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"

tests/integration/04_retrieve.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/retrieve-integration" --destination $tmpdir/testbuild --compression gzip test/b
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/retrieve-integration" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/b
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package B installed' "[ -e '$tmpdir/testrootfs/b' ]"
+    val=$(cat "$tmpdir/testrootfs/b")
+    assertEquals 'package B content comes from a' "$val" "a"
+    assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/a' ]"
+}
+
+
+testUnInstall() {
+    luet uninstall --config $tmpdir/luet.yaml test/b
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/b' ]"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/a' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/b-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/05_upgrade.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/b-1.0 
+    buildst=$?
+    assertTrue 'create package B 1.0' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/b-1.1
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package B 1.1' "[ -e '$tmpdir/testbuild/b-test-1.1.package.tar.gz' ]"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/a-1.0 
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package A 1.0' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/a-1.1
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    assertTrue 'create package A 1.1' "[ -e '$tmpdir/testbuild/a-test-1.1.package.tar.gz' ]"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/a-1.2
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    assertTrue 'create package A 1.2' "[ -e '$tmpdir/testbuild/a-test-1.2.package.tar.gz' ]"
+
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/c-1.0
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package C 1.0' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/b-1.0
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/test5' ]"
+
+    luet install --config $tmpdir/luet.yaml test/a-1.0
+    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    luet install --config $tmpdir/luet.yaml test/a-1.1
+    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package keeps old A' "[ -e '$tmpdir/testrootfs/testaa' ]"
+    assertTrue 'package new A was not installed' "[ ! -e '$tmpdir/testrootfs/testlatest' ]"
+
+    luet install --config $tmpdir/luet.yaml test/c-1.0
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed C' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testUpgrade() {
+    luet --config $tmpdir/luet.yaml upgrade
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package uninstalled B' "[ ! -e '$tmpdir/testrootfs/test5' ]"
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/newc' ]"
+    assertTrue 'package uninstalled A' "[ ! -e '$tmpdir/testrootfs/testaa' ]"
+    assertTrue 'package installed new A' "[ -e '$tmpdir/testrootfs/testlatest' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

vendor/github.com/Sabayon/pkgs-checker/pkg/gentoo/pkg.go

@@ -54,11 +54,17 @@ const (
 	PkgCondMatchVersion = 8
 )
 
+const (
+	RegexCatString     = `(^[a-z]+[0-9]*[a-z]*[-][a-z]+[0-9]*[a-z]*|^virtual)`
+	RegexPkgNameString = `([a-z]+[0-9a-zA-Z\-[+]*]*[+]*)`
+)
+
 type GentooPackage struct {
 	Name          string `json:"name",omitempty"`
 	Category      string `json:"category",omitempty"`
 	Version       string `json:"version",omitempty"`
 	VersionSuffix string `json:"version_suffix",omitempty"`
+	VersionBuild  string `json:"version_build",omitempty"`
 	Slot          string `json:"slot",omitempty"`
 	Condition     PackageCond
 	Repository    string   `json:"repository",omitempty"`
@@ -191,14 +197,23 @@ func (p *GentooPackage) Admit(i *GentooPackage) (bool, error) {
 			fmt.Sprintf("Wrong name for package %s", i.Name))
 	}
 
-	if p.Version != "" {
-		v1, err = version.NewVersion(p.Version)
+	v1s := p.Version
+	v2s := i.Version
+
+	if v1s != "" {
+		if p.VersionBuild != "" {
+			v1s = p.Version + "+" + p.VersionBuild
+		}
+		v1, err = version.NewVersion(v1s)
 		if err != nil {
 			return false, err
 		}
 	}
-	if i.Version != "" {
-		v2, err = version.NewVersion(i.Version)
+	if v2s != "" {
+		if i.VersionBuild != "" {
+			v2s = i.Version + "+" + i.VersionBuild
+		}
+		v2, err = version.NewVersion(v2s)
 		if err != nil {
 			return false, err
 		}
@@ -210,7 +225,8 @@ func (p *GentooPackage) Admit(i *GentooPackage) (bool, error) {
 	} else {
 		if p.Condition == PkgCondInvalid || p.Condition == PkgCondEqual {
 			// case 1: source-pkg-1.0 and dest-pkg-1.0 or dest-pkg without version
-			if i.Version != "" && i.Version == p.Version && p.VersionSuffix == i.VersionSuffix {
+			if i.Version != "" && i.Version == p.Version && p.VersionSuffix == i.VersionSuffix &&
+				p.VersionBuild == i.VersionBuild {
 				ans = true
 			}
 		} else if p.Condition == PkgCondAnyRevision {
@@ -270,8 +286,9 @@ func ParsePackageStr(pkg string) (*GentooPackage, error) {
 	}
 
 	ans := GentooPackage{
-		Slot:      "0",
-		Condition: PkgCondInvalid,
+		Slot:         "0",
+		Condition:    PkgCondInvalid,
+		VersionBuild: "",
 	}
 
 	// Check if pkg string contains inline use flags
@@ -315,6 +332,67 @@ func ParsePackageStr(pkg string) (*GentooPackage, error) {
 		ans.Condition = PkgCondNot
 	}
 
+	regexVerString := fmt.Sprintf("[-](%s|%s|%s|%s|%s|%s)((%s|%s|%s|%s|%s|%s|%s)+)*",
+		// Version regex
+		// 1.1
+		"[0-9]+[.][0-9]+[a-z]*",
+		// 1
+		"[0-9]+[a-z]*",
+		// 1.1.1
+		"[0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+		// 1.1.1.1
+		"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+		// 1.1.1.1.1
+		"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+		// 1.1.1.1.1.1
+		"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
+		// suffix
+		"-r[0-9]+",
+		"_p[0-9]+",
+		"_pre[0-9]*",
+		"_rc[0-9]+",
+		// handle also rc without number
+		"_rc",
+		"_alpha",
+		"_beta",
+	)
+
+	hasBuild, _ := regexp.MatchString(
+		fmt.Sprintf("(%s[/]%s%s([[:]{1,2}[0-9a-zA-Z]*]*)*[+])",
+			RegexCatString, RegexPkgNameString, regexVerString),
+		pkg,
+	)
+
+	if hasBuild {
+		// Check if build number is present
+		buildIdx := strings.LastIndex(pkg, "+")
+		if buildIdx > 0 {
+			// <pre-release> ::= <dot-separated pre-release identifiers>
+			//
+			// <dot-separated pre-release identifiers> ::=
+			//      <pre-release identifier> | <pre-release identifier> "."
+			//      <dot-separated pre-release identifiers>
+			//
+			// <build> ::= <dot-separated build identifiers>
+			//
+			// <dot-separated build identifiers> ::= <build identifier>
+			//      | <build identifier> "." <dot-separated build identifiers>
+			//
+			// <pre-release identifier> ::= <alphanumeric identifier>
+			//                            | <numeric identifier>
+			//
+			// <build identifier> ::= <alphanumeric identifier>
+			//      | <digits>
+			//
+			// <alphanumeric identifier> ::= <non-digit>
+			//      | <non-digit> <identifier characters>
+			//      | <identifier characters> <non-digit>
+			//      | <identifier characters> <non-digit> <identifier characters>
+			ans.VersionBuild = pkg[buildIdx+1:]
+			pkg = pkg[0:buildIdx]
+		}
+	}
+
 	words := strings.Split(pkg, "/")
 	if len(words) != 2 {
 		return nil, errors.New(fmt.Sprintf("Invalid package string %s", pkg))
@@ -337,31 +415,9 @@ func ParsePackageStr(pkg string) (*GentooPackage, error) {
 	}
 
 	regexPkg := regexp.MustCompile(
-		fmt.Sprintf("[-](%s|%s|%s|%s|%s|%s)((%s|%s|%s|%s|%s|%s|%s)+)*$",
-			// Version regex
-			// 1.1
-			"[0-9]+[.][0-9]+[a-z]*",
-			// 1
-			"[0-9]+[a-z]*",
-			// 1.1.1
-			"[0-9]+[.][0-9]+[.][0-9]+[a-z]*",
-			// 1.1.1.1
-			"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
-			// 1.1.1.1.1
-			"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
-			// 1.1.1.1.1.1
-			"[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*",
-			// suffix
-			"-r[0-9]+",
-			"_p[0-9]+",
-			"_pre[0-9]*",
-			"_rc[0-9]+",
-			// handle also rc without number
-			"_rc",
-			"_alpha",
-			"_beta",
-		),
+		fmt.Sprintf("%s$", regexVerString),
 	)
+
 	matches := regexPkg.FindAllString(pkgname, -1)
 
 	// NOTE: Now suffix comples like _alpha_rc1 are not supported.

vendor/modules.txt

@@ -23,7 +23,7 @@ github.com/Microsoft/hcsshim/internal/longpath
 github.com/Microsoft/hcsshim/internal/safefile
 # github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5
 github.com/Nvveen/Gotty
-# github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7
+# github.com/Sabayon/pkgs-checker v0.5.1-0.20200221202320-073693f2c657
 github.com/Sabayon/pkgs-checker/pkg/gentoo
 # github.com/apex/log v1.1.1
 github.com/apex/log