Tag 0.8.8

- Allows to override the system db and create/remove entries as desired.
  The input format is the same metadata as the one generated by the
  artifacts. It contains the Package and the file list that we need.
- Add integration test

Closes #47

.github/workflows/release.yml

@@ -0,0 +1,23 @@
+on: push
+name: Build and release on push
+jobs:
+  release:
+    name: Test and Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Tests
+        run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage
+      - name: Build
+        run: sudo -E env "PATH=$PATH" make multiarch-build && sudo chmod -R 777 release/
+      - name: Release
+        uses: fnkr/github-action-ghr@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GHR_PATH: release/
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+
+on: pull_request
+name: Build and Test
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage

.travis.yml

@@ -6,14 +6,14 @@ go:
 env:
   - "GO15VENDOREXPERIMENT=1"
 before_install:
-  - make deps
-  - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && mkdir -p $HOME/bin && export PATH=$PATH:$HOME/bin && mv container-diff-linux-amd64 $HOME/bin/container-diff
+  - sudo -E env "PATH=$PATH" apt-get install -y libcap2-bin
+  - sudo -E env "PATH=$PATH" make deps
 script:
-  - make multiarch-build test-integration test-coverage
-after_success:
-  - |
-      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
-        git config --global user.name "Deployer" && git config --global user.email foo@bar.com
-        go get github.com/tcnksm/ghr
-        ghr -u mudler -r luet --replace $TRAVIS_TAG release/
-      fi
+  - sudo -E env "PATH=$PATH" make multiarch-build test-integration test-coverage
+#after_success:
+#  - |
+#      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+#        sudo -E env "PATH=$PATH" git config --global user.name "Deployer" && git config --global user.email foo@bar.com
+#        sudo -E env "PATH=$PATH" go get github.com/tcnksm/ghr
+#        sudo -E env "PATH=$PATH" ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+#      fi

Dockerfile

@@ -1,6 +1,7 @@
 FROM golang as builder
+RUN apt-get update && apt-get install -y upx
 ADD . /luet
-RUN cd /luet && make build
+RUN cd /luet && make build-small
 
 FROM scratch
 ENV LUET_NOLOCK=true

Makefile

@@ -1,7 +1,7 @@
 
 # go tool nm ./luet | grep Commit
-LDFLAGS += -X "github.com/mudler/luet/cmd.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S %Z')"
-LDFLAGS += -X "github.com/mudler/luet/cmd.BuildCommit=$(shell git rev-parse HEAD)"
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S %Z')"
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildCommit=$(shell git rev-parse HEAD)"
 
 NAME ?= luet
 PACKAGE_NAME ?= $(NAME)
@@ -9,7 +9,6 @@ PACKAGE_CONFLICT ?= $(PACKAGE_NAME)-beta
 REVISION := $(shell git rev-parse --short HEAD || echo dev)
 VERSION := $(shell git describe --tags || echo $(REVISION))
 VERSION := $(shell echo $(VERSION) | sed -e 's/^v//g')
-ITTERATION := $(shell date +%s)
 BUILD_PLATFORMS ?= -osarch="linux/amd64" -osarch="linux/386" -osarch="linux/arm"
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
@@ -66,14 +65,15 @@ deps:
 build:
 	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)'
 
+.PHONY: build-small
+build-small:
+	@$(MAKE) LDFLAGS+="-s -w" build
+	upx --brute -1 $(NAME)
+
 .PHONY: image
 image:
 	docker build --rm -t luet/base .
 
-.PHONY: gox-build
-gox-build:
-	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
-
 .PHONY: lint
 lint:
 	golint ./... | grep -v "be unexported"
@@ -90,4 +90,4 @@ test-docker:
 
 .PHONY: multiarch-build
 multiarch-build:
-	gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+	CGO_ENABLED=0 gox $(BUILD_PLATFORMS) -ldflags '$(LDFLAGS)' -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"

cmd/build.go

@@ -18,10 +18,10 @@ import (
 	"io/ioutil"
 	"os"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	tree "github.com/mudler/luet/pkg/tree"
@@ -78,6 +78,9 @@ var buildCmd = &cobra.Command{
 		nodeps := viper.GetBool("nodeps")
 		onlydeps := viper.GetBool("onlydeps")
 		keepExportedImages := viper.GetBool("keep-exported-images")
+		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
+		full, _ := cmd.Flags().GetBool("full")
+		skip, _ := cmd.Flags().GetBool("skip-if-metadata-exists")
 
 		compilerSpecs := compiler.NewLuetCompilationspecs()
 		var compilerBackend compiler.CompilerBackend
@@ -142,11 +145,23 @@ var buildCmd = &cobra.Command{
 		opts.OnlyDeps = onlydeps
 		opts.NoDeps = nodeps
 		opts.KeepImageExport = keepExportedImages
+		opts.SkipIfMetadataExists = skip
+		opts.PackageTargetOnly = onlyTarget
 
 		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts)
 		luetCompiler.SetConcurrency(concurrency)
 		luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType))
-		if !all {
+		if full {
+			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
+			if err != nil {
+				Fatal(err.Error())
+			}
+			for _, spec := range specs {
+				Info(":package: Selecting ", spec.GetPackage().GetName(), spec.GetPackage().GetVersion())
+
+				compilerSpecs.Add(spec)
+			}
+		} else if !all {
 			for _, a := range args {
 
 				pack, err := helpers.ParsePackageStr(a)
@@ -208,7 +223,9 @@ func init() {
 	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
 	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
-	buildCmd.Flags().Bool("all", false, "Build all packages in the tree")
+	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
+	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
+
 	buildCmd.Flags().String("destination", path, "Destination folder")
 	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
 	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
@@ -218,7 +235,8 @@ func init() {
 	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
 	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
 	buildCmd.Flags().Bool("keep-exported-images", false, "Keep exported images used during building")
-
+	buildCmd.Flags().Bool("skip-if-metadata-exists", false, "Skip package if metadata exists")
+	buildCmd.Flags().Bool("only-target-package", false, "Build packages of only the required target. Otherwise builds all the necessary ones not present in the destination")
 	buildCmd.Flags().String("solver-type", "", "Solver strategy")
 	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
 	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")

cmd/config.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	config "github.com/mudler/luet/pkg/config"
+	installer "github.com/mudler/luet/pkg/installer"
 
 	"github.com/spf13/cobra"
 )
@@ -52,6 +53,23 @@ var configCmd = &cobra.Command{
 			}
 		}
 
+		if len(config.LuetCfg.ConfigProtectConfDir) > 0 {
+
+			// Load config protect configs
+			installer.LoadConfigProtectConfs(config.LuetCfg)
+
+			fmt.Println("config_protect_confdir:")
+			for _, dir := range config.LuetCfg.ConfigProtectConfDir {
+				fmt.Println("  - ", dir)
+			}
+
+			if len(config.LuetCfg.GetConfigProtectConfFiles()) > 0 {
+				fmt.Println("protect_conf_files:")
+				for _, file := range config.LuetCfg.GetConfigProtectConfFiles() {
+					fmt.Println("  - ", file.String())
+				}
+			}
+		}
 	},
 }
 

cmd/convert.go

@@ -28,7 +28,7 @@ import (
 )
 
 var convertCmd = &cobra.Command{
-	Use:   "convert",
+	Use:   "convert [portage-tree] [luet-tree]",
 	Short: "convert other package manager tree into luet",
 	Long:  `Parses external PM and produces a luet parsable tree`,
 	PreRun: func(cmd *cobra.Command, args []string) {

cmd/database.go

@@ -0,0 +1,37 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/database"
+
+	"github.com/spf13/cobra"
+)
+
+var databaseGroupCmd = &cobra.Command{
+	Use:   "database [command] [OPTIONS]",
+	Short: "Manage system database (dangerous commands ahead!)",
+}
+
+func init() {
+	RootCmd.AddCommand(databaseGroupCmd)
+
+	databaseGroupCmd.AddCommand(
+		NewDatabaseCreateCommand(),
+		NewDatabaseRemoveCommand(),
+	)
+}

cmd/database/create.go

@@ -0,0 +1,81 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/mudler/luet/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseCreateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "create <artifact_metadata1.yaml> <artifact_metadata1.yaml>",
+		Short: "Insert a package in the system DB",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			var systemDB pkg.PackageDatabase
+
+			for _, a := range args {
+				dat, err := ioutil.ReadFile(a)
+				if err != nil {
+					Fatal("Failed reading ", a, ": ", err.Error())
+				}
+				art, err := compiler.NewPackageArtifactFromYaml(dat)
+				if err != nil {
+					Fatal("Failed reading yaml ", a, ": ", err.Error())
+				}
+
+				if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+					systemDB = pkg.NewBoltDatabase(
+						filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+				} else {
+					systemDB = pkg.NewInMemoryDatabase(true)
+				}
+
+				files, err := art.FileList()
+				if err != nil {
+					Fatal("Failed getting file list for ", a, ": ", err.Error())
+				}
+
+				if _, err := systemDB.CreatePackage(art.GetCompileSpec().GetPackage()); err != nil {
+					Fatal("Failed to create ", a, ": ", err.Error())
+				}
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.GetCompileSpec().GetPackage().GetFingerPrint(), Files: files}); err != nil {
+					Fatal("Failed setting package files for ", a, ": ", err.Error())
+				}
+
+				Info(art.GetCompileSpec().GetPackage().HumanReadableString(), " created")
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/database/remove.go

@@ -0,0 +1,69 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"path/filepath"
+
+	. "github.com/mudler/luet/pkg/logger"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	. "github.com/mudler/luet/pkg/config"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseRemoveCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "remove [package1] [package2] ...",
+		Short: "Remove a package from the system DB (forcefully - you normally don't want to do that)",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+			LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var systemDB pkg.PackageDatabase
+
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					Fatal("Invalid package string ", a, ": ", err.Error())
+				}
+
+				if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
+					systemDB = pkg.NewBoltDatabase(
+						filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+				} else {
+					systemDB = pkg.NewInMemoryDatabase(true)
+				}
+
+				if err := systemDB.RemovePackage(pack); err != nil {
+					Fatal("Failed removing ", a, ": ", err.Error())
+				}
+
+				if err := systemDB.RemovePackageFiles(pack); err != nil {
+					Fatal("Failed removing files for ", a, ": ", err.Error())
+				}
+			}
+
+		},
+	}
+
+	return ans
+}

cmd/helpers/cli.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program; if not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package cmd_helpers
 
 import (
 	"errors"

cmd/install.go

@@ -20,8 +20,8 @@ import (
 
 	installer "github.com/mudler/luet/pkg/installer"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -80,6 +80,9 @@ var installCmd = &cobra.Command{
 
 		Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
 			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
 			SolverOptions:               *LuetCfg.GetSolverOptions(),

cmd/root.go

@@ -24,11 +24,11 @@ import (
 	"strings"
 
 	"github.com/marcsauter/single"
+	extensions "github.com/mudler/cobra-extensions"
 	config "github.com/mudler/luet/pkg/config"
 	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	repo "github.com/mudler/luet/pkg/repository"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -38,7 +38,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.7.9"
+	LuetCLIVersion = "0.8.8"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -88,8 +88,12 @@ func LoadConfig(c *config.LuetConfig) error {
 		return err
 	}
 
+	noSpinner := c.Viper.GetBool("no_spinner")
+
 	InitAurora()
-	NewSpinner()
+	if !noSpinner {
+		NewSpinner()
+	}
 
 	Debug("Using config file:", c.Viper.ConfigFileUsed())
 
@@ -133,15 +137,6 @@ func Execute() {
 	}
 
 	if err := RootCmd.Execute(); err != nil {
-		if len(os.Args) > 0 {
-			for _, c := range RootCmd.Commands() {
-				if c.Name() == os.Args[1] {
-					os.Exit(-1) // Something failed
-				}
-			}
-			// Try to load a bin from path.
-			helpers.Exec("luet-"+os.Args[1], os.Args[1:], os.Environ())
-		}
 		fmt.Println(err)
 		os.Exit(-1)
 	}
@@ -154,22 +149,21 @@ func init() {
 	pflags.BoolP("debug", "d", false, "verbose output")
 	pflags.Bool("fatal", false, "Enables Warnings to exit")
 	pflags.Bool("enable-logfile", false, "Enable log to file")
+	pflags.Bool("no-spinner", false, "Disable spinner.")
 	pflags.Bool("color", config.LuetCfg.GetLogging().Color, "Enable/Disable color.")
 	pflags.Bool("emoji", config.LuetCfg.GetLogging().EnableEmoji, "Enable/Disable emoji.")
+	pflags.Bool("skip-config-protect", config.LuetCfg.ConfigProtectSkip,
+		"Disable config protect analysis.")
 	pflags.StringP("logfile", "l", config.LuetCfg.GetLogging().Path,
 		"Logfile path. Empty value disable log to file.")
 
-	sameOwner := false
-	u, err := user.Current()
-	// os/user doesn't work in from scratch environments
+	// os/user doesn't work in from scratch environments.
+	// Check if i can retrieve user informations.
+	_, err := user.Current()
 	if err != nil {
 		Warning("failed to retrieve user identity:", err.Error())
-		sameOwner = true
 	}
-	if u != nil && u.Uid == "0" {
-		sameOwner = true
-	}
-	pflags.Bool("same-owner", sameOwner, "Maintain same owner on uncompress.")
+	pflags.Bool("same-owner", config.LuetCfg.GetGeneral().SameOwner, "Maintain same owner on uncompress.")
 	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
 
 	config.LuetCfg.Viper.BindPFlag("logging.color", pflags.Lookup("color"))
@@ -181,26 +175,54 @@ func init() {
 	config.LuetCfg.Viper.BindPFlag("general.debug", pflags.Lookup("debug"))
 	config.LuetCfg.Viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
 	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
+	// Currently I maintain this only from cli.
+	config.LuetCfg.Viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))
+	config.LuetCfg.Viper.BindPFlag("config_protect_skip", pflags.Lookup("skip-config-protect"))
+
+	// Extensions must be binary with the "luet-" prefix to be able to be shown in the help.
+	// we also accept extensions in the relative path where luet is being started, "extensions/"
+	exts := extensions.Discover("luet", "extensions")
+	for _, ex := range exts {
+		cobraCmd := ex.CobraCommand()
+		RootCmd.AddCommand(cobraCmd)
+	}
+
 }
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
+	// Luet support these priorities on read configuration file:
+	// - command line option (if available)
+	// - $PWD/.luet.yaml
+	// - $HOME/.luet.yaml
+	// - /etc/luet/luet.yaml
+	//
+	// Note: currently a single viper instance support only one config name.
 
-	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
-	if err != nil {
-		Error(err)
-		os.Exit(1)
-	}
 	viper.SetEnvPrefix(LuetEnvPrefix)
 	viper.SetConfigType("yaml")
-	viper.SetConfigName(".luet") // name of config file (without extension)
-	if cfgFile != "" {           // enable ability to specify config file via flag
+
+	if cfgFile != "" { // enable ability to specify config file via flag
 		viper.SetConfigFile(cfgFile)
 	} else {
-		viper.AddConfigPath(dir)
-		viper.AddConfigPath(".")
-		viper.AddConfigPath("$HOME")
-		viper.AddConfigPath("/etc/luet")
+		// Retrieve pwd directory
+		pwdDir, err := os.Getwd()
+		if err != nil {
+			Error(err)
+			os.Exit(1)
+		}
+		homeDir := helpers.GetHomeDir()
+
+		if helpers.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && helpers.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
+			viper.AddConfigPath(".")
+			if homeDir != "" {
+				viper.AddConfigPath(homeDir)
+			}
+			viper.SetConfigName(".luet")
+		} else {
+			viper.SetConfigName("luet")
+			viper.AddConfigPath("/etc/luet")
+		}
 	}
 
 	viper.AutomaticEnv() // read in environment variables that match

cmd/search.go

@@ -32,6 +32,7 @@ type PackageResult struct {
 	Category   string `json:"category"`
 	Version    string `json:"version"`
 	Repository string `json:"repository"`
+	Hidden     bool   `json:"hidden"`
 }
 
 type Results struct {
@@ -58,6 +59,9 @@ var searchCmd = &cobra.Command{
 		if len(args) != 1 {
 			Fatal("Wrong number of arguments (expected 1)")
 		}
+
+		hidden, _ := cmd.Flags().GetBool("hidden")
+
 		installed := LuetCfg.Viper.GetBool("installed")
 		stype := LuetCfg.Viper.GetString("solver.type")
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
@@ -114,25 +118,31 @@ var searchCmd = &cobra.Command{
 			}
 			for _, m := range matches {
 				if !revdeps {
-					Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
-					results.Packages = append(results.Packages,
-						PackageResult{
-							Name:       m.Package.GetName(),
-							Version:    m.Package.GetVersion(),
-							Category:   m.Package.GetCategory(),
-							Repository: m.Repo.GetName(),
-						})
+					if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
+						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
+						results.Packages = append(results.Packages,
+							PackageResult{
+								Name:       m.Package.GetName(),
+								Version:    m.Package.GetVersion(),
+								Category:   m.Package.GetCategory(),
+								Repository: m.Repo.GetName(),
+								Hidden:     m.Package.IsHidden(),
+							})
+					}
 				} else {
 					visited := make(map[string]interface{})
 					for _, revdep := range m.Package.ExpandedRevdeps(m.Repo.GetTree().GetDatabase(), visited) {
-						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
-						results.Packages = append(results.Packages,
-							PackageResult{
-								Name:       revdep.GetName(),
-								Version:    revdep.GetVersion(),
-								Category:   revdep.GetCategory(),
-								Repository: m.Repo.GetName(),
-							})
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: m.Repo.GetName(),
+									Hidden:     revdep.IsHidden(),
+								})
+						}
 					}
 				}
 			}
@@ -162,26 +172,32 @@ var searchCmd = &cobra.Command{
 
 			for _, pack := range iMatches {
 				if !revdeps {
-					Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
-					results.Packages = append(results.Packages,
-						PackageResult{
-							Name:       pack.GetName(),
-							Version:    pack.GetVersion(),
-							Category:   pack.GetCategory(),
-							Repository: "system",
-						})
+					if !pack.IsHidden() || pack.IsHidden() && hidden {
+						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+						results.Packages = append(results.Packages,
+							PackageResult{
+								Name:       pack.GetName(),
+								Version:    pack.GetVersion(),
+								Category:   pack.GetCategory(),
+								Repository: "system",
+								Hidden:     pack.IsHidden(),
+							})
+					}
 				} else {
 					visited := make(map[string]interface{})
 
 					for _, revdep := range pack.ExpandedRevdeps(system.Database, visited) {
-						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
-						results.Packages = append(results.Packages,
-							PackageResult{
-								Name:       revdep.GetName(),
-								Version:    revdep.GetVersion(),
-								Category:   revdep.GetCategory(),
-								Repository: "system",
-							})
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: "system",
+									Hidden:     revdep.IsHidden(),
+								})
+						}
 					}
 				}
 			}
@@ -223,5 +239,7 @@ func init() {
 	searchCmd.Flags().Bool("by-label", false, "Search packages through label")
 	searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex")
 	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
+
 	RootCmd.AddCommand(searchCmd)
 }

cmd/tree/bump.go

@@ -18,11 +18,11 @@ package cmd_tree
 
 import (
 	"fmt"
-	//"os"
-	//"sort"
 
 	. "github.com/mudler/luet/pkg/logger"
+	spectooling "github.com/mudler/luet/pkg/spectooling"
 	tree "github.com/mudler/luet/pkg/tree"
+	version "github.com/mudler/luet/pkg/versioner"
 
 	"github.com/spf13/cobra"
 )
@@ -42,19 +42,28 @@ func NewTreeBumpCommand() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			spec, _ := cmd.Flags().GetString("definition-file")
 			toStdout, _ := cmd.Flags().GetBool("to-stdout")
+			pkgVersion, _ := cmd.Flags().GetString("pkg-version")
 			pack, err := tree.ReadDefinitionFile(spec)
 			if err != nil {
 				Fatal(err.Error())
 			}
 
-			// Retrieve version build section with Gentoo parser
-			err = pack.BumpBuildVersion()
-			if err != nil {
-				Fatal("Error on increment build version: " + err.Error())
+			if pkgVersion != "" {
+				validator := &version.WrappedVersioner{}
+				err := validator.Validate(pkgVersion)
+				if err != nil {
+					Fatal("Invalid version string: " + err.Error())
+				}
+				pack.SetVersion(pkgVersion)
+			} else {
+				// Retrieve version build section with Gentoo parser
+				err = pack.BumpBuildVersion()
+				if err != nil {
+					Fatal("Error on increment build version: " + err.Error())
+				}
 			}
-
 			if toStdout {
-				data, err := pack.Yaml()
+				data, err := spectooling.NewDefaultPackageSanitized(&pack).Yaml()
 				if err != nil {
 					Fatal("Error on yaml conversion: " + err.Error())
 				}
@@ -71,6 +80,7 @@ func NewTreeBumpCommand() *cobra.Command {
 		},
 	}
 
+	ans.Flags().StringP("pkg-version", "p", "", "Set a specific package version")
 	ans.Flags().StringP("definition-file", "f", "", "Path of the definition to bump.")
 	ans.Flags().BoolP("to-stdout", "o", false, "Bump package to output.")
 

cmd/tree/pkglist.go

@@ -22,10 +22,11 @@ import (
 
 	//. "github.com/mudler/luet/pkg/config"
 	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/spf13/cobra"
@@ -74,15 +75,24 @@ func NewTreePkglistCommand() *cobra.Command {
 			if len(t) == 0 {
 				Fatal("Mandatory tree param missing.")
 			}
+
+			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
+			if revdeps && deps {
+				Fatal("Both revdeps and deps option used. Choice only one.")
+			}
+
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			var results TreeResults
+			var depSolver solver.PackageSolver
 
 			treePath, _ := cmd.Flags().GetStringArray("tree")
 			verbose, _ := cmd.Flags().GetBool("verbose")
 			buildtime, _ := cmd.Flags().GetBool("buildtime")
 			full, _ := cmd.Flags().GetBool("full")
 			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
 
 			out, _ := cmd.Flags().GetString("output")
 			if out != "terminal" {
@@ -95,6 +105,7 @@ func NewTreePkglistCommand() *cobra.Command {
 			} else {
 				reciper = tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
 			}
+
 			for _, t := range treePath {
 				err := reciper.Load(t)
 				if err != nil {
@@ -102,6 +113,15 @@ func NewTreePkglistCommand() *cobra.Command {
 				}
 			}
 
+			if deps {
+				emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+
+				depSolver = solver.NewSolver(pkg.NewInMemoryDatabase(false),
+					reciper.GetDatabase(),
+					emptyInstallationDb)
+
+			}
+
 			regExcludes, err := helpers.CreateRegexArray(excludes)
 			if err != nil {
 				Fatal(err.Error())
@@ -146,15 +166,8 @@ func NewTreePkglistCommand() *cobra.Command {
 				}
 
 				if addPkg {
-					if !revdeps {
-						plist = append(plist, pkgstr)
-						results.Packages = append(results.Packages, TreePackageResult{
-							Name:     p.GetName(),
-							Version:  p.GetVersion(),
-							Category: p.GetCategory(),
-							Path:     p.GetPath(),
-						})
-					} else {
+					if revdeps {
+
 						visited := make(map[string]interface{})
 						for _, revdep := range p.ExpandedRevdeps(reciper.GetDatabase(), visited) {
 							if full {
@@ -172,7 +185,58 @@ func NewTreePkglistCommand() *cobra.Command {
 								Path:     revdep.GetPath(),
 							})
 						}
+					} else if deps {
+
+						Spinner(32)
+						solution, err := depSolver.Install(pkg.Packages{p})
+						if err != nil {
+							Fatal(err.Error())
+						}
+						ass := solution.SearchByName(p.GetPackageName())
+						solution, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+						if err != nil {
+							Fatal(err.Error())
+						}
+						SpinnerStop()
+
+						for _, pa := range solution {
+
+							if pa.Value {
+								// Exclude itself
+								if pa.Package.GetName() == p.GetName() && pa.Package.GetCategory() == p.GetCategory() {
+									continue
+								}
+
+								if full {
+									pkgstr = pkgDetail(pa.Package)
+								} else if verbose {
+									pkgstr = pa.Package.HumanReadableString()
+								} else {
+									pkgstr = fmt.Sprintf("%s/%s", pa.Package.GetCategory(), pa.Package.GetName())
+								}
+								plist = append(plist, pkgstr)
+								results.Packages = append(results.Packages, TreePackageResult{
+									Name:     pa.Package.GetName(),
+									Version:  pa.Package.GetVersion(),
+									Category: pa.Package.GetCategory(),
+									Path:     pa.Package.GetPath(),
+								})
+							}
+
+						}
+
+					} else {
+
+						plist = append(plist, pkgstr)
+						results.Packages = append(results.Packages, TreePackageResult{
+							Name:     p.GetName(),
+							Version:  p.GetVersion(),
+							Category: p.GetCategory(),
+							Path:     p.GetPath(),
+						})
+
 					}
+
 				}
 			}
 
@@ -192,7 +256,9 @@ func NewTreePkglistCommand() *cobra.Command {
 				}
 				fmt.Println(string(j2))
 			default:
-				sort.Strings(plist)
+				if !deps {
+					sort.Strings(plist)
+				}
 				for _, p := range plist {
 					fmt.Println(p)
 				}
@@ -204,6 +270,7 @@ func NewTreePkglistCommand() *cobra.Command {
 	ans.Flags().BoolP("buildtime", "b", false, "Build time match")
 	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
 	ans.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	ans.Flags().Bool("deps", false, "Search package dependencies")
 
 	ans.Flags().BoolP("verbose", "v", false, "Add package version")
 	ans.Flags().BoolP("full", "f", false, "Show package detail")

cmd/tree/validate.go

@@ -25,8 +25,8 @@ import (
 	"strconv"
 	"sync"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
@@ -35,248 +35,398 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func validateWorker(i int,
-	wg *sync.WaitGroup,
-	c <-chan pkg.Package,
-	reciper tree.Builder,
-	withSolver bool,
-	regExcludes, regMatches []*regexp.Regexp,
-	excludes, matches []string,
-	errs chan error) {
+type ValidateOpts struct {
+	WithSolver    bool
+	OnlyRuntime   bool
+	OnlyBuildtime bool
+	RegExcludes   []*regexp.Regexp
+	RegMatches    []*regexp.Regexp
+	Excludes      []string
+	Matches       []string
 
-	defer wg.Done()
+	// Runtime validate stuff
+	RuntimeCacheDeps *pkg.InMemoryDatabase
+	RuntimeReciper   *tree.InstallerRecipe
+
+	// Buildtime validate stuff
+	BuildtimeCacheDeps *pkg.InMemoryDatabase
+	BuildtimeReciper   *tree.CompilerRecipe
+
+	Mutex      sync.Mutex
+	BrokenPkgs int
+	BrokenDeps int
+
+	Errors []error
+}
+
+func (o *ValidateOpts) IncrBrokenPkgs() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenPkgs++
+}
+
+func (o *ValidateOpts) IncrBrokenDeps() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenDeps++
+}
+
+func (o *ValidateOpts) AddError(err error) {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.Errors = append(o.Errors, err)
+}
+
+func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, reciper tree.Builder, cacheDeps *pkg.InMemoryDatabase) error {
+	var errstr string
+	var ans error
 
 	var depSolver solver.PackageSolver
-	var cacheDeps *pkg.InMemoryDatabase
-	brokenPkgs := 0
-	brokenDeps := 0
-	var errstr string
 
-	emptyInstallationDb := pkg.NewInMemoryDatabase(false)
-	if withSolver {
+	if opts.WithSolver {
+		emptyInstallationDb := pkg.NewInMemoryDatabase(false)
 		depSolver = solver.NewSolver(pkg.NewInMemoryDatabase(false),
 			reciper.GetDatabase(),
 			emptyInstallationDb)
-
-		// Use Singleton in memory cache for speedup dependencies
-		// analysis
-		cacheDeps = pkg.NewInMemoryDatabase(true).(*pkg.InMemoryDatabase)
 	}
 
-	for p := range c {
+	found, err := reciper.GetDatabase().FindPackages(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  ">=0",
+		},
+	)
 
-		found, err := reciper.GetDatabase().FindPackages(
-			&pkg.DefaultPackage{
-				Name:     p.GetName(),
-				Category: p.GetCategory(),
-				Version:  ">=0",
-			},
+	if err != nil || len(found) < 1 {
+		if err != nil {
+			errstr = err.Error()
+		} else {
+			errstr = "No packages"
+		}
+		Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			errstr,
+		))
+
+		opts.IncrBrokenDeps()
+
+		return errors.New(
+			fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+				checkType,
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				errstr,
+			))
+	}
+
+	// Ensure that we use the right package from right recipier for deps
+	pReciper, err := reciper.GetDatabase().FindPackage(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  p.GetVersion(),
+		},
+	)
+	if err != nil {
+		errstr = fmt.Sprintf("[%9s] %s/%s-%s: Error on retrieve package - %s.",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			err.Error(),
 		)
+		Error(errstr)
 
-		if err != nil || len(found) < 1 {
+		return errors.New(errstr)
+	}
+	p = pReciper
+
+	pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
+		p.GetVersion())
+
+	validpkg := true
+
+	if len(opts.Matches) > 0 {
+		matched := false
+		for _, rgx := range opts.RegMatches {
+			if rgx.MatchString(pkgstr) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return nil
+		}
+	}
+
+	if len(opts.Excludes) > 0 {
+		excluded := false
+		for _, rgx := range opts.RegExcludes {
+			if rgx.MatchString(pkgstr) {
+				excluded = true
+				break
+			}
+		}
+
+		if excluded {
+			return nil
+		}
+	}
+
+	Info(fmt.Sprintf("[%9s] Checking package ", checkType)+
+		fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
+		"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
+
+	all := p.GetRequires()
+	all = append(all, p.GetConflicts()...)
+	for idx, r := range all {
+
+		var deps pkg.Packages
+		var err error
+		if r.IsSelector() {
+			deps, err = reciper.GetDatabase().FindPackages(
+				&pkg.DefaultPackage{
+					Name:     r.GetName(),
+					Category: r.GetCategory(),
+					Version:  r.GetVersion(),
+				},
+			)
+		} else {
+			deps = append(deps, r)
+		}
+
+		if err != nil || len(deps) < 1 {
 			if err != nil {
 				errstr = err.Error()
 			} else {
 				errstr = "No packages"
 			}
-			Error(fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
+			Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+				checkType,
 				p.GetCategory(), p.GetName(), p.GetVersion(),
+				r.GetCategory(), r.GetName(), r.GetVersion(),
 				errstr,
 			))
 
-			errs <- errors.New(
-				fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
-					p.GetCategory(), p.GetName(), p.GetVersion(),
-					errstr,
-				))
+			opts.IncrBrokenDeps()
 
-			brokenPkgs++
-		}
-
-		pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
-			p.GetVersion())
-
-		validpkg := true
-
-		if len(matches) > 0 {
-			matched := false
-			for _, rgx := range regMatches {
-				if rgx.MatchString(pkgstr) {
-					matched = true
-					break
-				}
-			}
-
-			if !matched {
-				continue
-			}
-		}
-
-		if len(excludes) > 0 {
-			excluded := false
-			for _, rgx := range regExcludes {
-				if rgx.MatchString(pkgstr) {
-					excluded = true
-					break
-				}
-			}
-
-			if excluded {
-				continue
-			}
-		}
-
-		Info("Checking package "+
-			fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
-			"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
-
-		all := p.GetRequires()
-		all = append(all, p.GetConflicts()...)
-		for idx, r := range all {
-
-			var deps pkg.Packages
-			var err error
-			if r.IsSelector() {
-				deps, err = reciper.GetDatabase().FindPackages(
-					&pkg.DefaultPackage{
-						Name:     r.GetName(),
-						Category: r.GetCategory(),
-						Version:  r.GetVersion(),
-					},
-				)
-			} else {
-				deps = append(deps, r)
-			}
-
-			if err != nil || len(deps) < 1 {
-				if err != nil {
-					errstr = err.Error()
-				} else {
-					errstr = "No packages"
-				}
-				Error(fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
+			ans = errors.New(
+				fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+					checkType,
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					r.GetCategory(), r.GetName(), r.GetVersion(),
+					errstr))
+
+			validpkg = false
+
+		} else {
+
+			Debug(fmt.Sprintf("[%9s] Find packages for dep", checkType),
+				fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+
+			if opts.WithSolver {
+
+				Info(fmt.Sprintf("[%9s]  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
+					checkType,
+					idx+1, len(all),
 					p.GetCategory(), p.GetName(), p.GetVersion(),
 					r.GetCategory(), r.GetName(), r.GetVersion(),
-					errstr,
 				))
 
-				errs <- errors.New(
-					fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
-						p.GetCategory(), p.GetName(), p.GetVersion(),
-						r.GetCategory(), r.GetName(), r.GetVersion(),
-						errstr))
-
-				brokenDeps++
-
-				validpkg = false
-
-			} else {
-
-				Debug("Find packages for dep",
-					fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
-
-				if withSolver {
-
-					Info(fmt.Sprintf("  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
-						idx+1, len(all),
+				// Check if the solver is already been done for the deep
+				_, err := cacheDeps.Get(r.HashFingerprint(""))
+				if err == nil {
+					Debug(fmt.Sprintf("[%9s]  :direct_hit: Cache Hit for dep", checkType),
+						fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+					continue
+				}
+
+				Spinner(32)
+				solution, err := depSolver.Install(pkg.Packages{r})
+				ass := solution.SearchByName(r.GetPackageName())
+				if err == nil {
+					_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+				}
+				SpinnerStop()
+
+				if err != nil {
+
+					Error(fmt.Sprintf("[%9s] %s/%s-%s: solver broken for dep %s/%s-%s - %s",
+						checkType,
 						p.GetCategory(), p.GetName(), p.GetVersion(),
 						r.GetCategory(), r.GetName(), r.GetVersion(),
+						err.Error(),
 					))
 
-					// Check if the solver is already been done for the deep
-					_, err := cacheDeps.Get(r.HashFingerprint())
-					if err == nil {
-						Debug("  :direct_hit: Cache Hit for dep",
-							fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
-						continue
-					}
-
-					Spinner(32)
-					solution, err := depSolver.Install(pkg.Packages{r})
-					ass := solution.SearchByName(r.GetPackageName())
-					if err == nil {
-						_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
-					}
-					SpinnerStop()
-
-					if err != nil {
-
-						Error(fmt.Sprintf("%s/%s-%s: solver broken for dep %s/%s-%s - %s",
+					ans = errors.New(
+						fmt.Sprintf("[%9s] %s/%s-%s: solver broken for Dep %s/%s-%s - %s",
+							checkType,
 							p.GetCategory(), p.GetName(), p.GetVersion(),
 							r.GetCategory(), r.GetName(), r.GetVersion(),
-							err.Error(),
-						))
-
-						errs <- errors.New(
-							fmt.Sprintf("%s/%s-%s: solver broken for Dep %s/%s-%s - %s",
-								p.GetCategory(), p.GetName(), p.GetVersion(),
-								r.GetCategory(), r.GetName(), r.GetVersion(),
-								err.Error()))
-
-						brokenDeps++
-						validpkg = false
-					}
-
-					// Register the key
-					cacheDeps.Set(r.HashFingerprint(), "1")
+							err.Error()))
 
+					opts.IncrBrokenDeps()
+					validpkg = false
 				}
+
+				// Register the key
+				cacheDeps.Set(r.HashFingerprint(""), "1")
+
+			}
+		}
+
+	}
+
+	if !validpkg {
+		opts.IncrBrokenPkgs()
+	}
+
+	return ans
+}
+
+func validateWorker(i int,
+	wg *sync.WaitGroup,
+	c <-chan pkg.Package,
+	opts *ValidateOpts) {
+
+	defer wg.Done()
+
+	for p := range c {
+
+		if opts.OnlyBuildtime {
+			// Check buildtime compiler/deps
+			err := validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+		} else if opts.OnlyRuntime {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+		} else {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+			// Check buildtime compiler/deps
+			err = validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
 			}
 
 		}
 
-		if !validpkg {
-			brokenPkgs++
+	}
+}
+
+func initOpts(opts *ValidateOpts, onlyRuntime, onlyBuildtime, withSolver bool, treePaths []string) {
+	var err error
+
+	opts.OnlyBuildtime = onlyBuildtime
+	opts.OnlyRuntime = onlyRuntime
+	opts.WithSolver = withSolver
+	opts.RuntimeReciper = nil
+	opts.BuildtimeReciper = nil
+	opts.BrokenPkgs = 0
+	opts.BrokenDeps = 0
+
+	if onlyBuildtime {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+	} else if onlyRuntime {
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	} else {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	}
+
+	opts.RuntimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+	opts.BuildtimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+
+	for _, treePath := range treePaths {
+		Info(fmt.Sprintf("Loading :deciduous_tree: %s...", treePath))
+		if opts.BuildtimeReciper != nil {
+			err = opts.BuildtimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
+		}
+		if opts.RuntimeReciper != nil {
+			err = opts.RuntimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
 		}
 	}
+
+	opts.RegExcludes, err = helpers.CreateRegexArray(opts.Excludes)
+	if err != nil {
+		Fatal(err.Error())
+	}
+	opts.RegMatches, err = helpers.CreateRegexArray(opts.Matches)
+	if err != nil {
+		Fatal(err.Error())
+	}
+
 }
 
 func NewTreeValidateCommand() *cobra.Command {
 	var excludes []string
 	var matches []string
 	var treePaths []string
+	var opts ValidateOpts
 
 	var ans = &cobra.Command{
 		Use:   "validate [OPTIONS]",
 		Short: "Validate a tree or a list of packages",
 		Args:  cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
 			if len(treePaths) < 1 {
 				Fatal("Mandatory tree param missing.")
 			}
+			if onlyRuntime && onlyBuildtime {
+				Fatal("Both --only-runtime and --only-buildtime options are not possibile.")
+			}
 		},
 		Run: func(cmd *cobra.Command, args []string) {
+			var reciper tree.Builder
+
 			concurrency := LuetCfg.GetGeneral().Concurrency
 
 			withSolver, _ := cmd.Flags().GetBool("with-solver")
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
 
-			reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
-			for _, treePath := range treePaths {
-				err := reciper.Load(treePath)
-				if err != nil {
-					Fatal("Error on load tree ", err)
-				}
-			}
+			opts.Excludes = excludes
+			opts.Matches = matches
+			initOpts(&opts, onlyRuntime, onlyBuildtime, withSolver, treePaths)
 
-			regExcludes, err := helpers.CreateRegexArray(excludes)
-			if err != nil {
-				Fatal(err.Error())
-			}
-			regMatches, err := helpers.CreateRegexArray(matches)
-			if err != nil {
-				Fatal(err.Error())
+			// We need at least one valid reciper for get list of the packages.
+			if onlyBuildtime {
+				reciper = opts.BuildtimeReciper
+			} else {
+				reciper = opts.RuntimeReciper
 			}
 
 			all := make(chan pkg.Package)
-			errs := make(chan error)
 
 			var wg = new(sync.WaitGroup)
 
 			for i := 0; i < concurrency; i++ {
 				wg.Add(1)
-				go validateWorker(i, wg, all,
-					reciper, withSolver, regExcludes, regMatches, excludes, matches,
-					errs)
+				go validateWorker(i, wg, all, &opts)
 			}
 			for _, p := range reciper.GetDatabase().World() {
 				all <- p
@@ -286,11 +436,10 @@ func NewTreeValidateCommand() *cobra.Command {
 			// Wait separately and once done close the channel
 			go func() {
 				wg.Wait()
-				close(errs)
 			}()
 
 			stringerrs := []string{}
-			for e := range errs {
+			for _, e := range opts.Errors {
 				stringerrs = append(stringerrs, e.Error())
 			}
 			sort.Strings(stringerrs)
@@ -300,9 +449,9 @@ func NewTreeValidateCommand() *cobra.Command {
 
 			// fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
 			if len(stringerrs) != 0 {
+				Error(fmt.Sprintf("Found %d broken packages and %d broken deps.",
+					opts.BrokenPkgs, opts.BrokenDeps))
 				Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
-				//	if brokenPkgs > 0 {
-				//os.Exit(1)
 			} else {
 				Info("All good! :white_check_mark:")
 				os.Exit(0)
@@ -310,6 +459,8 @@ func NewTreeValidateCommand() *cobra.Command {
 		},
 	}
 
+	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
+	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")
 	ans.Flags().BoolP("with-solver", "s", false,
 		"Enable check of requires also with solver.")
 	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{},

cmd/uninstall.go

@@ -18,8 +18,8 @@ import (
 	"os"
 	"path/filepath"
 
+	helpers "github.com/mudler/luet/cmd/helpers"
 	. "github.com/mudler/luet/pkg/config"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -57,9 +57,10 @@ var uninstallCmd = &cobra.Command{
 			rate := LuetCfg.Viper.GetFloat64("solver.rate")
 			attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 			force := LuetCfg.Viper.GetBool("force")
-			nodeps := LuetCfg.Viper.GetBool("nodeps")
+			nodeps, _ := cmd.Flags().GetBool("nodeps")
 			full, _ := cmd.Flags().GetBool("full")
 			checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
+			fullClean, _ := cmd.Flags().GetBool("full-clean")
 
 			LuetCfg.GetSolverOptions().Type = stype
 			LuetCfg.GetSolverOptions().LearnRate = float32(rate)
@@ -69,12 +70,13 @@ var uninstallCmd = &cobra.Command{
 			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
 			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-				Concurrency:    LuetCfg.GetGeneral().Concurrency,
-				SolverOptions:  *LuetCfg.GetSolverOptions(),
-				NoDeps:         nodeps,
-				Force:          force,
-				FullUninstall:  full,
-				CheckConflicts: checkconflicts,
+				Concurrency:        LuetCfg.GetGeneral().Concurrency,
+				SolverOptions:      *LuetCfg.GetSolverOptions(),
+				NoDeps:             nodeps,
+				Force:              force,
+				FullUninstall:      full,
+				FullCleanUninstall: fullClean,
+				CheckConflicts:     checkconflicts,
 			})
 
 			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
@@ -107,6 +109,7 @@ func init() {
 	uninstallCmd.Flags().Bool("force", false, "Force uninstall")
 	uninstallCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
 	uninstallCmd.Flags().Bool("conflictscheck", true, "Check if the package marked for deletion is required by other packages")
+	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
 
 	RootCmd.AddCommand(uninstallCmd)
 }

cmd/upgrade.go

@@ -58,6 +58,11 @@ var upgradeCmd = &cobra.Command{
 		rate := LuetCfg.Viper.GetFloat64("solver.rate")
 		attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
 		force := LuetCfg.Viper.GetBool("force")
+		nodeps, _ := cmd.Flags().GetBool("nodeps")
+		full, _ := cmd.Flags().GetBool("full")
+		universe, _ := cmd.Flags().GetBool("universe")
+		clean, _ := cmd.Flags().GetBool("clean")
+		sync, _ := cmd.Flags().GetBool("sync")
 
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
@@ -67,9 +72,14 @@ var upgradeCmd = &cobra.Command{
 		Debug("Solver", LuetCfg.GetSolverOptions().String())
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:   LuetCfg.GetGeneral().Concurrency,
-			SolverOptions: *LuetCfg.GetSolverOptions(),
-			Force:         force,
+			Concurrency:                LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:              *LuetCfg.GetSolverOptions(),
+			Force:                      force,
+			FullUninstall:              full,
+			NoDeps:                     nodeps,
+			SolverUpgrade:              universe,
+			RemoveUnavailableOnUpgrade: clean,
+			UpgradeNewRevisions:        sync,
 		})
 		inst.Repositories(repos)
 		_, err := inst.SyncRepositories(false)
@@ -103,6 +113,11 @@ func init() {
 	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
 	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
+	upgradeCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
+	upgradeCmd.Flags().Bool("full", true, "Attempts to remove as much packages as possible which aren't required (slow)")
+	upgradeCmd.Flags().Bool("universe", false, "Use ONLY the SAT solver to compute upgrades (experimental)")
+	upgradeCmd.Flags().Bool("clean", false, "Try to drop removed packages (experimental, only when --universe is enabled)")
+	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

contrib/config/config.protect.d/01_etc.yml.example

@@ -0,0 +1,3 @@
+name: "etc_conf"
+dirs:
+  - "/etc/"

contrib/config/luet.yaml

@@ -16,6 +16,12 @@
 #   Enable JSON log format instead of console mode.
 #   json_format: false.
 #
+#   Disable/Enable color
+#   color: true
+#
+#   Enable/Disable emoji
+#   enable_emoji: true
+#
 # ---------------------------------------------
 # General configuration section:
 # ---------------------------------------------
@@ -73,7 +79,20 @@
 #   - /etc/luet/repos.conf.d
 #
 #
-# ---------------------------------------------
+# ------------------------------------------------
+# Config protect configuration files directories.
+# -----------------------------------------------
+# Define the list of directories where load
+# configuration files with the list of config
+# protect paths.
+# config_protect_confdir:
+#   - /etc/luet/config.protect.d
+#
+# Permit to ignore rules defined on
+# config protect confdir and packages
+# annotation.
+# config_protect_skip: false
+#
 # System repositories
 # ---------------------------------------------
 # In alternative to define repositories files

go.mod

@@ -4,12 +4,13 @@ go 1.12
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/Sabayon/pkgs-checker v0.6.2-0.20200404093625-076438c31739
+	github.com/Sabayon/pkgs-checker v0.6.3-0.20200912135508-97c41780e9b6
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/briandowns/spinner v1.7.0
 	github.com/cavaliercoder/grab v2.0.0+incompatible
 	github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200417035958-130b0bc6032c+incompatible
+	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
 	github.com/ghodss/yaml v1.0.0
 	github.com/hashicorp/go-version v1.2.0
@@ -22,17 +23,16 @@ require (
 	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
 	github.com/mattn/go-isatty v0.0.10 // indirect
 	github.com/moby/sys/mount v0.1.1-0.20200320164225-6154f11e6840 // indirect
+	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.0
-	github.com/otiai10/copy v1.0.2
+	github.com/onsi/ginkgo v1.12.1
+	github.com/onsi/gomega v1.10.0
+	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
 	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
-	github.com/pkg/errors v0.8.1
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.5.0
+	github.com/pkg/errors v0.9.1
+	github.com/spf13/cobra v1.0.0
+	github.com/spf13/viper v1.6.3
 	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
 	go.etcd.io/bbolt v1.3.4
 	go.uber.org/atomic v1.5.1 // indirect
@@ -40,10 +40,10 @@ require (
 	go.uber.org/zap v1.13.0
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
 	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
-	golang.org/x/tools v0.0.0-20200102200121-6de373a2766c // indirect
-	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
-	gopkg.in/yaml.v2 v2.2.7
+	gopkg.in/yaml.v2 v2.2.8
+	gotest.tools/v3 v3.0.2 // indirect
+	helm.sh/helm/v3 v3.3.4
 	mvdan.cc/sh/v3 v3.0.0-beta1
 )
 
-replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200418093736-b2b0766ef22c+incompatible
+replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200605210607-749178b8f80d+incompatible

pkg/compiler/artifact.go

@@ -18,6 +18,8 @@ package compiler
 import (
 	"archive/tar"
 	"bufio"
+	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -25,6 +27,7 @@ import (
 	"path/filepath"
 	"regexp"
 
+	system "github.com/docker/docker/pkg/system"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
@@ -34,6 +37,7 @@ import (
 	. "github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/solver"
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v2"
@@ -277,8 +281,97 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	return errors.New("Compression type must be supplied")
 }
 
+func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	// If the destination path already exists I rename target file name with postfix.
+	var destPath string
+
+	// Read data. TODO: We need change archive callback to permit to return a Reader
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	// If file is not present on archive but is defined on mods
+	// I receive the callback. Prevent nil exception.
+	if header != nil {
+		switch header.Typeflag {
+		case tar.TypeReg:
+			destPath = filepath.Join(dst, path)
+		default:
+			// Nothing to do. I return original reader
+			return header, buffer.Bytes(), nil
+		}
+
+		// Check if exists
+		if helpers.Exists(destPath) {
+			for i := 1; i < 1000; i++ {
+				name := filepath.Join(filepath.Join(filepath.Dir(path),
+					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
+
+				if helpers.Exists(name) {
+					continue
+				}
+				Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
+					filepath.Join(dst, name)))
+				return &tar.Header{
+					Mode:       header.Mode,
+					Typeflag:   header.Typeflag,
+					PAXRecords: header.PAXRecords,
+					Name:       name,
+				}, buffer.Bytes(), nil
+			}
+		}
+	}
+
+	return header, buffer.Bytes(), nil
+}
+
+func (a *PackageArtifact) GetProtectFiles() []string {
+	ans := []string{}
+
+	if !LuetCfg.ConfigProtectSkip &&
+		LuetCfg.GetConfigProtectConfFiles() != nil &&
+		len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+
+		for _, file := range a.Files {
+			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
+				for _, dir := range conf.Directories {
+					// Note file is without / at begin.
+					if strings.HasPrefix("/"+file, filepath.Clean(dir)) {
+						// docker archive modifier works with path without / at begin.
+						ans = append(ans, file)
+						goto nextFile
+					}
+				}
+			}
+
+			if a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+				dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+				if ok {
+					if strings.HasPrefix("/"+file, filepath.Clean(dir)) {
+						ans = append(ans, file)
+						goto nextFile
+					}
+				}
+			}
+
+		nextFile:
+		}
+	}
+
+	return ans
+}
+
 // Unpack Untar and decompress (TODO) to the given path
 func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
+
+	// Create
+	protectedFiles := a.GetProtectFiles()
+
+	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
+
 	switch a.CompressionType {
 	case GZip:
 		// Create the uncompressed archive
@@ -307,15 +400,16 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
 			return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
 		}
 
-		err = helpers.Untar(a.GetPath()+".uncompressed", dst,
-			LuetCfg.GetGeneral().SameOwner)
+		err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
+			LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
 		if err != nil {
 			return err
 		}
 		return nil
 	// Defaults to tar only (covers when "none" is supplied)
 	default:
-		return helpers.Untar(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner)
+		return helpers.UntarProtect(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner,
+			protectedFiles, tarModifier)
 	}
 	return errors.New("Compression type must be supplied")
 }
@@ -386,6 +480,27 @@ type CopyJob struct {
 	Artifact string
 }
 
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
@@ -399,10 +514,13 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 		// 	continue
 		// }
 
-		if !helpers.Exists(job.Dst) {
+		_, err := os.Lstat(job.Dst)
+		if err != nil {
+			Debug("Copying ", job.Src)
 			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
+			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
@@ -459,19 +577,33 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 					}
 				}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	} else {
 		// Otherwise just grab all
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 			for _, a := range l.Diffs.Additions {
+				Debug("File ", a.Name, " added")
 				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	}
 
 	close(toCopy)
 	wg.Wait()
+
 	a := NewPackageArtifact(dst)
 	a.SetCompressionType(t)
 	err = a.Compress(archive, concurrency)

pkg/compiler/backend/diff.go

@@ -0,0 +1,199 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/pkg/errors"
+)
+
+// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
+// example of json return: [
+//   {
+//     "Image1": "luet/base",
+//     "Image2": "alpine",
+//     "DiffType": "File",
+//     "Diff": {
+//       "Adds": null,
+//       "Dels": [
+//         {
+//           "Name": "/luetbuild",
+//           "Size": 5830706
+//         },
+//         {
+//           "Name": "/luetbuild/Dockerfile",
+//           "Size": 50
+//         },
+//         {
+//           "Name": "/luetbuild/output1",
+//           "Size": 5830656
+//         }
+//       ],
+//       "Mods": null
+//     }
+//   }
+// ]
+func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]compiler.ArtifactLayer, error) {
+
+	res := compiler.ArtifactLayer{FromImage: srcImage, ToImage: dstImage}
+
+	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(srcRootFS) // clean up
+
+	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(dstRootFS) // clean up
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(srcImage, "/") {
+		srcImageTar, err := ioutil.TempFile(tmpdiffs, "srctar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(srcImageTar.Name()) // clean up
+		srcImageExport := compiler.CompilerBackendOptions{
+			ImageName:   srcImage,
+			Destination: srcImageTar.Name(),
+		}
+		err = b.ExportImage(srcImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting src image "+srcImage)
+		}
+		srcImage = srcImageTar.Name()
+	}
+
+	srcImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  srcImage,
+		Destination: srcRootFS,
+	}
+	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+srcImage)
+	}
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(dstImage, "/") {
+		dstImageTar, err := ioutil.TempFile(tmpdiffs, "dsttar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(dstImageTar.Name()) // clean up
+		dstImageExport := compiler.CompilerBackendOptions{
+			ImageName:   dstImage,
+			Destination: dstImageTar.Name(),
+		}
+		err = b.ExportImage(dstImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting dst image "+dstImage)
+		}
+		dstImage = dstImageTar.Name()
+	}
+
+	dstImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  dstImage,
+		Destination: dstRootFS,
+	}
+	err = b.ExtractRootfs(dstImageExtract, false)
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+dstImage)
+	}
+
+	// Get Additions/Changes. dst -> src
+	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, dstRootFS, "", -1)
+		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
+		if err == nil {
+			var sizeA, sizeB int64
+			sizeA = fileInfo.Size()
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+
+			if sizeA != sizeB {
+				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
+				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
+					Name: filepath.Join("/", realpath),
+					Size: int(sizeB),
+				})
+			} else {
+				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
+			}
+		} else {
+			var sizeB int64
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+				Size: int(sizeB),
+			})
+
+			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+	}
+
+	// Get deletions. src -> dst
+	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, srcRootFS, "", -1)
+		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
+			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
+			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+			})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+	}
+
+	return []compiler.ArtifactLayer{res}, nil
+}

pkg/compiler/backend/diff_test.go

@@ -0,0 +1,73 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend_test
+
+import (
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Docker image diffs", func() {
+	var b compiler.CompilerBackend
+
+	BeforeEach(func() {
+		b = NewSimpleDockerBackend()
+	})
+
+	Context("Generate diffs from docker images", func() {
+		It("Detect no changes", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "alpine:latest",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, "alpine:latest", "alpine:latest")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+
+		It("Detects additions and changed files", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/micro",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			err = b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/extra",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, "quay.io/mocaccino/micro", "quay.io/mocaccino/extra")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+
+			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+	})
+})

pkg/compiler/backend/simpledocker.go

@@ -16,7 +16,6 @@
 package backend
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -90,6 +89,19 @@ func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
+func (*SimpleDocker) ImageExists(imagename string) bool {
+	buildarg := []string{"inspect", "--type=image", imagename}
+	Debug(":whale: Checking existance of docker image: " + imagename)
+	cmd := exec.Command("docker", buildarg...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		Warning("Image not present")
+		Debug(string(out))
+		return false
+	}
+	return true
+}
+
 func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
@@ -209,64 +221,9 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 	return nil
 }
 
-// 	container-diff diff daemon://luet/base alpine --type=file -j
-// [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("tmpdiffs")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-	var errorBuffer bytes.Buffer
-
-	diffargs := []string{"diff", fromImage, toImage, "-v", "error", "-q", "--type=file", "-j", "-n", "-c", tmpdiffs}
-	cmd := exec.Command("container-diff", diffargs...)
-	cmd.Stderr = &errorBuffer
-	out, err := cmd.Output()
-
-	if string(errorBuffer.Bytes()) != "" {
-		Warning("container-diff errored with: " + string(errorBuffer.Bytes()))
-	}
-
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
-	}
-
-	if config.LuetCfg.GetGeneral().ShowBuildOutput {
-		Info(string(out))
-	}
-
-	var diffs []compiler.ArtifactLayer
-
-	err = json.Unmarshal(out, &diffs)
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
-	}
+// Changes retrieves changes between image layers
+func (d *SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+	diffs, err := GenerateChanges(d, fromImage, toImage)
 
 	if config.LuetCfg.GetGeneral().Debug {
 		summary := compiler.ComputeArtifactLayerSummary(diffs)
@@ -279,5 +236,5 @@ func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLaye
 		}
 	}
 
-	return diffs, nil
+	return diffs, err
 }

pkg/compiler/backend/simpleimg.go

@@ -97,6 +97,13 @@ func (*SimpleImg) CopyImage(src, dst string) error {
 	return nil
 }
 
+func (*SimpleImg) ImageExists(imagename string) bool {
+	// NOOP: not implemented
+	// TODO: Since img doesn't have an inspect command,
+	// we need to parse the ls output manually
+	return false
+}
+
 func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
@@ -142,8 +149,8 @@ func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms
 
 // TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
 // Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	return NewSimpleDockerBackend().Changes(fromImage, toImage)
+func (i *SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+	return GenerateChanges(i, fromImage, toImage)
 }
 
 func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {

pkg/compiler/compiler.go

@@ -20,9 +20,13 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+
+	"github.com/ghodss/yaml"
+
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -33,6 +37,7 @@ import (
 )
 
 const BuildFile = "build.yaml"
+const DefinitionFile = "definition.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
@@ -228,14 +233,38 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 	return nil
 }
 
-func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec) (Artifact, error) {
+func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec, generateArtifact bool) (Artifact, error) {
+
+	pkgTag := ":package:  " + p.GetPackage().GetName()
+
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	// TODO: As the salt contains the packageImage ( in registry/organization/imagename:tag format)
+	// the images hashes are broken with registry mirrors.
+	// We should use the image tag, or pass by the package assertion hash which is unique
+	// and identifies the deptree of the package.
+
+	fp := p.GetPackage().HashFingerprint(packageImage)
+
+	if buildertaggedImage == "" {
+		buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder"
+		Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image)
+	}
+
+	// TODO:  Cleanup, not actually hit
+	if packageImage == "" {
+		packageImage = cs.ImageRepository + "-" + fp
+	}
+
 	if !cs.Clean {
-		if art, err := LoadArtifactFromYaml(p); err == nil {
+		exists := cs.Backend.ImageExists(buildertaggedImage) && cs.Backend.ImageExists(packageImage)
+		if art, err := LoadArtifactFromYaml(p); err == nil && (cs.Options.SkipIfMetadataExists || exists) {
 			Debug("Artifact reloaded. Skipping build")
 			return art, err
 		}
 	}
-	pkgTag := ":package:  " + p.GetPackage().GetName()
 
 	p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
 	// and we build all the images first.
@@ -265,14 +294,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 	}
 
-	fp := p.GetPackage().HashFingerprint()
-	if buildertaggedImage == "" {
-		buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder"
-	}
-	if packageImage == "" {
-		packageImage = cs.ImageRepository + "-" + fp
-	}
-
 	Info(pkgTag, "Generating :whale: definition for builder image from", image)
 
 	// First we create the builder image
@@ -319,12 +340,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 	}
 
-	// if !keepPackageImg {
-	// 	err = cs.Backend.ImageDefinitionToTar(runnerOpts)
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "Could not export image to tar")
-	// 	}
-	// } else {
 	buildPackageImage := true
 	if cs.Options.PullFirst {
 		//Best effort pull
@@ -353,9 +368,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 			return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
 		}
 	}
-	//	}
 
-	var diffs []ArtifactLayer
 	var artifact Artifact
 	unpack := p.ImageUnpack()
 
@@ -365,14 +378,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		unpack = true
 	}
 
-	if !unpack {
-		// we have to get diffs only if spec is not unpacked
-		diffs, err = cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate changes from layers")
-		}
-	}
-
 	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
@@ -402,8 +407,11 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 	}
 
-	if unpack {
+	if !generateArtifact {
+		return &PackageArtifact{}, nil
+	}
 
+	if unpack {
 		if p.GetPackageDir() != "" {
 			Info(":tophat: Packing from output dir", p.GetPackageDir())
 			rootfs = filepath.Join(rootfs, p.GetPackageDir())
@@ -415,6 +423,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 		artifact.SetCompressionType(cs.CompressionType)
+
 		err = artifact.Compress(rootfs, concurrency)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating package archive")
@@ -423,7 +432,10 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		artifact.SetCompileSpec(p)
 	} else {
 		Info(pkgTag, "Generating delta")
-
+		diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not generate changes from layers")
+		}
 		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not generate deltas")
@@ -439,6 +451,8 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 
 	artifact.SetFiles(filelist)
 
+	artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
+
 	err = artifact.WriteYaml(p.GetOutputPath())
 	if err != nil {
 		return artifact, errors.Wrap(err, "Failed while writing metadata file")
@@ -448,6 +462,53 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	return artifact, nil
 }
 
+func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) {
+	compilerSpecs := NewLuetCompilationspecs()
+
+	w := db.World()
+
+	for _, p := range w {
+		spec, err := cs.FromPackage(p)
+		if err != nil {
+			return nil, err
+		}
+		if dst != "" {
+			spec.SetOutputPath(dst)
+		}
+		compilerSpecs.Add(spec)
+	}
+
+	switch minimum {
+	case true:
+		return cs.ComputeMinimumCompilableSet(compilerSpecs.Unique().All()...)
+	default:
+		return compilerSpecs.Unique().All(), nil
+	}
+}
+
+// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
+func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error) {
+	// Generate a set with all the deps of the provided specs
+	// we will use that set to remove the deps from the list of provided compilation specs
+	allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
+	result := []CompilationSpec{}
+	for _, spec := range p {
+		ass, err := cs.ComputeDepTree(spec)
+		if err != nil {
+			return result, errors.Wrap(err, "computin specs deptree")
+		}
+
+		allDependencies = append(allDependencies, ass.Drop(spec.GetPackage())...)
+	}
+
+	for _, spec := range p {
+		if found := allDependencies.Search(spec.GetPackage().GetFingerPrint()); found == nil {
+			result = append(result, spec)
+		}
+	}
+	return result, nil
+}
+
 func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
 
 	s := solver.NewResolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
@@ -466,7 +527,6 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
 	for _, assertion := range dependencies { //highly dependent on the order
 		if assertion.Value {
 			nthsolution := dependencies.Cut(assertion.Package)
-
 			assertion.Hash = solver.PackageHash{
 				BuildHash:   nthsolution.HashFrom(assertion.Package),
 				PackageHash: nthsolution.AssertionHash(),
@@ -493,7 +553,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 
 	if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" {
 		Error("Package with no deps and no seed image supplied, bailing out")
-		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() + "with no deps and no seed image supplied, bailing out")
+		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() +
+			" with no deps and no seed image supplied, bailing out")
 	}
 
 	targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
@@ -502,7 +563,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p)
+		return cs.compileWithImage(p.GetImage(), "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@@ -516,6 +577,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	depsN := 0
 	currentN := 0
 
+	packageDeps := !cs.Options.PackageTargetOnly
 	if !cs.Options.NoDeps {
 		Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
 		for _, assertion := range dependencies { //highly dependent on the order
@@ -541,7 +603,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 			lastHash = currentPackageImageHash
 			if compileSpec.GetImage() != "" {
 				Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
-				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 				if err != nil {
 					return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				}
@@ -551,7 +613,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 			}
 
 			Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
-			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec)
+			artifact, err := cs.compileWithImage(buildImageHash, "", currentPackageImageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
 			if err != nil {
 				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 				//	deperrs = append(deperrs, err)
@@ -567,7 +629,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 
 	if !cs.Options.OnlyDeps {
 		Info(":package:", p.GetPackage().HumanReadableString(), ":cyclone:  Building package target from:", lastHash)
-		artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p)
+		artifact, err := cs.compileWithImage(lastHash, "", targetPackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
 		if err != nil {
 			return artifact, err
 		}
@@ -580,6 +642,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	}
 }
 
+type templatedata map[string]interface{}
+
 func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 
 	pack, err := cs.Database.FindPackageCandidate(p)
@@ -591,12 +655,28 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 	if !helpers.Exists(buildFile) {
 		return nil, errors.New("No build file present for " + p.GetFingerPrint())
 	}
-
-	dat, err := ioutil.ReadFile(buildFile)
+	defFile := pack.Rel(DefinitionFile)
+	if !helpers.Exists(defFile) {
+		return nil, errors.New("No build file present for " + p.GetFingerPrint())
+	}
+	def, err := ioutil.ReadFile(defFile)
 	if err != nil {
 		return nil, err
 	}
-	return NewLuetCompilationSpec(dat, pack)
+
+	build, err := ioutil.ReadFile(buildFile)
+	if err != nil {
+		return nil, err
+	}
+	var values templatedata
+	if err = yaml.Unmarshal(def, &values); err != nil {
+		return nil, err
+	}
+	out, err := helpers.RenderHelm(string(build), values)
+	if err != nil {
+		return nil, err
+	}
+	return NewLuetCompilationSpec([]byte(out), pack)
 }
 
 func (cs *LuetCompiler) GetBackend() CompilerBackend {

pkg/compiler/compiler_test.go

@@ -108,6 +108,27 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Templated packages",func(){
+		It("Renders", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/templates")
+			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+			pkg ,err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec, err := compiler.FromPackage(pkg)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetImage()).To(Equal("b:bar"))
+		})
+	})
+
 	Context("Reconstruct image tree", func() {
 		It("Compiles it", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -706,6 +727,25 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Compilation of whole tree", func() {
+		It("doesn't include dependencies that would be compiled anyway", func() {
+			// As some specs are dependent from each other, don't pull it in if they would
+			// be eventually
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/includeimage")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+
+			specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(specs)).To(Equal(1))
+
+			Expect(specs[0].GetPackage().GetFingerPrint()).To(Equal("b-test-1.0"))
+		})
+	})
+
 	Context("File list", func() {
 		It("is generated after the compilation process and annotated in the metadata", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))

pkg/compiler/interface.go

@@ -28,9 +28,10 @@ type Compiler interface {
 	CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
+	ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error)
 	SetConcurrency(i int)
 	FromPackage(pkg.Package) (CompilationSpec, error)
-
+	FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error)
 	SetBackend(CompilerBackend)
 	GetBackend() CompilerBackend
 	SetCompressionType(t CompressionImplementation)
@@ -51,9 +52,12 @@ type CompilerOptions struct {
 	Clean                    bool
 	KeepImageExport          bool
 
-	OnlyDeps      bool
-	NoDeps        bool
-	SolverOptions config.LuetSolverOptions
+	OnlyDeps             bool
+	NoDeps               bool
+	SolverOptions        config.LuetSolverOptions
+	SkipIfMetadataExists bool
+
+	PackageTargetOnly bool
 }
 
 func NewDefaultCompilerOptions() *CompilerOptions {
@@ -82,6 +86,8 @@ type CompilerBackend interface {
 	DownloadImage(opts CompilerBackendOptions) error
 
 	Push(opts CompilerBackendOptions) error
+
+	ImageExists(string) bool
 }
 
 type Artifact interface {

pkg/config/config.go

@@ -200,9 +200,13 @@ type LuetConfig struct {
 	System  LuetSystemConfig  `mapstructure:"system"`
 	Solver  LuetSolverOptions `mapstructure:"solver"`
 
-	RepositoriesConfDir []string         `mapstructure:"repos_confdir"`
-	CacheRepositories   []LuetRepository `mapstructure:"repetitors"`
-	SystemRepositories  []LuetRepository `mapstructure:"repositories"`
+	RepositoriesConfDir  []string         `mapstructure:"repos_confdir"`
+	ConfigProtectConfDir []string         `mapstructure:"config_protect_confdir"`
+	ConfigProtectSkip    bool             `mapstructure:"config_protect_skip"`
+	CacheRepositories    []LuetRepository `mapstructure:"repetitors"`
+	SystemRepositories   []LuetRepository `mapstructure:"repositories"`
+
+	ConfigProtectConfFiles []ConfigProtectConfFile
 }
 
 func NewLuetConfig(viper *v.Viper) *LuetConfig {
@@ -211,7 +215,7 @@ func NewLuetConfig(viper *v.Viper) *LuetConfig {
 	}
 
 	GenDefault(viper)
-	return &LuetConfig{Viper: viper}
+	return &LuetConfig{Viper: viper, ConfigProtectConfFiles: nil}
 }
 
 func GenDefault(viper *v.Viper) {
@@ -231,7 +235,7 @@ func GenDefault(viper *v.Viper) {
 
 	u, err := user.Current()
 	// os/user doesn't work in from scratch environments
-	if err != nil || u.Uid == "0" {
+	if err != nil || (u != nil && u.Uid == "0") {
 		viper.SetDefault("general.same_owner", true)
 	} else {
 		viper.SetDefault("general.same_owner", false)
@@ -244,6 +248,8 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("system.pkgs_cache_path", "packages")
 
 	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
+	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
+	viper.SetDefault("config_protect_skip", false)
 	viper.SetDefault("cache_repositories", []string{})
 	viper.SetDefault("system_repositories", []string{})
 
@@ -273,6 +279,18 @@ func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
 	return &c.Solver
 }
 
+func (c *LuetConfig) GetConfigProtectConfFiles() []ConfigProtectConfFile {
+	return c.ConfigProtectConfFiles
+}
+
+func (c *LuetConfig) AddConfigProtectConfFile(file *ConfigProtectConfFile) {
+	if c.ConfigProtectConfFiles == nil {
+		c.ConfigProtectConfFiles = []ConfigProtectConfFile{*file}
+	} else {
+		c.ConfigProtectConfFiles = append(c.ConfigProtectConfFiles, *file)
+	}
+}
+
 func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
 	var ans *LuetRepository = nil
 

pkg/config/config_protect.go

@@ -0,0 +1,41 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"fmt"
+)
+
+type ConfigProtectConfFile struct {
+	Filename string
+
+	Name        string   `mapstructure:"name" yaml:"name" json:"name"`
+	Directories []string `mapstructure:"dirs" yaml:"dirs" json:"dirs"`
+}
+
+func NewConfigProtectConfFile(filename string) *ConfigProtectConfFile {
+	return &ConfigProtectConfFile{
+		Filename:    filename,
+		Name:        "",
+		Directories: []string{},
+	}
+}
+
+func (c *ConfigProtectConfFile) String() string {
+	return fmt.Sprintf("[%s] filename: %s, dirs: %s", c.Name, c.Filename,
+		c.Directories)
+}

pkg/helpers/archive.go

@@ -17,6 +17,7 @@ package helpers
 
 import (
 	"archive/tar"
+	"bytes"
 	"io"
 	"os"
 	"path/filepath"
@@ -49,6 +50,146 @@ func Tar(src, dest string) error {
 	return err
 }
 
+type TarModifierWrapperFunc func(path, dst string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
+type TarModifierWrapper struct {
+	DestinationPath string
+	Modifier        TarModifierWrapperFunc
+}
+
+func NewTarModifierWrapper(dst string, modifier TarModifierWrapperFunc) *TarModifierWrapper {
+	return &TarModifierWrapper{
+		DestinationPath: dst,
+		Modifier:        modifier,
+	}
+}
+
+func (m *TarModifierWrapper) GetModifier() archive.TarModifierFunc {
+	return func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+		return m.Modifier(m.DestinationPath, path, header, content)
+	}
+}
+
+func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modifier *TarModifierWrapper) error {
+	var ans error
+
+	if len(protectedFiles) <= 0 {
+		return Untar(src, dst, sameOwner)
+	}
+
+	// POST: we have files to protect. I create a ReplaceFileTarWrapper
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	// Create modifier map
+	mods := make(map[string]archive.TarModifierFunc)
+	for _, file := range protectedFiles {
+		mods[file] = modifier.GetModifier()
+	}
+
+	if sameOwner {
+		// PRE: i have root privileged.
+
+		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
+
+		opts := &archive.TarOptions{
+			// NOTE: NoLchown boolean is used for chmod of the symlink
+			// Probably it's needed set this always to true.
+			NoLchown:        true,
+			ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+			ContinueOnError: true,
+		}
+
+		ans = archive.Untar(replacerArchive, dst, opts)
+	} else {
+		ans = unTarIgnoreOwner(dst, in, mods)
+	}
+
+	return ans
+}
+
+func unTarIgnoreOwner(dest string, in io.ReadCloser, mods map[string]archive.TarModifierFunc) error {
+	tr := tar.NewReader(in)
+	for {
+		header, err := tr.Next()
+
+		var data []byte
+		var headerReplaced = false
+
+		switch {
+		case err == io.EOF:
+			goto tarEof
+		case err != nil:
+			return err
+		case header == nil:
+			continue
+		}
+
+		// the target location where the dir/file should be created
+		target := filepath.Join(dest, header.Name)
+		if mods != nil {
+			modifier, ok := mods[header.Name]
+			if ok {
+				header, data, err = modifier(header.Name, header, tr)
+				if err != nil {
+					return err
+				}
+
+				// Override target path
+				target = filepath.Join(dest, header.Name)
+				headerReplaced = true
+			}
+
+		}
+
+		// Check the file type
+		switch header.Typeflag {
+
+		// if its a dir and it doesn't exist create it
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return err
+				}
+			}
+
+			// handle creation of file
+		case tar.TypeReg:
+
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+
+			// copy over contents
+			if headerReplaced {
+				_, err = io.Copy(f, bytes.NewReader(data))
+			} else {
+				_, err = io.Copy(f, tr)
+			}
+			if err != nil {
+				return err
+			}
+
+			// manually close here after each file operation; defering would cause each
+			// file close to wait until all operations have completed.
+			f.Close()
+
+		case tar.TypeSymlink:
+			source := header.Linkname
+			err := os.Symlink(source, target)
+			if err != nil {
+				return err
+			}
+		}
+	}
+tarEof:
+
+	return nil
+}
+
 // Untar just a wrapper around the docker functions
 func Untar(src, dest string, sameOwner bool) error {
 	var ans error
@@ -72,62 +213,7 @@ func Untar(src, dest string, sameOwner bool) error {
 
 		ans = archive.Untar(in, dest, opts)
 	} else {
-
-		// TODO: replace with https://github.com/mholt/archiver ?
-		var fileReader io.ReadCloser = in
-
-		tr := tar.NewReader(fileReader)
-		for {
-			header, err := tr.Next()
-
-			switch {
-			case err == io.EOF:
-				goto tarEof
-			case err != nil:
-				return err
-			case header == nil:
-				continue
-			}
-
-			// the target location where the dir/file should be created
-			target := filepath.Join(dest, header.Name)
-
-			// Check the file type
-			switch header.Typeflag {
-
-			// if its a dir and it doesn't exist create it
-			case tar.TypeDir:
-				if _, err := os.Stat(target); err != nil {
-					if err := os.MkdirAll(target, 0755); err != nil {
-						return err
-					}
-				}
-
-				// handle creation of file
-			case tar.TypeReg:
-				f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
-				if err != nil {
-					return err
-				}
-
-				// copy over contents
-				if _, err := io.Copy(f, tr); err != nil {
-					return err
-				}
-
-				// manually close here after each file operation; defering would cause each
-				// file close to wait until all operations have completed.
-				f.Close()
-
-			case tar.TypeSymlink:
-				source := header.Linkname
-				err := os.Symlink(source, target)
-				if err != nil {
-					return err
-				}
-			}
-		}
-	tarEof:
+		ans = unTarIgnoreOwner(dest, in, nil)
 	}
 
 	return ans

pkg/helpers/archive_test.go

@@ -0,0 +1,134 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/archive"
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// Code from moby/moby pkg/archive/archive_test
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(filepath.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+		if makeLinks {
+			if err := os.Link(filepath.Join(targetPath, fileName), filepath.Join(targetPath, fileName+"-link")); err != nil {
+				return 0, err
+			}
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}
+
+func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	// If the destination path already exists I rename target file name with postfix.
+	var basePath string
+
+	// Read data. TODO: We need change archive callback to permit to return a Reader
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	if header != nil {
+
+		switch header.Typeflag {
+		case tar.TypeReg:
+			basePath = filepath.Base(path)
+		default:
+			// Nothing to do. I return original reader
+			return header, buffer.Bytes(), nil
+		}
+
+		if basePath == "file-0" {
+			name := filepath.Join(filepath.Join(filepath.Dir(path), fmt.Sprintf("._cfg%04d_%s", 1, basePath)))
+			return &tar.Header{
+				Mode:       header.Mode,
+				Typeflag:   header.Typeflag,
+				PAXRecords: header.PAXRecords,
+				Name:       name,
+			}, buffer.Bytes(), nil
+		} else if basePath == "file-1" {
+			return header, []byte("newcontent"), nil
+		}
+
+		// else file not present
+	}
+
+	return header, buffer.Bytes(), nil
+}
+
+var _ = Describe("Helpers Archive", func() {
+	Context("Untar Protect", func() {
+
+		It("Detect existing and not-existing files", func() {
+
+			archiveSourceDir, err := ioutil.TempDir("", "archive-source")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(archiveSourceDir)
+
+			_, err = prepareUntarSourceDirectory(10, archiveSourceDir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			targetDir, err := ioutil.TempDir("", "archive-target")
+			Expect(err).ToNot(HaveOccurred())
+			//	defer os.RemoveAll(targetDir)
+
+			sourceArchive, err := archive.TarWithOptions(archiveSourceDir, &archive.TarOptions{})
+			Expect(err).ToNot(HaveOccurred())
+			defer sourceArchive.Close()
+
+			tarModifier := NewTarModifierWrapper(targetDir, tarModifierWrapperFunc)
+			mods := make(map[string]archive.TarModifierFunc)
+			mods["file-0"] = tarModifier.GetModifier()
+			mods["file-1"] = tarModifier.GetModifier()
+			mods["file-9999"] = tarModifier.GetModifier()
+
+			replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
+			//replacerArchive := archive.ReplaceFileTarWrapper(sourceArchive, mods)
+			opts := &archive.TarOptions{
+				// NOTE: NoLchown boolean is used for chmod of the symlink
+				// Probably it's needed set this always to true.
+				NoLchown:        true,
+				ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+				ContinueOnError: true,
+			}
+
+			err = archive.Untar(replacerArchive, targetDir, opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(Exists(filepath.Join(targetDir, "._cfg0001_file-0"))).Should(Equal(true))
+		})
+	})
+})

pkg/helpers/file.go

@@ -93,7 +93,7 @@ func ensureDir(fileName string) {
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
 func CopyFile(src, dst string) (err error) {
-	return copy.Copy(src, dst)
+	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }
 
 func IsDirectory(path string) (bool, error) {
@@ -110,5 +110,5 @@ func IsDirectory(path string) (bool, error) {
 func CopyDir(src string, dst string) (err error) {
 	src = filepath.Clean(src)
 	dst = filepath.Clean(dst)
-	return copy.Copy(src, dst)
+	return copy.Copy(src, dst, copy.Options{OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
 }

pkg/helpers/helm.go

@@ -0,0 +1,34 @@
+package helpers
+
+import (
+
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+	"github.com/pkg/errors"
+)
+
+// RenderHelm renders the template string with helm
+func RenderHelm(template string, values map[string]interface{}) (string,error) {
+	c := &chart.Chart{
+		Metadata: &chart.Metadata{
+			Name:    "",
+			Version: "",
+		},
+		Templates: []*chart.File{
+			{Name: "templates", Data: []byte(template)},
+		},
+		Values:    map[string]interface{}{"Values":values},
+	}
+
+	v, err := chartutil.CoalesceValues(c, map[string]interface{}{})
+	if err != nil {
+		return "",errors.Wrap(err,"while rendering template")
+	}
+	out, err := engine.Render(c, v)
+	if err != nil {
+		return "",errors.Wrap(err,"while rendering template")
+	}
+
+	return out["templates"],nil
+}

pkg/helpers/helm_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Helpers", func() {
+	Context("RenderHelm", func() {
+		It("Renders templates", func() {
+			out, err := RenderHelm("{{.Values.Test}}",map[string]interface{}{"Test":"foo"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foo"))
+		})
+	})
+})

pkg/helpers/match.go

@@ -0,0 +1,49 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"regexp"
+)
+
+func MapMatchRegex(m *map[string]string, r *regexp.Regexp) bool {
+	ans := false
+
+	if m != nil {
+		for k, v := range *m {
+			if r.MatchString(k + "=" + v) {
+				ans = true
+				break
+			}
+		}
+	}
+
+	return ans
+}
+
+func MapHasKey(m *map[string]string, label string) bool {
+	ans := false
+	if m != nil {
+		for k, _ := range *m {
+			if k == label {
+				ans = true
+				break
+			}
+		}
+	}
+	return ans
+}

pkg/helpers/sys.go

@@ -16,7 +16,9 @@
 package helpers
 
 import (
+	"os"
 	"os/exec"
+	"os/user"
 	"syscall"
 
 	"github.com/pkg/errors"
@@ -30,3 +32,17 @@ func Exec(cmd string, args []string, env []string) error {
 	}
 	return syscall.Exec(path, args, env)
 }
+
+func GetHomeDir() (ans string) {
+	// os/user doesn't work in from scratch environments
+	u, err := user.Current()
+	if err == nil {
+		ans = u.HomeDir
+	} else {
+		ans = ""
+	}
+	if os.Getenv("HOME") != "" {
+		ans = os.Getenv("HOME")
+	}
+	return ans
+}

pkg/installer/config_protect.go

@@ -0,0 +1,86 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"io/ioutil"
+	"path"
+	"regexp"
+
+	"github.com/ghodss/yaml"
+
+	. "github.com/mudler/luet/pkg/config"
+	. "github.com/mudler/luet/pkg/logger"
+)
+
+func LoadConfigProtectConfs(c *LuetConfig) error {
+	var regexConfs = regexp.MustCompile(`.yml$`)
+
+	for _, cdir := range c.ConfigProtectConfDir {
+		Debug("Parsing Config Protect Directory", cdir, "...")
+
+		files, err := ioutil.ReadDir(cdir)
+		if err != nil {
+			Debug("Skip dir", cdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexConfs.MatchString(file.Name()) {
+				Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(cdir, file.Name()))
+			if err != nil {
+				Warning("On read file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := LoadConfigProtectConFile(file.Name(), content)
+			if err != nil {
+				Warning("On parse file", file.Name(), ":", err.Error())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Directories) == 0 {
+				Warning("Invalid config protect file", file.Name())
+				Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddConfigProtectConfFile(r)
+		}
+	}
+	return nil
+
+}
+
+func LoadConfigProtectConFile(filename string, data []byte) (*ConfigProtectConfFile, error) {
+	ans := NewConfigProtectConfFile(filename)
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}

pkg/installer/installer.go

@@ -16,6 +16,7 @@
 package installer
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -35,14 +36,15 @@ import (
 )
 
 type LuetInstallerOptions struct {
-	SolverOptions               config.LuetSolverOptions
-	Concurrency                 int
-	NoDeps                      bool
-	OnlyDeps                    bool
-	Force                       bool
-	PreserveSystemEssentialData bool
-	FullUninstall               bool
-	CheckConflicts              bool
+	SolverOptions                                                  config.LuetSolverOptions
+	Concurrency                                                    int
+	NoDeps                                                         bool
+	OnlyDeps                                                       bool
+	Force                                                          bool
+	PreserveSystemEssentialData                                    bool
+	FullUninstall, FullCleanUninstall                              bool
+	CheckConflicts                                                 bool
+	SolverUpgrade, RemoveUnavailableOnUpgrade, UpgradeNewRevisions bool
 }
 
 type LuetInstaller struct {
@@ -66,24 +68,80 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 	if err != nil {
 		return err
 	}
+
+	Info(":thinking: Computing upgrade, please hang tight")
 	// First match packages against repositories by priority
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	// compute a "big" world
 	solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
-	uninstall, solution, err := solv.Upgrade(false)
-	if err != nil {
-		return errors.Wrap(err, "Failed solving solution for upgrade")
+	var uninstall pkg.Packages
+	var solution solver.PackagesAssertions
+
+	if l.Options.SolverUpgrade {
+		uninstall, solution, err = solv.UpgradeUniverse(l.Options.RemoveUnavailableOnUpgrade)
+		if err != nil {
+			return errors.Wrap(err, "Failed solving solution for upgrade")
+		}
+	} else {
+		uninstall, solution, err = solv.Upgrade(!l.Options.FullUninstall, l.Options.NoDeps)
+		if err != nil {
+			return errors.Wrap(err, "Failed solving solution for upgrade")
+		}
+	}
+
+	if len(uninstall) > 0 {
+		Info("Packages marked for uninstall:")
+	}
+
+	for _, p := range uninstall {
+		Info(fmt.Sprintf("- %s", p.HumanReadableString()))
+	}
+
+	if len(solution) > 0 {
+		Info("Packages marked for upgrade:")
 	}
 
 	toInstall := pkg.Packages{}
 	for _, assertion := range solution {
 		// Be sure to filter from solutions packages already installed in the system
 		if _, err := s.Database.FindPackage(assertion.Package); err != nil && assertion.Value {
+			Info(fmt.Sprintf("- %s", assertion.Package.HumanReadableString()))
 			toInstall = append(toInstall, assertion.Package)
 		}
 	}
 
+	if l.Options.UpgradeNewRevisions {
+		Info("Checking packages with new revisions available")
+		for _, p := range s.Database.World() {
+			matches := syncedRepos.PackageMatches(pkg.Packages{p})
+			if len(matches) == 0 {
+				// Package missing. the user should run luet upgrade --universe
+				Info("Installed packages seems to be missing from remote repositories.")
+				Info("It is suggested to run 'luet upgrade --universe'")
+				continue
+			}
+			for _, artefact := range matches[0].Repo.GetIndex() {
+				if artefact.GetCompileSpec().GetPackage() == nil {
+					return errors.New("Package in compilespec empty")
+
+				}
+				if artefact.GetCompileSpec().GetPackage().Matches(p) && artefact.GetCompileSpec().GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
+					toInstall = append(toInstall, matches[0].Package).Unique()
+					uninstall = append(uninstall, p).Unique()
+					Info(
+						fmt.Sprintf("- %s ( %s vs %s ) repo: %s (date: %s)",
+							p.HumanReadableString(),
+							artefact.GetCompileSpec().GetPackage().GetBuildTimestamp(),
+							p.GetBuildTimestamp(),
+							matches[0].Repo.GetName(),
+							matches[0].Repo.GetLastUpdate(),
+						))
+				}
+			}
+		}
+	}
+
 	return l.swap(syncedRepos, uninstall, toInstall, s)
 }
 
@@ -328,6 +386,7 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 
 			}
 			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+				currentPack.SetBuildTimestamp(artefact.GetCompileSpec().GetPackage().GetBuildTimestamp())
 				// Filter out already installed
 				if _, err := s.Database.FindPackage(currentPack); err != nil {
 					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
@@ -598,10 +657,20 @@ func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
 	if !l.Options.NoDeps {
 		Info("Finding :package:", p.HumanReadableString(), "dependency graph :deciduous_tree:")
 		solv := solver.NewResolver(installedtmp, installedtmp, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
-		solution, err := solv.Uninstall(p, checkConflicts, full)
-		if err != nil && !l.Options.Force {
-			return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+		var solution pkg.Packages
+		var err error
+		if l.Options.FullCleanUninstall {
+			solution, err = solv.UninstallUniverse(pkg.Packages{p})
+			if err != nil {
+				return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+			}
+		} else {
+			solution, err = solv.Uninstall(p, checkConflicts, full)
+			if err != nil && !l.Options.Force {
+				return errors.Wrap(err, "Could not solve the uninstall constraints. Tip: try with --solver-type qlearning or with --force, or by removing packages excluding their dependencies with --nodeps")
+			}
 		}
+
 		for _, p := range solution {
 			Info("Uninstalling", p.HumanReadableString())
 			err := l.uninstall(p, s)

pkg/installer/interface.go

@@ -70,6 +70,6 @@ type Repository interface {
 	SetPriority(int)
 	GetRepositoryFile(string) (LuetRepositoryFile, error)
 	SetRepositoryFile(string, LuetRepositoryFile)
-
+	SetName(p string)
 	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
 }

pkg/installer/repository.go

@@ -265,6 +265,8 @@ func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact
 		// We want to include packages that are ONLY referenced in the tree.
 		// the ones which aren't should be deleted. (TODO: by another cli command?)
 		if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
+			Info(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				artifact.GetCompileSpec().GetPackage().HumanReadableString()))
 			return nil
 		}
 
@@ -302,6 +304,11 @@ func (r *LuetSystemRepository) GetType() string {
 func (r *LuetSystemRepository) SetType(p string) {
 	r.LuetRepository.Type = p
 }
+
+func (r *LuetSystemRepository) SetName(p string) {
+	r.LuetRepository.Name = p
+}
+
 func (r *LuetSystemRepository) AddUrl(p string) {
 	r.LuetRepository.Urls = append(r.LuetRepository.Urls, p)
 }
@@ -685,6 +692,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	repo.SetAuthentication(r.GetAuthentication())
 	repo.SetType(r.GetType())
 	repo.SetPriority(r.GetPriority())
+	repo.SetName(r.GetName())
 	InfoC(
 		aurora.Bold(
 			aurora.Yellow(":information_source: Repository "+repo.GetName()+" priority: ")).String() +

pkg/logger/logger.go

@@ -78,23 +78,25 @@ func Spinner(i int) {
 		i = 43
 	}
 
-	if !s.Active() {
+	if s != nil && !s.Active() {
 		//	s.UpdateCharSet(spinner.CharSets[i])
 		s.Start() // Start the spinner
 	}
 }
 
 func SpinnerText(suffix, prefix string) {
-	s.Lock()
-	defer s.Unlock()
-	if LuetCfg.GetGeneral().Debug {
-		fmt.Println(fmt.Sprintf("%s %s",
-			Bold(Cyan(prefix)).String(),
-			Bold(Magenta(suffix)).BgBlack().String(),
-		))
-	} else {
-		s.Suffix = Bold(Magenta(suffix)).BgBlack().String()
-		s.Prefix = Bold(Cyan(prefix)).String()
+	if s != nil {
+		s.Lock()
+		defer s.Unlock()
+		if LuetCfg.GetGeneral().Debug {
+			fmt.Println(fmt.Sprintf("%s %s",
+				Bold(Cyan(prefix)).String(),
+				Bold(Magenta(suffix)).BgBlack().String(),
+			))
+		} else {
+			s.Suffix = Bold(Magenta(suffix)).BgBlack().String()
+			s.Prefix = Bold(Cyan(prefix)).String()
+		}
 	}
 }
 
@@ -108,7 +110,9 @@ func SpinnerStop() {
 	if 2 > confLevel {
 		return
 	}
-	s.Stop()
+	if s != nil {
+		s.Stop()
+	}
 }
 
 func level2Number(level string) int {

pkg/package/annotations.go

@@ -0,0 +1,23 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package pkg
+
+type AnnotationKey string
+
+const (
+	ConfigProtectAnnnotation AnnotationKey = "config_protect"
+)

pkg/package/package.go

@@ -26,11 +26,13 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/mudler/luet/pkg/helpers"
+	version "github.com/mudler/luet/pkg/versioner"
+
 	gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
 	"github.com/crillab/gophersat/bf"
 	"github.com/ghodss/yaml"
 	"github.com/jinzhu/copier"
-	version "github.com/mudler/luet/pkg/versioner"
 	"github.com/pkg/errors"
 )
 
@@ -40,8 +42,7 @@ type Package interface {
 	Encode(PackageDatabase) (string, error)
 
 	BuildFormula(PackageDatabase, PackageDatabase) ([]bf.Formula, error)
-	IsFlagged(bool) Package
-	Flagged() bool
+
 	GetFingerPrint() string
 	GetPackageName() string
 	Requires([]*DefaultPackage) Package
@@ -63,6 +64,7 @@ type Package interface {
 	GetCategory() string
 
 	GetVersion() string
+	SetVersion(string)
 	RequiresContains(PackageDatabase, Package) (bool, error)
 	Matches(m Package) bool
 	BumpBuildVersion() error
@@ -92,13 +94,22 @@ type Package interface {
 	HasLabel(string) bool
 	MatchLabel(*regexp.Regexp) bool
 
+	AddAnnotation(string, string)
+	GetAnnotations() map[string]string
+	HasAnnotation(string) bool
+	MatchAnnotation(*regexp.Regexp) bool
+
+	IsHidden() bool
 	IsSelector() bool
 	VersionMatchSelector(string, version.Versioner) (bool, error)
 	SelectorMatchVersion(string, version.Versioner) (bool, error)
 
 	String() string
 	HumanReadableString() string
-	HashFingerprint() string
+	HashFingerprint(string) string
+
+	SetBuildTimestamp(s string)
+	GetBuildTimestamp() string
 
 	Clone() Package
 }
@@ -154,17 +165,19 @@ type DefaultPackage struct {
 	State            State             `json:"state,omitempty"`
 	PackageRequires  []*DefaultPackage `json:"requires"`           // Affects YAML field names too.
 	PackageConflicts []*DefaultPackage `json:"conflicts"`          // Affects YAML field names too.
-	IsSet            bool              `json:"set,omitempty"`      // Affects YAML field names too.
 	Provides         []*DefaultPackage `json:"provides,omitempty"` // Affects YAML field names too.
+	Hidden           bool              `json:"hidden,omitempty"`   // Affects YAML field names too.
 
-	// TODO: Annotations?
+	// Annotations are used for core features/options
+	Annotations map[string]string `json:"annotations,omitempty"` // Affects YAML field names too
 
 	// Path is set only internally when tree is loaded from disk
 	Path string `json:"path,omitempty"`
 
-	Description string   `json:"description,omitempty"`
-	Uri         []string `json:"uri,omitempty"`
-	License     string   `json:"license,omitempty"`
+	Description    string   `json:"description,omitempty"`
+	Uri            []string `json:"uri,omitempty"`
+	License        string   `json:"license,omitempty"`
+	BuildTimestamp string   `json:"buildtimestamp,omitempty"`
 
 	Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
 }
@@ -197,9 +210,9 @@ func (p *DefaultPackage) GetFingerPrint() string {
 	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version)
 }
 
-func (p *DefaultPackage) HashFingerprint() string {
+func (p *DefaultPackage) HashFingerprint(salt string) string {
 	h := md5.New()
-	io.WriteString(h, p.GetFingerPrint())
+	io.WriteString(h, fmt.Sprintf("%s-%s", p.GetFingerPrint(), salt))
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
@@ -221,6 +234,16 @@ func (p *DefaultPackage) GetPackageName() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.Category)
 }
 
+// GetBuildTimestamp returns the package build timestamp
+func (p *DefaultPackage) GetBuildTimestamp() string {
+	return p.BuildTimestamp
+}
+
+// SetBuildTimestamp sets the package Build timestamp
+func (p *DefaultPackage) SetBuildTimestamp(s string) {
+	p.BuildTimestamp = s
+}
+
 // GetPath returns the path where the definition file was found
 func (p *DefaultPackage) GetPath() string {
 	return p.Path
@@ -238,26 +261,24 @@ func (p *DefaultPackage) IsSelector() bool {
 	return strings.ContainsAny(p.GetVersion(), "<>=")
 }
 
+func (p *DefaultPackage) IsHidden() bool {
+	return p.Hidden
+}
+
 func (p *DefaultPackage) HasLabel(label string) bool {
-	ans := false
-	for k, _ := range p.Labels {
-		if k == label {
-			ans = true
-			break
-		}
-	}
-	return ans
+	return helpers.MapHasKey(&p.Labels, label)
 }
 
 func (p *DefaultPackage) MatchLabel(r *regexp.Regexp) bool {
-	ans := false
-	for k, v := range p.Labels {
-		if r.MatchString(k + "=" + v) {
-			ans = true
-			break
-		}
-	}
-	return ans
+	return helpers.MapMatchRegex(&p.Labels, r)
+}
+
+func (p *DefaultPackage) HasAnnotation(label string) bool {
+	return helpers.MapHasKey(&p.Annotations, label)
+}
+
+func (p *DefaultPackage) MatchAnnotation(r *regexp.Regexp) bool {
+	return helpers.MapMatchRegex(&p.Annotations, r)
 }
 
 // AddUse adds a use to a package
@@ -300,15 +321,6 @@ func (p *DefaultPackage) Yaml() ([]byte, error) {
 	return y, nil
 }
 
-func (p *DefaultPackage) IsFlagged(b bool) Package {
-	p.IsSet = b
-	return p
-}
-
-func (p *DefaultPackage) Flagged() bool {
-	return p.IsSet
-}
-
 func (p *DefaultPackage) GetName() string {
 	return p.Name
 }
@@ -316,6 +328,9 @@ func (p *DefaultPackage) GetName() string {
 func (p *DefaultPackage) GetVersion() string {
 	return p.Version
 }
+func (p *DefaultPackage) SetVersion(v string) {
+	p.Version = v
+}
 func (p *DefaultPackage) GetDescription() string {
 	return p.Description
 }
@@ -354,9 +369,18 @@ func (p *DefaultPackage) AddLabel(k, v string) {
 	}
 	p.Labels[k] = v
 }
+func (p *DefaultPackage) AddAnnotation(k, v string) {
+	if p.Annotations == nil {
+		p.Annotations = make(map[string]string, 0)
+	}
+	p.Annotations[k] = v
+}
 func (p *DefaultPackage) GetLabels() map[string]string {
 	return p.Labels
 }
+func (p *DefaultPackage) GetAnnotations() map[string]string {
+	return p.Annotations
+}
 func (p *DefaultPackage) GetProvides() []*DefaultPackage {
 	return p.Provides
 }
@@ -738,7 +762,6 @@ func (p *DefaultPackage) Explain() {
 	fmt.Println("Name: ", p.GetName())
 	fmt.Println("Category: ", p.GetCategory())
 	fmt.Println("Version: ", p.GetVersion())
-	fmt.Println("Installed: ", p.IsSet)
 
 	for _, req := range p.GetRequires() {
 		fmt.Println("\t-> ", req)

pkg/package/package_test.go

@@ -36,8 +36,8 @@ var _ = Describe("Package", func() {
 		})
 
 		It("Generates packages fingerprint's hashes", func() {
-			Expect(a.HashFingerprint()).ToNot(Equal(a1.HashFingerprint()))
-			Expect(a.HashFingerprint()).To(Equal("c64caa391b79adb598ad98e261aa79a0"))
+			Expect(a.HashFingerprint("")).ToNot(Equal(a1.HashFingerprint("")))
+			Expect(a.HashFingerprint("")).To(Equal("76972ef6991ec6102f33b401105c1351"))
 		})
 	})
 
@@ -46,6 +46,7 @@ var _ = Describe("Package", func() {
 		a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a11 := NewPackage("A", "1.1", []*DefaultPackage{}, []*DefaultPackage{})
 		a01 := NewPackage("A", "0.1", []*DefaultPackage{}, []*DefaultPackage{})
+		re := regexp.MustCompile("project[0-9][=].*")
 		It("Expands correctly", func() {
 			definitions := NewInMemoryDatabase(false)
 			for _, p := range []Package{a1, a11, a01} {
@@ -60,6 +61,8 @@ var _ = Describe("Package", func() {
 			Expect(len(lst)).To(Equal(2))
 			p := lst.Best(nil)
 			Expect(p).To(Equal(a11))
+			// Test annotation with null map
+			Expect(a.MatchAnnotation(re)).To(Equal(false))
 		})
 	})
 
@@ -91,6 +94,34 @@ var _ = Describe("Package", func() {
 		})
 	})
 
+	Context("Find annotations on packages", func() {
+		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		a.AddAnnotation("project1", "test1")
+		a.AddAnnotation("label2", "value1")
+		b := NewPackage("B", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+		b.AddAnnotation("project2", "test2")
+		b.AddAnnotation("label2", "value1")
+		It("Expands correctly", func() {
+			var err error
+			definitions := NewInMemoryDatabase(false)
+			for _, p := range []Package{a, b} {
+				_, err = definitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			re := regexp.MustCompile("project[0-9][=].*")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(re).ToNot(BeNil())
+			Expect(a.HasAnnotation("label2")).To(Equal(true))
+			Expect(a.HasAnnotation("label3")).To(Equal(false))
+			Expect(a.HasAnnotation("project1")).To(Equal(true))
+			Expect(b.HasAnnotation("project2")).To(Equal(true))
+			Expect(b.HasAnnotation("label2")).To(Equal(true))
+			Expect(b.MatchAnnotation(re)).To(Equal(true))
+			Expect(a.MatchAnnotation(re)).To(Equal(true))
+
+		})
+	})
+
 	Context("Check description", func() {
 		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a.SetDescription("Description A")
@@ -283,7 +314,6 @@ var _ = Describe("Package", func() {
 
 			Expect(p.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(p.GetName()).To(Equal(a.GetName()))
-			Expect(p.Flagged()).To(Equal(a.Flagged()))
 			Expect(p.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(p.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(p.GetRequires())).To(Equal(len(a.GetRequires())))
@@ -343,7 +373,6 @@ var _ = Describe("Package", func() {
 			a2 := a.Clone()
 			Expect(a2.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(a2.GetName()).To(Equal(a.GetName()))
-			Expect(a2.Flagged()).To(Equal(a.Flagged()))
 			Expect(a2.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(a2.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(a2.GetRequires())).To(Equal(len(a.GetRequires())))

pkg/repository/loader.go

@@ -28,7 +28,7 @@ import (
 )
 
 func LoadRepositories(c *LuetConfig) error {
-	var regexRepo = regexp.MustCompile(`.yml$`)
+	var regexRepo = regexp.MustCompile(`.yml$|.yaml$`)
 
 	for _, rdir := range c.RepositoriesConfDir {
 		Debug("Parsing Repository Directory", rdir, "...")

pkg/repository/repository_test.go

@@ -35,12 +35,22 @@ var _ = Describe("Repository", func() {
 
 		It("Chec Load Repository 1", func() {
 			Expect(err).Should(BeNil())
-			Expect(len(cfg.SystemRepositories)).Should(Equal(1))
+			Expect(len(cfg.SystemRepositories)).Should(Equal(2))
 			Expect(cfg.SystemRepositories[0].Name).Should(Equal("test1"))
 			Expect(cfg.SystemRepositories[0].Priority).Should(Equal(999))
 			Expect(cfg.SystemRepositories[0].Type).Should(Equal("disk"))
 			Expect(len(cfg.SystemRepositories[0].Urls)).Should(Equal(1))
 			Expect(cfg.SystemRepositories[0].Urls[0]).Should(Equal("tests/repos/test1"))
 		})
+
+		It("Chec Load Repository 2", func() {
+			Expect(err).Should(BeNil())
+			Expect(len(cfg.SystemRepositories)).Should(Equal(2))
+			Expect(cfg.SystemRepositories[1].Name).Should(Equal("test2"))
+			Expect(cfg.SystemRepositories[1].Priority).Should(Equal(1000))
+			Expect(cfg.SystemRepositories[1].Type).Should(Equal("disk"))
+			Expect(len(cfg.SystemRepositories[1].Urls)).Should(Equal(1))
+			Expect(cfg.SystemRepositories[1].Urls[0]).Should(Equal("tests/repos/test2"))
+		})
 	})
 })

pkg/solver/solver.go

@@ -36,7 +36,10 @@ type PackageSolver interface {
 	Conflicts(pack pkg.Package, lsp pkg.Packages) (bool, error)
 
 	World() pkg.Packages
-	Upgrade(checkconflicts bool) (pkg.Packages, PackagesAssertions, error)
+	Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAssertions, error)
+
+	UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssertions, error)
+	UninstallUniverse(toremove pkg.Packages) (pkg.Packages, error)
 
 	SetResolver(PackageResolver)
 
@@ -95,6 +98,16 @@ func (s *Solver) noRulesWorld() bool {
 	return true
 }
 
+func (s *Solver) noRulesInstalled() bool {
+	for _, p := range s.Installed() {
+		if len(p.GetConflicts()) != 0 || len(p.GetRequires()) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
 func (s *Solver) BuildInstalled() (bf.Formula, error) {
 	var formulas []bf.Formula
 	for _, p := range s.Installed() {
@@ -251,7 +264,164 @@ func (s *Solver) ConflictsWithInstalled(p pkg.Package) (bool, error) {
 	return s.ConflictsWith(p, s.Installed())
 }
 
-func (s *Solver) Upgrade(checkconflicts bool) (pkg.Packages, PackagesAssertions, error) {
+// UninstallUniverse takes a list of candidate package and return a list of packages that would be removed
+// in order to purge the candidate. Uses the solver to check constraints and nothing else
+//
+// It can be compared to the counterpart Uninstall as this method acts like a uninstall --full
+// it removes all the packages and its deps. taking also in consideration other packages that might have
+// revdeps
+func (s *Solver) UninstallUniverse(toremove pkg.Packages) (pkg.Packages, error) {
+
+	if s.noRulesInstalled() {
+		return s.getList(s.InstalledDatabase, toremove)
+	}
+
+	// resolve to packages from the db
+	toRemove, err := s.getList(s.InstalledDatabase, toremove)
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	var formulas []bf.Formula
+	r, err := s.BuildInstalled()
+	if err != nil {
+		return nil, errors.Wrap(err, "Package not found in definition db")
+	}
+
+	// SAT encode the clauses against the world
+	for _, p := range toRemove.Unique() {
+		encodedP, err := p.Encode(s.InstalledDatabase)
+		if err != nil {
+			return nil, errors.Wrap(err, "Package not found in definition db")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(bf.Not(P), r))
+	}
+
+	markedForRemoval := pkg.Packages{}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, errors.New("Failed finding a solution")
+	}
+	assertion, err := DecodeModel(model, s.InstalledDatabase)
+	if err != nil {
+		return nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+	}
+	return markedForRemoval, nil
+}
+
+// UpgradeUniverse mark packages for removal and returns a solution. It considers
+// the Universe db as authoritative
+// See also on the subject: https://arxiv.org/pdf/1007.1021.pdf
+func (s *Solver) UpgradeUniverse(dropremoved bool) (pkg.Packages, PackagesAssertions, error) {
+	// we first figure out which aren't up-to-date
+	// which has to be removed
+	// and which needs to be upgraded
+	notUptodate := pkg.Packages{}
+	removed := pkg.Packages{}
+	toUpgrade := pkg.Packages{}
+
+	// TODO: this is memory expensive, we need to optimize this
+	universe := pkg.NewInMemoryDatabase(false)
+	for _, p := range s.DefinitionDatabase.World() {
+		universe.CreatePackage(p)
+	}
+	for _, p := range s.Installed() {
+		universe.CreatePackage(p)
+	}
+
+	// Grab all the installed ones, see if they are eligible for update
+	for _, p := range s.Installed() {
+		available, err := universe.FindPackageVersions(p)
+		if err != nil {
+			removed = append(removed, p)
+		}
+		if len(available) == 0 {
+			continue
+		}
+
+		bestmatch := available.Best(nil)
+		// Found a better version available
+		if !bestmatch.Matches(p) {
+			notUptodate = append(notUptodate, p)
+			toUpgrade = append(toUpgrade, bestmatch)
+		}
+	}
+
+	// resolve to packages from the db to be able to encode correctly
+	oldPackages, err := s.getList(universe, notUptodate)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't get package marked for removal from universe")
+	}
+
+	updates, err := s.getList(universe, toUpgrade)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't get package marked for update from universe")
+	}
+
+	var formulas []bf.Formula
+
+	// Build constraints for the whole defdb
+	r, err := s.BuildWorld(true)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't build world constraints")
+	}
+
+	// Treat removed packages from universe as marked for deletion
+	if dropremoved {
+		oldPackages = append(oldPackages, removed...)
+	}
+
+	// SAT encode the clauses against the world
+	for _, p := range oldPackages.Unique() {
+		encodedP, err := p.Encode(universe)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "couldn't encode package")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(bf.Not(P), r))
+	}
+
+	for _, p := range updates {
+		encodedP, err := p.Encode(universe)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "couldn't encode package")
+		}
+		P := bf.Var(encodedP)
+		formulas = append(formulas, bf.And(P, r))
+	}
+
+	markedForRemoval := pkg.Packages{}
+	model := bf.Solve(bf.And(formulas...))
+	if model == nil {
+		return nil, nil, errors.New("Failed finding a solution")
+	}
+
+	assertion, err := DecodeModel(model, universe)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "while decoding model from solution")
+	}
+	for _, a := range assertion {
+		if !a.Value {
+			if p, err := s.InstalledDatabase.FindPackage(a.Package); err == nil {
+				markedForRemoval = append(markedForRemoval, p)
+			}
+
+		}
+
+	}
+	return markedForRemoval, assertion, nil
+}
+
+func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAssertions, error) {
 
 	// First get candidates that needs to be upgraded..
 
@@ -277,8 +447,16 @@ func (s *Solver) Upgrade(checkconflicts bool) (pkg.Packages, PackagesAssertions,
 			}
 		}
 	}
+
 	s2 := NewSolver(installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
 	s2.SetResolver(s.Resolver)
+	if !full {
+		ass := PackagesAssertions{}
+		for _, i := range toInstall {
+			ass = append(ass, PackageAssert{Package: i.(*pkg.DefaultPackage), Value: true})
+		}
+	}
+
 	// Then try to uninstall the versions in the system, and store that tree
 	for _, p := range toUninstall {
 		r, err := s.Uninstall(p, checkconflicts, false)
@@ -291,8 +469,8 @@ func (s *Solver) Upgrade(checkconflicts bool) (pkg.Packages, PackagesAssertions,
 				return nil, nil, errors.Wrap(err, "Could not compute upgrade - couldn't remove copy of package targetted for removal")
 			}
 		}
-
 	}
+
 	r, e := s2.Install(toInstall)
 	return toUninstall, r, e
 	// To that tree, ask to install the versions that should be upgraded, and try to solve
@@ -354,7 +532,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 	for _, a := range asserts {
 		if a.Value {
 			if !checkconflicts {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 				continue
 			}
 
@@ -365,7 +543,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 
 			// If doesn't conflict with installed we just consider it for removal and look for the next one
 			if !c {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 				continue
 			}
 
@@ -375,7 +553,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 				return nil, err
 			}
 			if !c {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 			}
 
 		}

pkg/solver/solver_test.go

@@ -574,57 +574,366 @@ var _ = Describe("Solver", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 
-		It("Uninstalls simple package correctly", func() {
+		Context("Uninstall", func() {
+			It("Uninstalls simple package correctly", func() {
 
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(dbInstalled, dbDefinitions, db)
+
+				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("Uninstalls simple package expanded correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1.2", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(dbInstalled, dbDefinitions, db)
+
+				solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"}, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
 
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
+				Expect(solution).To(ContainElement(A))
 
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("Uninstalls simple packages not in world correctly", func() {
 
-			//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
-			Expect(len(solution)).To(Equal(1))
-		})
-		It("Uninstalls simple package expanded correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
 
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "1.2", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
 
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("Uninstalls complex packages not in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
-			}
-			s = NewSolver(dbInstalled, dbDefinitions, db)
 
-			solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"}, true, true)
-			Expect(err).ToNot(HaveOccurred())
+				Expect(solution).To(ContainElement(A))
 
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("Uninstalls complex packages correctly, even if shared deps are required by system packages", func() {
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.Uninstall(A, true, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).ToNot(ContainElement(B))
+
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("Uninstalls complex packages in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{C}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Uninstall(A, true, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(C))
+
+				Expect(len(solution)).To(Equal(2))
+			})
+
+			It("Uninstalls complex package correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				//	C // installed
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Uninstall(A, true, true)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(D))
+
+				Expect(len(solution)).To(Equal(3))
+
+			})
+
+			It("UninstallUniverse simple package correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(dbInstalled, dbDefinitions, db)
+
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("UninstallUniverse simple package expanded correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "1.2", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				s = NewSolver(dbInstalled, dbDefinitions, db)
+
+				solution, err := s.UninstallUniverse(pkg.Packages{
+					&pkg.DefaultPackage{Name: "A", Version: ">1.0"}})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+			It("UninstallUniverse simple packages not in world correctly", func() {
+
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+
+				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
+				Expect(len(solution)).To(Equal(1))
+			})
+
+			It("UninstallUniverse complex packages not in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+
+				Expect(len(solution)).To(Equal(2))
+			})
+
+			It("UninstallUniverse complex packages correctly, even if shared deps are required by system packages", func() {
+				// Here we diff a lot from standard Uninstall:
+				// all the packages that has reverse deps will be removed (aka --full)
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(C))
+
+				Expect(len(solution)).To(Equal(3))
+			})
+
+			It("UninstallUniverse complex packages in world correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{C}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(C))
+
+				Expect(len(solution)).To(Equal(2))
+			})
+
+			It("UninstallUniverse complex package correctly", func() {
+				C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+				//	C // installed
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{A, B, C, D} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.UninstallUniverse(pkg.Packages{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(D))
+
+				Expect(len(solution)).To(Equal(3))
+
+			})
 
-			//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
-			Expect(len(solution)).To(Equal(1))
 		})
 		It("Find conflicts", func() {
 
@@ -810,131 +1119,6 @@ var _ = Describe("Solver", func() {
 			Expect(val).ToNot(BeTrue())
 		})
 
-		It("Uninstalls simple packages not in world correctly", func() {
-
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-
-			for _, p := range []pkg.Package{B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-
-			//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
-			Expect(len(solution)).To(Equal(1))
-		})
-
-		It("Uninstalls complex packages not in world correctly", func() {
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
-
-			for _, p := range []pkg.Package{B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(1))
-		})
-
-		It("Uninstalls complex packages correctly, even if shared deps are required by system packages", func() {
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-			Expect(solution).ToNot(ContainElement(B.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(1))
-		})
-
-		It("Uninstalls complex packages in world correctly", func() {
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{C}, []*pkg.DefaultPackage{})
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-			Expect(solution).To(ContainElement(C.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(2))
-		})
-
-		It("Uninstalls complex package correctly", func() {
-			C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
-			B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
-			A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
-			//	C // installed
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbDefinitions.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			for _, p := range []pkg.Package{A, B, C, D} {
-				_, err := dbInstalled.CreatePackage(p)
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			solution, err := s.Uninstall(A, true, true)
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(solution).To(ContainElement(A.IsFlagged(false)))
-			Expect(solution).To(ContainElement(B.IsFlagged(false)))
-			Expect(solution).To(ContainElement(D.IsFlagged(false)))
-
-			Expect(len(solution)).To(Equal(3))
-
-		})
-
 	})
 
 	Context("Conflict set", func() {
@@ -1045,7 +1229,7 @@ var _ = Describe("Solver", func() {
 				_, err := dbInstalled.CreatePackage(p)
 				Expect(err).ToNot(HaveOccurred())
 			}
-			uninstall, solution, err := s.Upgrade(true)
+			uninstall, solution, err := s.Upgrade(true, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(len(uninstall)).To(Equal(1))
@@ -1058,5 +1242,30 @@ var _ = Describe("Solver", func() {
 			Expect(len(solution)).To(Equal(3))
 
 		})
+
+		It("UpgradeUniverse upgrades correctly", func() {
+			for _, p := range []pkg.Package{A1, B, C} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, B} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.UpgradeUniverse(true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(1))
+			Expect(uninstall[0].GetName()).To(Equal("a"))
+			Expect(uninstall[0].GetVersion()).To(Equal("1.1"))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: A1, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
+			Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
+
+			Expect(len(solution)).To(Equal(4))
+		})
 	})
 })

pkg/spectooling/definition.go

@@ -0,0 +1,112 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>,
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package spectooling
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"gopkg.in/yaml.v2"
+)
+
+type DefaultPackageSanitized struct {
+	Name             string                     `json:"name" yaml:"name"`
+	Version          string                     `json:"version" yaml:"version"`
+	Category         string                     `json:"category" yaml:"category"`
+	UseFlags         []string                   `json:"use_flags,omitempty" yaml:"use_flags,omitempty"`
+	PackageRequires  []*DefaultPackageSanitized `json:"requires,omitempty" yaml:"requires,omitempty"`
+	PackageConflicts []*DefaultPackageSanitized `json:"conflicts,omitempty" yaml:"conflicts,omitempty"`
+	Provides         []*DefaultPackageSanitized `json:"provides,omitempty" yaml:"provides,omitempty"`
+
+	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
+
+	// Path is set only internally when tree is loaded from disk
+	Path string `json:"path,omitempty" yaml:"path,omitempty"`
+
+	Description string   `json:"description,omitempty" yaml:"description,omitempty"`
+	Uri         []string `json:"uri,omitempty" yaml:"uri,omitempty"`
+	License     string   `json:"license,omitempty" yaml:"license,omitempty"`
+	Hidden      bool     `json:"hidden,omitempty" yaml:"hidden,omitempty"`
+
+	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
+}
+
+func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
+	ans := &DefaultPackageSanitized{
+		Name:        p.GetName(),
+		Version:     p.GetVersion(),
+		Category:    p.GetCategory(),
+		UseFlags:    p.GetUses(),
+		Hidden:      p.IsHidden(),
+		Path:        p.GetPath(),
+		Description: p.GetDescription(),
+		Uri:         p.GetURI(),
+		License:     p.GetLicense(),
+		Labels:      p.GetLabels(),
+		Annotations: p.GetAnnotations(),
+	}
+
+	if p.GetRequires() != nil && len(p.GetRequires()) > 0 {
+		ans.PackageRequires = []*DefaultPackageSanitized{}
+		for _, r := range p.GetRequires() {
+			// I avoid recursive call of NewDefaultPackageSanitized
+			ans.PackageRequires = append(ans.PackageRequires,
+				&DefaultPackageSanitized{
+					Name:     r.Name,
+					Version:  r.Version,
+					Category: r.Category,
+					Hidden:   r.IsHidden(),
+				},
+			)
+		}
+	}
+
+	if p.GetConflicts() != nil && len(p.GetConflicts()) > 0 {
+		ans.PackageConflicts = []*DefaultPackageSanitized{}
+		for _, c := range p.GetConflicts() {
+			// I avoid recursive call of NewDefaultPackageSanitized
+			ans.PackageConflicts = append(ans.PackageConflicts,
+				&DefaultPackageSanitized{
+					Name:     c.Name,
+					Version:  c.Version,
+					Category: c.Category,
+					Hidden:   c.IsHidden(),
+				},
+			)
+		}
+	}
+
+	if p.GetProvides() != nil && len(p.GetProvides()) > 0 {
+		ans.Provides = []*DefaultPackageSanitized{}
+		for _, prov := range p.GetProvides() {
+			// I avoid recursive call of NewDefaultPackageSanitized
+			ans.Provides = append(ans.Provides,
+				&DefaultPackageSanitized{
+					Name:     prov.Name,
+					Version:  prov.Version,
+					Category: prov.Category,
+					Hidden:   prov.IsHidden(),
+				},
+			)
+		}
+	}
+
+	return ans
+}
+
+func (p *DefaultPackageSanitized) Yaml() ([]byte, error) {
+	return yaml.Marshal(p)
+}

pkg/spectooling/package_test.go

@@ -0,0 +1,87 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>,
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package spectooling_test
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+	. "github.com/mudler/luet/pkg/spectooling"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Spec Tooling", func() {
+	Context("Conversion1", func() {
+
+		b := pkg.NewPackage("B", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		c := pkg.NewPackage("C", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		d := pkg.NewPackage("D", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		p1 := pkg.NewPackage("A", "1.0", []*pkg.DefaultPackage{b, c}, []*pkg.DefaultPackage{d})
+		virtual := pkg.NewPackage("E", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+		virtual.SetCategory("virtual")
+		p1.Provides = []*pkg.DefaultPackage{virtual}
+		p1.AddLabel("label1", "value1")
+		p1.AddLabel("label2", "value2")
+		p1.SetDescription("Package1")
+		p1.SetCategory("cat1")
+		p1.SetLicense("GPL")
+		p1.AddURI("https://github.com/mudler/luet")
+		p1.AddUse("systemd")
+		It("Convert pkg1", func() {
+			res := NewDefaultPackageSanitized(p1)
+			expected_res := &DefaultPackageSanitized{
+				Name:     "A",
+				Version:  "1.0",
+				Category: "cat1",
+				PackageRequires: []*DefaultPackageSanitized{
+					&DefaultPackageSanitized{
+						Name:    "B",
+						Version: "1.0",
+					},
+					&DefaultPackageSanitized{
+						Name:    "C",
+						Version: "1.0",
+					},
+				},
+				PackageConflicts: []*DefaultPackageSanitized{
+					&DefaultPackageSanitized{
+						Name:    "D",
+						Version: "1.0",
+					},
+				},
+				Provides: []*DefaultPackageSanitized{
+					&DefaultPackageSanitized{
+						Name:     "E",
+						Category: "virtual",
+						Version:  "1.0",
+					},
+				},
+				Labels: map[string]string{
+					"label1": "value1",
+					"label2": "value2",
+				},
+				Description: "Package1",
+				License:     "GPL",
+				Uri:         []string{"https://github.com/mudler/luet"},
+				UseFlags:    []string{"systemd"},
+			}
+
+			Expect(res).To(Equal(expected_res))
+		})
+
+	})
+})

pkg/spectooling/spectooling_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>,
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package spectooling_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Spec Tooling Suite")
+}

pkg/tree/builder/gentoo/simpleparser.go

@@ -353,6 +353,12 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) (pkg.Packages, error) {
 		return pkg.Packages{}, err
 	}
 
+	// Retrieve slot
+	slot, ok := vars["SLOT"]
+	if ok && slot.String() != "0" {
+		pack.SetCategory(fmt.Sprintf("%s-%s", gp.Category, slot.String()))
+	}
+
 	// TODO: Handle this a bit better
 	iuse, ok := vars["IUSE"]
 	if ok {
@@ -417,6 +423,7 @@ func (ep *SimpleEbuildParser) ScanEbuild(path string) (pkg.Packages, error) {
 		for _, d := range gRDEPEND.GetDependencies() {
 
 			//TODO: Resolve to db or create a new one.
+			//TODO: handle SLOT too.
 			dep := &pkg.DefaultPackage{
 				Name:     d.Dep.Name,
 				Version:  d.Dep.Version + d.Dep.VersionSuffix,

pkg/tree/installer_recipe.go

@@ -21,6 +21,7 @@
 package tree
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -70,10 +71,12 @@ func (r *InstallerRecipe) Save(path string) error {
 
 func (r *InstallerRecipe) Load(path string) error {
 
-	// tmpfile, err := ioutil.TempFile("", "luet")
-	// if err != nil {
-	// 	return err
-	// }
+	if !helpers.Exists(path) {
+		return errors.New(fmt.Sprintf(
+			"Path %s doesn't exit.", path,
+		))
+	}
+
 	r.SourcePath = append(r.SourcePath, path)
 
 	//r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))

pkg/tree/recipes.go

@@ -21,11 +21,14 @@
 package tree
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
+	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
+	spectooling "github.com/mudler/luet/pkg/spectooling"
 	"github.com/pkg/errors"
 )
 
@@ -42,7 +45,7 @@ type Recipe struct {
 }
 
 func WriteDefinitionFile(p pkg.Package, definitionFilePath string) error {
-	data, err := p.Yaml()
+	data, err := spectooling.NewDefaultPackageSanitized(p).Yaml()
 	if err != nil {
 		return err
 	}
@@ -73,6 +76,12 @@ func (r *Recipe) Load(path string) error {
 	// if err != nil {
 	// 	return err
 	// }
+	if !helpers.Exists(path) {
+		return errors.New(fmt.Sprintf(
+			"Path %s doesn't exit.", path,
+		))
+	}
+
 	r.SourcePath = append(r.SourcePath, path)
 
 	if r.Database == nil {

pkg/tree/tree_test.go

@@ -23,6 +23,7 @@ package tree_test
 import (
 	"io/ioutil"
 	"os"
+	"regexp"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -171,4 +172,23 @@ var _ = Describe("Tree", func() {
 		})
 	})
 
+	Context("Simple tree with annotations", func() {
+		It("Read tree with annotations", func() {
+			db := pkg.NewInMemoryDatabase(false)
+			generalRecipe := NewCompilerRecipe(db)
+			r := regexp.MustCompile("^label")
+
+			err := generalRecipe.Load("../../tests/fixtures/annotations")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().World())).To(Equal(1))
+			pack, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "pkgA", Category: "test", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(pack.HasAnnotation("label1")).To(Equal(true))
+			Expect(pack.HasAnnotation("label3")).To(Equal(false))
+			Expect(pack.MatchAnnotation(r)).To(Equal(true))
+		})
+	})
+
 })

tests/fixtures/annotations/pkgA/0.1/build.yaml

@@ -0,0 +1,3 @@
+image: "alpine"
+steps:
+  - echo "test" > /file1

tests/fixtures/annotations/pkgA/0.1/definition.yaml

@@ -0,0 +1,6 @@
+category: "test"
+name: "pkgA"
+version: "0.1"
+annotations:
+  label1: "value1"
+  label2: "value2"

tests/fixtures/caps/pkgA/0.1/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+- apk add libcap
+unpack: true
+includes:
+  - /file1
+steps:
+- echo "test" > /file1
+- setcap cap_net_raw+ep /file1

tests/fixtures/caps/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "caps"
+version: "0.1"

tests/fixtures/caps/pkgB/0.1/build.yaml

@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+- apk add libcap
+steps:
+- echo "test" > /file2
+- setcap cap_net_raw+ep /file2

tests/fixtures/caps/pkgB/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "caps2"
+version: "0.1"

tests/fixtures/config_protect/a/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo c > /c
+  - echo c > /cd
+  - mkdir /etc/a
+  - echo config > /etc/a/conf

tests/fixtures/config_protect/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.0"

tests/fixtures/config_protect_annotation/a/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo c > /c
+  - echo c > /cd
+  - mkdir /opt/etc
+  - echo config > /opt/etc/conf

tests/fixtures/config_protect_annotation/a/definition.yaml

@@ -0,0 +1,5 @@
+category: "test"
+name: "a"
+version: "1.0"
+annotations:
+  config_protect: "/opt/etc"

tests/fixtures/repos.conf.d/repo-test2.yaml

@@ -0,0 +1,9 @@
+name: "test2"
+description: "Test2 Repository"
+type: "disk"
+urls:
+  - "tests/repos/test2"
+priority: 1000
+enable: true
+# auth:
+#   token: "xxxxx"

tests/fixtures/symlinks/pkgA/0.1/build.yaml

@@ -0,0 +1,5 @@
+image: "alpine"
+prelude:
+- echo "test" > /file2
+steps:
+- ln -s /file2 /file1

tests/fixtures/symlinks/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "pkgAsym"
+version: "0.1"

tests/fixtures/symlinks/pkgB/0.1/build.yaml

@@ -0,0 +1,8 @@
+image: "alpine"
+unpack: true
+includes:
+- /file3
+prelude:
+- echo "test" > /file2
+steps:
+- ln -s /file2 /file3

tests/fixtures/symlinks/pkgB/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "pkgBsym"
+version: "0.1"

tests/fixtures/templates/build.yaml

@@ -0,0 +1 @@
+image: "{{.Values.name}}:{{.Values.test.foo}}"

tests/fixtures/templates/definition.yaml

@@ -0,0 +1,5 @@
+category: "test"
+name: "b"
+version: "1.0"
+test:
+  foo: "bar"

tests/fixtures/upgrade_old_repo_revision/c/build.yaml

@@ -0,0 +1,10 @@
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo c > /c
+  - echo c > /cd
+requires:
+- category: "test"
+  name: "a"
+  version: ">=1.0"

tests/fixtures/upgrade_old_repo_revision/c/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.0"

tests/fixtures/upgrade_old_repo_revision/cat/a/a/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4
+requires:
+- category: "test"
+  name: "b"
+  version: "1.0"

tests/fixtures/upgrade_old_repo_revision/cat/a/a/definition.yaml

@@ -0,0 +1,8 @@
+category: "test"
+name: "a"
+version: "1.1"
+requires:
+- category: "test2"
+  name: "b"
+  version: "1.0"
+

tests/fixtures/upgrade_old_repo_revision/cat/b-1.1/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+  - chmod +x generate.sh
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - ./generate.sh

tests/fixtures/upgrade_old_repo_revision/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/fixtures/upgrade_old_repo_revision/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/integration/05_upgrade.sh

@@ -110,6 +110,7 @@ testInstall() {
 testUpgrade() {
     upgrade=$(luet --config $tmpdir/luet.yaml upgrade)
     installst=$?
+    echo "$upgrade"
     assertEquals 'install test successfully' "$installst" "0"
     assertTrue 'package uninstalled B' "[ ! -e '$tmpdir/testrootfs/test5' ]"
     assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/newc' ]"

tests/integration/09_docker.sh

@@ -60,19 +60,15 @@ testInstall() {
     docker build --rm --no-cache -t luet:test .
     docker rm luet-runtime-test || true
     docker run --name luet-runtime-test \
-       -ti -v /tmp:/tmp \
-       -v $tmpdir/luet.yaml:/etc/luet/.luet.yaml:ro \
+       -v /tmp:/tmp \
+       -v $tmpdir/luet.yaml:/etc/luet/luet.yaml:ro \
        luet:test install seed/alpine
     installst=$?
     assertEquals 'install test successfully' "0" "$installst"
 
     docker commit luet-runtime-test luet-runtime-test-image
-    test=$(docker run --rm -t --entrypoint /bin/sh luet-runtime-test-image -c 'echo "ftw"')
+    test=$(docker run --rm --entrypoint /bin/sh luet-runtime-test-image -c 'echo "ftw"')
     assertContains 'generated image runs successfully' "$test" "ftw"
-   # docker rm luet-runtime-test || true
-   # docker rmi luet-runtime-test-image || true
-
-   # docker rmi luet:test || true
 }
 
 # Load shUnit2.

tests/integration/11_upgrade_universe.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/b-1.0 
+    buildst=$?
+    assertTrue 'create package B 1.0' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/b-1.1
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package B 1.1' "[ -e '$tmpdir/testbuild/b-test-1.1.package.tar.gz' ]"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/a-1.0 
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package A 1.0' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/a-1.1
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    assertTrue 'create package A 1.1' "[ -e '$tmpdir/testbuild/a-test-1.1.package.tar.gz' ]"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/a-1.2
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    assertTrue 'create package A 1.2' "[ -e '$tmpdir/testbuild/a-test-1.2.package.tar.gz' ]"
+
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/c-1.0
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package C 1.0' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/b-1.0
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/test5' ]"
+
+    luet install --config $tmpdir/luet.yaml test/a-1.0
+    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    luet install --config $tmpdir/luet.yaml test/a-1.1
+    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package keeps old A' "[ -e '$tmpdir/testrootfs/testaa' ]"
+    assertTrue 'package new A was not installed' "[ ! -e '$tmpdir/testrootfs/testlatest' ]"
+
+    luet install --config $tmpdir/luet.yaml test/c-1.0
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed C' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testUpgrade() {
+    upgrade=$(luet --config $tmpdir/luet.yaml upgrade --universe --clean)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    echo "$upgrade"
+    assertTrue 'package uninstalled B' "[ ! -e '$tmpdir/testrootfs/test5' ]"
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/newc' ]"
+    assertTrue 'package uninstalled A' "[ ! -e '$tmpdir/testrootfs/testaa' ]"
+    assertTrue 'package installed new A' "[ -e '$tmpdir/testrootfs/testlatest' ]"
+
+    # It does remove C as well, no other package depends on it.
+    assertContains 'does contain test/c-1.0' "$upgrade" 'test/c-1.0'
+    assertNotContains 'does not attempt to download test/c-1.0' "$upgrade" 'test/c-1.0 downloaded'
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/12_config_protect.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/config_protect" --destination $tmpdir/testbuild --compression gzip test/a
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+
+    mkdir $tmpdir/config.protect.d
+
+    cat <<EOF > $tmpdir/config.protect.d/conf1.yml
+name: "protect1"
+dirs:
+- /etc/
+EOF
+
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_protect_confdir:
+    - $tmpdir/config.protect.d
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+
+
+testInstall() {
+
+    # Simulate previous installation
+    mkdir $tmpdir/testrootfs/etc/a -p
+    echo "fakeconf" > $tmpdir/testrootfs/etc/a/conf
+
+    luet install --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+
+    # Simulate config protect
+    assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect created' "[ -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
+}
+
+
+testUnInstall() {
+    luet uninstall --full --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    # TODO: we need remove it or not??
+    assertTrue 'config protect created' "[ -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/a-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/13_config_protect_annotation.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/config_protect_annotation" --destination $tmpdir/testbuild --compression gzip test/a
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/config_protect_annotation" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+
+    mkdir $tmpdir/config.protect.d
+
+    cat <<EOF > $tmpdir/config.protect.d/conf1.yml
+name: "protect1"
+dirs:
+- /etc/
+EOF
+
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_protect_confdir:
+    - $tmpdir/config.protect.d
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+
+
+testInstall() {
+    # Simulate previous installation
+    mkdir $tmpdir/testrootfs/opt/etc -p
+    echo "fakeconf" > $tmpdir/testrootfs/opt/etc/conf
+
+    luet install --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+
+    # Simulate config protect
+    assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect created' "[ -e '$tmpdir/testrootfs/opt/etc/._cfg0001_conf' ]"
+}
+
+
+testUnInstall() {
+    luet uninstall --full --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    # TODO: we need remove it or not??
+    assertTrue 'config protect created' "[ -e '$tmpdir/testrootfs/opt/etc/._cfg0001_conf' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/a-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/14_upgrade_revision.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_old_repo" --destination $tmpdir/testbuild --compression gzip --full --clean=true
+    buildst=$?
+    assertTrue 'create package B 1.0' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    mkdir $tmpdir/testbuild_revision
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_old_repo_revision" --destination $tmpdir/testbuild_revision --compression gzip --full --clean=true
+    buildst=$?
+    assertTrue 'create package B 1.0' "[ -e '$tmpdir/testbuild_revision/b-test-1.0.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/upgrade_old_repo" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild_revision/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/upgrade_old_repo_revision" \
+    --output $tmpdir/testbuild_revision \
+    --packages $tmpdir/testbuild_revision \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild_revision/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testUpgrade() {
+    luet install --config $tmpdir/luet.yaml test/b-1.0
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/test5' ]"
+
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild_revision"
+EOF
+
+    luet cleanup --config $tmpdir/luet.yaml
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+
+    luet upgrade --sync --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'upgrade test successfully' "$installst" "0"
+    assertTrue 'package uninstalled B' "[ ! -e '$tmpdir/testrootfs/test5' ]"
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/newc' ]"
+
+    content=$(luet upgrade --sync --config $tmpdir/luet.yaml)
+    installst=$?
+    assertNotContains 'didn not upgrade' "$content" "Uninstalling"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/15_symlinks.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/symlinks" --destination $tmpdir/testbuild --compression gzip --full --clean=true
+    buildst=$?
+    assertTrue 'create package pkgAsym 0.1' "[ -e '$tmpdir/testbuild/pkgAsym-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/symlinks" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/pkgAsym test/pkgBsym
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    ls -liah $tmpdir/testrootfs/
+    assertTrue 'package did not install file2' "[ ! -e '$tmpdir/testrootfs/file2' ]"
+    assertTrue 'package installed file3 as symlink' "[ -L '$tmpdir/testrootfs/file3' ]"
+    assertTrue 'package installed file1 as symlink' "[ -L '$tmpdir/testrootfs/file1' ]"
+
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/16_caps.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build -d --tree "$ROOT_DIR/tests/fixtures/caps" --same-owner=true --destination $tmpdir/testbuild --compression gzip --full --clean=true
+    buildst=$?
+    assertTrue 'create package caps 0.1' "[ -e '$tmpdir/testbuild/caps-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/caps" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    $ROOT_DIR/tests/integration/bin/luet install --config $tmpdir/luet.yaml test/caps-0.1 test/caps2-0.1
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+   
+    assertTrue 'package installed file1' "[ -e '$tmpdir/testrootfs/file1' ]"
+    assertTrue 'package installed file2' "[ -e '$tmpdir/testrootfs/file2' ]"
+
+    assertContains 'caps' "$(getcap $tmpdir/testrootfs/file1)" "cap_net_raw+ep"
+    assertContains 'caps' "$(getcap $tmpdir/testrootfs/file2)" "cap_net_raw+ep"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/17_config_protect_skip.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/config_protect" --destination $tmpdir/testbuild --compression gzip test/a
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+
+    mkdir $tmpdir/config.protect.d
+
+    cat <<EOF > $tmpdir/config.protect.d/conf1.yml
+name: "protect1"
+dirs:
+- /etc/
+EOF
+
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_protect_skip: true
+config_protect_confdir:
+    - $tmpdir/config.protect.d
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+
+
+testInstall() {
+
+    # Simulate previous installation
+    mkdir $tmpdir/testrootfs/etc/a -p
+    echo "fakeconf" > $tmpdir/testrootfs/etc/a/conf
+
+    luet install --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+
+    # Simulate config protect
+    assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect created' "[ ! -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
+}
+
+
+testUnInstall() {
+    luet uninstall --full --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/a-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/18_database.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c #> /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testDatabase() {
+    luet database create --config $tmpdir/luet.yaml $tmpdir/testbuild/c-test-1.0.metadata.yaml
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    createst=$?
+    assertEquals 'created package successfully' "$createst" "0"
+    assertTrue 'package not installed' "[ ! -e '$tmpdir/testrootfs/c' ]"
+
+    installed=$(luet --config $tmpdir/luet.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+    assertContains 'contains test/c-1.0' "$installed" 'test/c-1.0'
+    touch $tmpdir/testrootfs/c
+    
+    luet database remove --config $tmpdir/luet.yaml test/c-1.0
+    removetest=$?
+    assertEquals 'package removed successfully' "$removetest" "0"
+    assertTrue 'file not touched' "[ -e '$tmpdir/testrootfs/c' ]"
+
+    luet database create --config $tmpdir/luet.yaml $tmpdir/testbuild/c-test-1.0.metadata.yaml
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    createst=$?
+    assertEquals 'created package successfully' "$createst" "0"
+    assertTrue 'file still present' "[ -e '$tmpdir/testrootfs/c' ]"
+    
+    luet uninstall --config $tmpdir/luet.yaml test/c
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

vendor/github.com/Masterminds/goutils/.travis.yml

@@ -0,0 +1,18 @@
+language: go
+
+go:
+  - 1.6
+  - 1.7
+  - 1.8
+  - tip
+
+script:
+  - go test -v
+
+notifications:
+  webhooks:
+    urls:
+      - https://webhooks.gitter.im/e/06e3328629952dabe3e0
+    on_success: change  # options: [always|never|change] default: always
+    on_failure: always  # options: [always|never|change] default: always
+    on_start: never     # options: [always|never|change] default: always

vendor/github.com/Masterminds/goutils/CHANGELOG.md

@@ -0,0 +1,8 @@
+# 1.0.1 (2017-05-31)
+
+## Fixed
+- #21: Fix generation of alphanumeric strings (thanks @dbarranco)
+
+# 1.0.0 (2014-04-30)
+
+- Initial release.

vendor/github.com/Masterminds/goutils/LICENSE.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/Masterminds/goutils/README.md

@@ -0,0 +1,70 @@
+GoUtils
+===========
+[![Stability: Maintenance](https://masterminds.github.io/stability/maintenance.svg)](https://masterminds.github.io/stability/maintenance.html)
+[![GoDoc](https://godoc.org/github.com/Masterminds/goutils?status.png)](https://godoc.org/github.com/Masterminds/goutils) [![Build Status](https://travis-ci.org/Masterminds/goutils.svg?branch=master)](https://travis-ci.org/Masterminds/goutils) [![Build status](https://ci.appveyor.com/api/projects/status/sc2b1ew0m7f0aiju?svg=true)](https://ci.appveyor.com/project/mattfarina/goutils)
+
+
+GoUtils provides users with utility functions to manipulate strings in various ways. It is a Go implementation of some
+string manipulation libraries of Java Apache Commons. GoUtils includes the following Java Apache Commons classes:
+* WordUtils    
+* RandomStringUtils  
+* StringUtils (partial implementation)
+
+## Installation
+If you have Go set up on your system, from the GOPATH directory within the command line/terminal, enter this:
+
+	go get github.com/Masterminds/goutils
+
+If you do not have Go set up on your system, please follow the [Go installation directions from the documenation](http://golang.org/doc/install), and then follow the instructions above to install GoUtils.
+
+
+## Documentation
+GoUtils doc is available here: [![GoDoc](https://godoc.org/github.com/Masterminds/goutils?status.png)](https://godoc.org/github.com/Masterminds/goutils)
+
+
+## Usage
+The code snippets below show examples of how to use GoUtils. Some functions return errors while others do not. The first instance below, which does not return an error, is the `Initials` function (located within the `wordutils.go` file).
+
+    package main
+
+    import (
+        "fmt"
+    	"github.com/Masterminds/goutils"
+    )
+
+    func main() {
+
+    	// EXAMPLE 1: A goutils function which returns no errors
+        fmt.Println (goutils.Initials("John Doe Foo")) // Prints out "JDF"
+
+    }
+Some functions return errors mainly due to illegal arguements used as parameters. The code example below illustrates how to deal with function that returns an error. In this instance, the function is the `Random` function (located within the `randomstringutils.go` file).
+
+    package main
+
+    import (
+        "fmt"
+        "github.com/Masterminds/goutils"
+    )
+
+    func main() {
+
+        // EXAMPLE 2: A goutils function which returns an error
+        rand1, err1 := goutils.Random (-1, 0, 0, true, true)  
+
+        if err1 != nil {
+			fmt.Println(err1) // Prints out error message because -1 was entered as the first parameter in goutils.Random(...)
+		} else {
+			fmt.Println(rand1)
+		}
+
+    }
+
+## License
+GoUtils is licensed under the Apache License, Version 2.0. Please check the LICENSE.txt file or visit http://www.apache.org/licenses/LICENSE-2.0 for a copy of the license.
+
+## Issue Reporting
+Make suggestions or report issues using the Git issue tracker: https://github.com/Masterminds/goutils/issues
+
+## Website
+* [GoUtils webpage](http://Masterminds.github.io/goutils/)

vendor/github.com/Masterminds/goutils/appveyor.yml

@@ -0,0 +1,21 @@
+version: build-{build}.{branch}
+
+clone_folder: C:\gopath\src\github.com\Masterminds\goutils
+shallow_clone: true
+
+environment:
+  GOPATH: C:\gopath
+
+platform:
+  - x64
+
+build: off
+
+install:
+  - go version
+  - go env
+
+test_script:
+  - go test -v
+
+deploy: off

vendor/github.com/Masterminds/goutils/cryptorandomstringutils.go

@@ -0,0 +1,251 @@
+/*
+Copyright 2014 Alexander Okoli
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package goutils
+
+import (
+	"crypto/rand"
+	"fmt"
+	"math"
+	"math/big"
+	"regexp"
+	"unicode"
+)
+
+/*
+CryptoRandomNonAlphaNumeric creates a random string whose length is the number of characters specified.
+Characters will be chosen from the set of all characters (ASCII/Unicode values between 0 to 2,147,483,647 (math.MaxInt32)).
+
+Parameter:
+	count - the length of random string to create
+
+Returns:
+	string - the random string
+	error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...)
+*/
+func CryptoRandomNonAlphaNumeric(count int) (string, error) {
+	return CryptoRandomAlphaNumericCustom(count, false, false)
+}
+
+/*
+CryptoRandomAscii creates a random string whose length is the number of characters specified.
+Characters will be chosen from the set of characters whose ASCII value is between 32 and 126 (inclusive).
+
+Parameter:
+	count - the length of random string to create
+
+Returns:
+	string - the random string
+	error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...)
+*/
+func CryptoRandomAscii(count int) (string, error) {
+	return CryptoRandom(count, 32, 127, false, false)
+}
+
+/*
+CryptoRandomNumeric creates a random string whose length is the number of characters specified.
+Characters will be chosen from the set of numeric characters.
+
+Parameter:
+	count - the length of random string to create
+
+Returns:
+	string - the random string
+	error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...)
+*/
+func CryptoRandomNumeric(count int) (string, error) {
+	return CryptoRandom(count, 0, 0, false, true)
+}
+
+/*
+CryptoRandomAlphabetic creates a random string whose length is the number of characters specified.
+Characters will be chosen from the set of alpha-numeric characters as indicated by the arguments.
+
+Parameters:
+	count - the length of random string to create
+	letters - if true, generated string may include alphabetic characters
+	numbers - if true, generated string may include numeric characters
+
+Returns:
+	string - the random string
+	error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...)
+*/
+func CryptoRandomAlphabetic(count int) (string, error) {
+	return CryptoRandom(count, 0, 0, true, false)
+}
+
+/*
+CryptoRandomAlphaNumeric creates a random string whose length is the number of characters specified.
+Characters will be chosen from the set of alpha-numeric characters.
+
+Parameter:
+	count - the length of random string to create
+
+Returns:
+	string - the random string
+	error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...)
+*/
+func CryptoRandomAlphaNumeric(count int) (string, error) {
+	if count == 0 {
+		return "", nil
+	}
+	RandomString, err := CryptoRandom(count, 0, 0, true, true)
+	if err != nil {
+		return "", fmt.Errorf("Error: %s", err)
+	}
+	match, err := regexp.MatchString("([0-9]+)", RandomString)
+	if err != nil {
+		panic(err)
+	}
+
+	if !match {
+		//Get the position between 0 and the length of the string-1  to insert a random number
+		position := getCryptoRandomInt(count)
+		//Insert a random number between [0-9] in the position
+		RandomString = RandomString[:position] + string('0' + getCryptoRandomInt(10)) + RandomString[position + 1:]
+		return RandomString, err
+	}
+	return RandomString, err
+
+}
+
+/*
+CryptoRandomAlphaNumericCustom creates a random string whose length is the number of characters specified.
+Characters will be chosen from the set of alpha-numeric characters as indicated by the arguments.
+
+Parameters:
+	count - the length of random string to create
+	letters - if true, generated string may include alphabetic characters
+	numbers - if true, generated string may include numeric characters
+
+Returns:
+	string - the random string
+	error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...)
+*/
+func CryptoRandomAlphaNumericCustom(count int, letters bool, numbers bool) (string, error) {
+	return CryptoRandom(count, 0, 0, letters, numbers)
+}
+
+/*
+CryptoRandom creates a random string based on a variety of options, using using golang's crypto/rand source of randomness.
+If the parameters start and end are both 0, start and end are set to ' ' and 'z', the ASCII printable characters, will be used,
+unless letters and numbers are both false, in which case, start and end are set to 0 and math.MaxInt32, respectively.
+If chars is not nil, characters stored in chars that are between start and end are chosen.
+
+Parameters:
+	count - the length of random string to create
+	start - the position in set of chars (ASCII/Unicode int) to start at
+	end - the position in set of chars (ASCII/Unicode int) to end before
+	letters - if true, generated string may include alphabetic characters
+	numbers - if true, generated string may include numeric characters
+	chars - the set of chars to choose randoms from. If nil, then it will use the set of all chars.
+
+Returns:
+	string - the random string
+	error - an error stemming from invalid parameters: if count < 0; or the provided chars array is empty; or end <= start; or end > len(chars)
+*/
+func CryptoRandom(count int, start int, end int, letters bool, numbers bool, chars ...rune) (string, error) {
+	if count == 0 {
+		return "", nil
+	} else if count < 0 {
+		err := fmt.Errorf("randomstringutils illegal argument: Requested random string length %v is less than 0.", count) // equiv to err := errors.New("...")
+		return "", err
+	}
+	if chars != nil && len(chars) == 0 {
+		err := fmt.Errorf("randomstringutils illegal argument: The chars array must not be empty")
+		return "", err
+	}
+
+	if start == 0 && end == 0 {
+		if chars != nil {
+			end = len(chars)
+		} else {
+			if !letters && !numbers {
+				end = math.MaxInt32
+			} else {
+				end = 'z' + 1
+				start = ' '
+			}
+		}
+	} else {
+		if end <= start {
+			err := fmt.Errorf("randomstringutils illegal argument: Parameter end (%v) must be greater than start (%v)", end, start)
+			return "", err
+		}
+
+		if chars != nil && end > len(chars) {
+			err := fmt.Errorf("randomstringutils illegal argument: Parameter end (%v) cannot be greater than len(chars) (%v)", end, len(chars))
+			return "", err
+		}
+	}
+
+	buffer := make([]rune, count)
+	gap := end - start
+
+	// high-surrogates range, (\uD800-\uDBFF) = 55296 - 56319
+	//  low-surrogates range, (\uDC00-\uDFFF) = 56320 - 57343
+
+	for count != 0 {
+		count--
+		var ch rune
+		if chars == nil {
+			ch = rune(getCryptoRandomInt(gap) + int64(start))
+		} else {
+			ch = chars[getCryptoRandomInt(gap) + int64(start)]
+		}
+
+		if letters && unicode.IsLetter(ch) || numbers && unicode.IsDigit(ch) || !letters && !numbers {
+			if ch >= 56320 && ch <= 57343 { // low surrogate range
+				if count == 0 {
+					count++
+				} else {
+					// Insert low surrogate
+					buffer[count] = ch
+					count--
+					// Insert high surrogate
+					buffer[count] = rune(55296 + getCryptoRandomInt(128))
+				}
+			} else if ch >= 55296 && ch <= 56191 { // High surrogates range (Partial)
+				if count == 0 {
+					count++
+				} else {
+					// Insert low surrogate
+					buffer[count] = rune(56320 + getCryptoRandomInt(128))
+					count--
+					// Insert high surrogate
+					buffer[count] = ch
+				}
+			} else if ch >= 56192 && ch <= 56319 {
+				// private high surrogate, skip it
+				count++
+			} else {
+				// not one of the surrogates*
+				buffer[count] = ch
+			}
+		} else {
+			count++
+		}
+	}
+	return string(buffer), nil
+}
+
+func getCryptoRandomInt(count int) int64 {
+	nBig, err := rand.Int(rand.Reader, big.NewInt(int64(count)))
+	if err != nil {
+		panic(err)
+	}
+	return nBig.Int64()
+}