Tag 0.7.2

Add Box finalizers

Makefile

@@ -19,7 +19,7 @@ fmt:
 test:
 	GO111MODULE=off go get github.com/onsi/ginkgo/ginkgo
 	GO111MODULE=off go get github.com/onsi/gomega/...
-	ginkgo -race -r ./...
+	ginkgo -race -r -flakeAttempts 3 ./...
 
 .PHONY: test-integration
 test-integration:

cmd/create-repo.go

@@ -40,7 +40,9 @@ var createrepoCmd = &cobra.Command{
 		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
 		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
 		viper.BindPFlag("tree-compression", cmd.Flags().Lookup("tree-compression"))
-		viper.BindPFlag("tree-path", cmd.Flags().Lookup("tree-path"))
+		viper.BindPFlag("tree-filename", cmd.Flags().Lookup("tree-filename"))
+		viper.BindPFlag("meta-compression", cmd.Flags().Lookup("meta-compression"))
+		viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename"))
 		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
 		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
 	},
@@ -57,9 +59,14 @@ var createrepoCmd = &cobra.Command{
 		t := viper.GetString("type")
 		reset := viper.GetBool("reset-revision")
 		treetype := viper.GetString("tree-compression")
-		treepath := viper.GetString("tree-path")
+		treeName := viper.GetString("tree-filename")
+		metatype := viper.GetString("meta-compression")
+		metaName := viper.GetString("meta-filename")
 		source_repo := viper.GetString("repo")
 
+		treeFile := installer.NewDefaultTreeRepositoryFile()
+		metaFile := installer.NewDefaultMetaRepositoryFile()
+
 		if source_repo != "" {
 			// Search for system repository
 			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
@@ -93,13 +100,24 @@ var createrepoCmd = &cobra.Command{
 		}
 
 		if treetype != "" {
-			repo.SetTreeCompressionType(compiler.CompressionImplementation(treetype))
+			treeFile.SetCompressionType(compiler.CompressionImplementation(treetype))
 		}
 
-		if treepath != "" {
-			repo.SetTreePath(treepath)
+		if treeName != "" {
+			treeFile.SetFileName(treeName)
 		}
 
+		if metatype != "" {
+			metaFile.SetCompressionType(compiler.CompressionImplementation(metatype))
+		}
+
+		if metaName != "" {
+			metaFile.SetFileName(metaName)
+		}
+
+		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
+		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
+
 		err = repo.Write(dst, reset)
 		if err != nil {
 			Fatal("Error: " + err.Error())
@@ -122,8 +140,10 @@ func init() {
 	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
 	createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
 
-	createrepoCmd.Flags().String("tree-compression", "none", "Compression alg: none, gzip")
-	createrepoCmd.Flags().String("tree-path", installer.TREE_TARBALL, "Repository tree filename")
+	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
+	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

cmd/exec.go

@@ -0,0 +1,86 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+
+	b64 "encoding/base64"
+
+	"github.com/mudler/luet/pkg/box"
+	. "github.com/mudler/luet/pkg/logger"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var execCmd = &cobra.Command{
+	Use:   "exec --rootfs /path [command]",
+	Short: "Execute a command in the rootfs context",
+	Long:  `Uses unshare technique and pivot root to execute a command inside a folder containing a valid rootfs`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("stdin", cmd.Flags().Lookup("stdin"))
+		viper.BindPFlag("stdout", cmd.Flags().Lookup("stdout"))
+		viper.BindPFlag("stderr", cmd.Flags().Lookup("stderr"))
+		viper.BindPFlag("rootfs", cmd.Flags().Lookup("rootfs"))
+		viper.BindPFlag("decode", cmd.Flags().Lookup("decode"))
+		viper.BindPFlag("entrypoint", cmd.Flags().Lookup("entrypoint"))
+
+	},
+	// If you change this, look at pkg/box/exec that runs this command and adapt
+	Run: func(cmd *cobra.Command, args []string) {
+
+		stdin := viper.GetBool("stdin")
+		stdout := viper.GetBool("stdout")
+		stderr := viper.GetBool("stderr")
+		rootfs := viper.GetString("rootfs")
+		base := viper.GetBool("decode")
+
+		entrypoint := viper.GetString("entrypoint")
+		if base {
+			var ss []string
+			for _, a := range args {
+				sDec, _ := b64.StdEncoding.DecodeString(a)
+				ss = append(ss, string(sDec))
+			}
+			//If the command to run is complex,using base64 to avoid bad input
+
+			args = ss
+		}
+		Info("Executing", args, "in", rootfs)
+
+		b := box.NewBox(entrypoint, args, rootfs, stdin, stdout, stderr)
+		err := b.Exec()
+		if err != nil {
+			Fatal(err)
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	execCmd.Hidden = true
+	execCmd.Flags().String("rootfs", path, "Rootfs path")
+	execCmd.Flags().Bool("stdin", false, "Attach to stdin")
+	execCmd.Flags().Bool("stdout", false, "Attach to stdout")
+	execCmd.Flags().Bool("stderr", false, "Attach to stderr")
+	execCmd.Flags().Bool("decode", false, "Base64 decode")
+
+	execCmd.Flags().String("entrypoint", "/bin/sh", "Entrypoint command (/bin/sh)")
+
+	RootCmd.AddCommand(execCmd)
+}

cmd/repo/update.go

@@ -62,7 +62,7 @@ $> luet repo update repo1 repo2
 
 			} else {
 				for _, repo := range LuetCfg.SystemRepositories {
-					if repo.Cached {
+					if repo.Cached && repo.Enable {
 						r := installer.NewSystemRepository(repo)
 						Spinner(32)
 						_, err := r.Sync(force)

cmd/root.go

@@ -35,9 +35,10 @@ import (
 
 var cfgFile string
 var Verbose bool
+var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.7.1"
+	LuetCLIVersion = "0.7.2"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -91,16 +92,21 @@ func LoadConfig(c *config.LuetConfig) error {
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
-	// XXX: This is mostly from scratch images.
+
 	if os.Getenv("LUET_NOLOCK") != "true" {
-		s := single.New("luet")
-		if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
-			Fatal("another instance of the app is already running, exiting")
-		} else if err != nil {
-			// Another error occurred, might be worth handling it as well
-			Fatal("failed to acquire exclusive app lock:", err.Error())
+		for _, lockedCmd := range LockedCommands {
+			if os.Args[1] == lockedCmd {
+				s := single.New("luet")
+				if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
+					Fatal("another instance of the app is already running, exiting")
+				} else if err != nil {
+					// Another error occurred, might be worth handling it as well
+					Fatal("failed to acquire exclusive app lock:", err.Error())
+				}
+				defer s.TryUnlock()
+				break
+			}
 		}
-		defer s.TryUnlock()
 	}
 
 	if err := RootCmd.Execute(); err != nil {

contrib/config/luet.yaml

@@ -96,8 +96,11 @@
 #     packages. By default caching is disable.
 #     cached: false
 #
-#     Path where store tree of the specifications. Default path is $database_path/repos/$repo_name
-#     tree_path: "/var/cache/luet/repos/local"
+#     Path where store tree of the specifications. Default path is $database_path/repos/$repo_name/treefs
+#     tree_path: "/var/cache/luet/repos/local/treefs"
+#
+#     Path where store repository metadata. Default path is $database_path/repos/$repo_name/meta
+#     meta_path: "/var/cache/luet/repos/local/meta"
 #
 #     Define the list of the URL where retrieve tree and packages.
 #     urls:
@@ -125,4 +128,4 @@
 #
 #   Number of overall attempts that the solver has available before bailing out.
 #   max_attempts: 9000
-#
+#

pkg/box/exec.go

@@ -0,0 +1,157 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package box
+
+import (
+	b64 "encoding/base64"
+	"fmt"
+	"os"
+	"os/exec"
+	"syscall"
+
+	"github.com/pkg/errors"
+
+	helpers "github.com/mudler/luet/pkg/helpers"
+)
+
+type Box interface {
+	Run() error
+	Exec() error
+}
+
+type DefaultBox struct {
+	Name string
+	Root string
+	Env  []string
+	Cmd  string
+	Args []string
+
+	Stdin, Stdout, Stderr bool
+}
+
+func NewBox(cmd string, args []string, rootfs string, stdin, stdout, stderr bool) Box {
+	return &DefaultBox{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+		Cmd:    cmd,
+		Args:   args,
+		Root:   rootfs,
+	}
+}
+
+func (b *DefaultBox) Exec() error {
+
+	if err := mountProc(b.Root); err != nil {
+		return errors.Wrap(err, "Failed mounting proc on rootfs")
+	}
+	if err := mountDev(b.Root); err != nil {
+		return errors.Wrap(err, "Failed mounting dev on rootfs")
+	}
+	if err := PivotRoot(b.Root); err != nil {
+		return errors.Wrap(err, "Failed switching pivot on rootfs")
+	}
+
+	cmd := exec.Command(b.Cmd, b.Args...)
+
+	if b.Stdin {
+		cmd.Stdin = os.Stdin
+	}
+
+	if b.Stderr {
+		cmd.Stderr = os.Stderr
+	}
+
+	if b.Stdout {
+		cmd.Stdout = os.Stdout
+	}
+
+	cmd.Env = b.Env
+
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Error running the %s command", b.Cmd))
+	}
+	return nil
+}
+
+func (b *DefaultBox) Run() error {
+
+	if !helpers.Exists(b.Root) {
+		return errors.New(b.Root + " does not exist")
+	}
+
+	// This matches with exec CLI command in luet
+	// TODO: Pass by env var as well
+	execCmd := []string{"exec", "--rootfs", b.Root, "--entrypoint", b.Cmd}
+
+	if b.Stdin {
+		execCmd = append(execCmd, "--stdin")
+	}
+
+	if b.Stderr {
+		execCmd = append(execCmd, "--stderr")
+	}
+
+	if b.Stdout {
+		execCmd = append(execCmd, "--stdout")
+	}
+	// Encode the command in base64 to avoid bad input from the args given
+	execCmd = append(execCmd, "--decode")
+
+	for _, a := range b.Args {
+		execCmd = append(execCmd, b64.StdEncoding.EncodeToString([]byte(a)))
+	}
+
+	cmd := exec.Command("/proc/self/exe", execCmd...)
+	if b.Stdin {
+		cmd.Stdin = os.Stdin
+	}
+
+	if b.Stderr {
+		cmd.Stderr = os.Stderr
+	}
+
+	if b.Stdout {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Cloneflags: syscall.CLONE_NEWNS |
+			syscall.CLONE_NEWUTS |
+			syscall.CLONE_NEWIPC |
+			syscall.CLONE_NEWPID |
+			syscall.CLONE_NEWNET |
+			syscall.CLONE_NEWUSER,
+		UidMappings: []syscall.SysProcIDMap{
+			{
+				ContainerID: 0,
+				HostID:      os.Getuid(),
+				Size:        1,
+			},
+		},
+		GidMappings: []syscall.SysProcIDMap{
+			{
+				ContainerID: 0,
+				HostID:      os.Getgid(),
+				Size:        1,
+			},
+		},
+	}
+
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "Failed running Box command")
+	}
+	return nil
+}

pkg/box/rootfs.go

@@ -0,0 +1,91 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package box
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+func PivotRoot(newroot string) error {
+	putold := filepath.Join(newroot, "/.pivot_root")
+
+	// bind mount newroot to itself - this is a slight hack needed to satisfy the
+	// pivot_root requirement that newroot and putold must not be on the same
+	// filesystem as the current root
+	if err := syscall.Mount(newroot, newroot, "", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
+		return err
+	}
+
+	// create putold directory
+	if err := os.MkdirAll(putold, 0700); err != nil {
+		return err
+	}
+
+	// call pivot_root
+	if err := syscall.PivotRoot(newroot, putold); err != nil {
+		return err
+	}
+
+	// ensure current working directory is set to new root
+	if err := os.Chdir("/"); err != nil {
+		return err
+	}
+
+	// umount putold, which now lives at /.pivot_root
+	putold = "/.pivot_root"
+	if err := syscall.Unmount(putold, syscall.MNT_DETACH); err != nil {
+		return err
+	}
+
+	// remove putold
+	if err := os.RemoveAll(putold); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountProc(newroot string) error {
+	source := "proc"
+	target := filepath.Join(newroot, "/proc")
+	fstype := "proc"
+	flags := 0
+	data := ""
+
+	os.MkdirAll(target, 0755)
+	if err := syscall.Mount(source, target, fstype, uintptr(flags), data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountDev(newroot string) error {
+
+	source := "/dev"
+	target := filepath.Join(newroot, "/dev")
+	fstype := "bind"
+	data := ""
+
+	os.MkdirAll(target, 0755)
+	if err := syscall.Mount(source, target, fstype, syscall.MS_BIND|syscall.MS_REC, data); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/config/config.go

@@ -131,6 +131,7 @@ type LuetRepository struct {
 	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
 	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
 	TreePath       string            `json:"tree_path,omitempty" yaml:"tree_path,omitempty" mapstructure:"tree_path"`
+	MetaPath       string            `json:"meta_path,omitempty" yaml:"meta_path,omitempty" mapstructure:"meta_path"`
 
 	// Serialized options not used in repository configuration
 
@@ -153,6 +154,7 @@ func NewLuetRepository(name, t, descr string, urls []string, priority int, enabl
 		Cached:         cached,
 		Authentication: make(map[string]string, 0),
 		TreePath:       "",
+		MetaPath:       "",
 	}
 }
 
@@ -164,6 +166,7 @@ func NewEmptyLuetRepository() *LuetRepository {
 		Type:           "",
 		Priority:       9999,
 		TreePath:       "",
+		MetaPath:       "",
 		Enable:         false,
 		Cached:         false,
 		Authentication: make(map[string]string, 0),

pkg/helpers/file.go

@@ -23,6 +23,22 @@ import (
 	copy "github.com/otiai10/copy"
 )
 
+func ListDir(dir string) ([]string, error) {
+	content := []string{}
+
+	err := filepath.Walk(dir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			content = append(content, path)
+
+			return nil
+		})
+
+	return content, err
+}
+
 // Exists reports whether the named file or directory exists.
 func Exists(name string) bool {
 	if _, err := os.Stat(name); err != nil {

pkg/installer/finalizer.go

@@ -0,0 +1,76 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"os/exec"
+
+	"github.com/ghodss/yaml"
+	box "github.com/mudler/luet/pkg/box"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/pkg/errors"
+)
+
+type LuetFinalizer struct {
+	Install   []string `json:"install"`
+	Uninstall []string `json:"uninstall"` // TODO: Where to store?
+}
+
+func (f *LuetFinalizer) RunInstall(s *System) error {
+	for _, c := range f.Install {
+		if s.Target == "/" {
+
+			Info("finalizer on / :", "sh", "-c", c)
+			cmd := exec.Command("sh", "-c", c)
+			stdoutStderr, err := cmd.CombinedOutput()
+			if err != nil {
+				return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
+			}
+			Info(string(stdoutStderr))
+		} else {
+			b := box.NewBox("sh", []string{"-c", c}, s.Target, false, true, true)
+			err := b.Run()
+			if err != nil {
+				return errors.Wrap(err, "Failed running command ")
+			}
+		}
+	}
+	return nil
+}
+
+// TODO: We don't store uninstall finalizers ?!
+func (f *LuetFinalizer) RunUnInstall() error {
+	for _, c := range f.Uninstall {
+		Debug("finalizer:", "sh", "-c", c)
+		cmd := exec.Command("sh", "-c", c)
+		stdoutStderr, err := cmd.CombinedOutput()
+		if err != nil {
+			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
+		}
+		Info(string(stdoutStderr))
+	}
+	return nil
+}
+
+func NewLuetFinalizerFromYaml(data []byte) (*LuetFinalizer, error) {
+	var p LuetFinalizer
+	err := yaml.Unmarshal(data, &p)
+	if err != nil {
+		return &p, err
+	}
+	return &p, err
+}

pkg/installer/installer.go

@@ -18,13 +18,11 @@ package installer
 import (
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
 
-	"github.com/ghodss/yaml"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
@@ -57,47 +55,6 @@ type ArtifactMatch struct {
 	Repository Repository
 }
 
-type LuetFinalizer struct {
-	Install   []string `json:"install"`
-	Uninstall []string `json:"uninstall"` // TODO: Where to store?
-}
-
-func (f *LuetFinalizer) RunInstall() error {
-	for _, c := range f.Install {
-		Debug("finalizer:", "sh", "-c", c)
-		cmd := exec.Command("sh", "-c", c)
-		stdoutStderr, err := cmd.CombinedOutput()
-		if err != nil {
-			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
-		}
-		Info(string(stdoutStderr))
-	}
-	return nil
-}
-
-// TODO: We don't store uninstall finalizers ?!
-func (f *LuetFinalizer) RunUnInstall() error {
-	for _, c := range f.Install {
-		Debug("finalizer:", "sh", "-c", c)
-		cmd := exec.Command("sh", "-c", c)
-		stdoutStderr, err := cmd.CombinedOutput()
-		if err != nil {
-			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
-		}
-		Info(string(stdoutStderr))
-	}
-	return nil
-}
-
-func NewLuetFinalizerFromYaml(data []byte) (*LuetFinalizer, error) {
-	var p LuetFinalizer
-	err := yaml.Unmarshal(data, &p)
-	if err != nil {
-		return &p, err
-	}
-	return &p, err
-}
-
 func NewLuetInstaller(opts LuetInstallerOptions) Installer {
 	return &LuetInstaller{Options: opts}
 }
@@ -119,7 +76,8 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 
 	toInstall := []pkg.Package{}
 	for _, assertion := range solution {
-		if assertion.Value {
+		// Be sure to filter from solutions packages already installed in the system
+		if _, err := s.Database.FindPackage(assertion.Package); err != nil && assertion.Value {
 			toInstall = append(toInstall, assertion.Package)
 		}
 	}
@@ -158,13 +116,12 @@ func (l *LuetInstaller) Swap(toRemove []pkg.Package, toInstall []pkg.Package, s
 }
 
 func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove []pkg.Package, toInstall []pkg.Package, s *System) error {
-
 	// First match packages against repositories by priority
 	allRepos := pkg.NewInMemoryDatabase(false)
 	syncedRepos.SyncDatabase(allRepos)
 	toInstall = syncedRepos.ResolveSelectors(toInstall)
 
-	if err := l.install(syncedRepos, toInstall, s, true); err != nil {
+	if err := l.download(syncedRepos, toInstall); err != nil {
 		return errors.Wrap(err, "Pre-downloading packages")
 	}
 
@@ -193,7 +150,7 @@ func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove []pkg.Package, t
 	}
 	l.Options.Force = forced
 
-	return l.install(syncedRepos, toInstall, s, false)
+	return l.install(syncedRepos, toInstall, s)
 }
 
 func (l *LuetInstaller) Install(cp []pkg.Package, s *System, downloadOnly bool) error {
@@ -201,10 +158,55 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System, downloadOnly bool)
 	if err != nil {
 		return err
 	}
-	return l.install(syncedRepos, cp, s, downloadOnly)
+	return l.install(syncedRepos, cp, s)
 }
 
-func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *System, downloadOnly bool) error {
+func (l *LuetInstaller) download(syncedRepos Repositories, cp []pkg.Package) error {
+	toDownload := map[string]ArtifactMatch{}
+
+	// FIXME: This can be optimized. We don't need to re-match this to the repository
+	// But we could just do it once
+
+	// Gathers things to download
+	for _, currentPack := range cp {
+		matches := syncedRepos.PackageMatches([]pkg.Package{currentPack})
+		if len(matches) == 0 {
+			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
+		}
+	A:
+		for _, artefact := range matches[0].Repo.GetIndex() {
+			if artefact.GetCompileSpec().GetPackage() == nil {
+				return errors.New("Package in compilespec empty")
+
+			}
+			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+
+				toDownload[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}
+
+				break A
+			}
+		}
+	}
+	// Download packages into cache in parallel.
+	all := make(chan ArtifactMatch)
+
+	var wg = new(sync.WaitGroup)
+
+	// Download
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.downloadWorker(i, wg, all)
+	}
+	for _, c := range toDownload {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
+
+	return nil
+}
+
+func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *System) error {
 	var p []pkg.Package
 
 	// Check if the package is installed first
@@ -283,50 +285,33 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *S
 
 	var wg = new(sync.WaitGroup)
 
-	if !downloadOnly {
-		// Download first
-		for i := 0; i < l.Options.Concurrency; i++ {
-			wg.Add(1)
-			go l.installerWorker(i, wg, all, s, true)
-		}
-
-		for _, c := range toInstall {
-			all <- c
-		}
-		close(all)
-		wg.Wait()
-
-		all = make(chan ArtifactMatch)
-
-		wg = new(sync.WaitGroup)
-
-		// Do the real install
-		for i := 0; i < l.Options.Concurrency; i++ {
-			wg.Add(1)
-			go l.installerWorker(i, wg, all, s, false)
-		}
-
-		for _, c := range toInstall {
-			all <- c
-		}
-		close(all)
-		wg.Wait()
-	} else {
-		for i := 0; i < l.Options.Concurrency; i++ {
-			wg.Add(1)
-			go l.installerWorker(i, wg, all, s, downloadOnly)
-		}
-
-		for _, c := range toInstall {
-			all <- c
-		}
-		close(all)
-		wg.Wait()
+	// Download first
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.downloadWorker(i, wg, all)
 	}
 
-	if downloadOnly {
-		return nil
+	for _, c := range toInstall {
+		all <- c
 	}
+	close(all)
+	wg.Wait()
+
+	all = make(chan ArtifactMatch)
+
+	wg = new(sync.WaitGroup)
+
+	// Do the real install
+	for i := 0; i < l.Options.Concurrency; i++ {
+		wg.Add(1)
+		go l.installerWorker(i, wg, all, s)
+	}
+
+	for _, c := range toInstall {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
 
 	for _, c := range toInstall {
 		// Annotate to the system that the package was installed
@@ -367,13 +352,14 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *S
 							if err != nil && !l.Options.Force {
 								return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
 							}
-							err = finalizer.RunInstall()
+							err = finalizer.RunInstall(s)
 							if err != nil && !l.Options.Force {
 								return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
 							}
 							executedFinalizer[ass.Package.GetFingerPrint()] = true
 						}
 					}
+
 				}
 			}
 
@@ -395,7 +381,7 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *S
 					if err != nil && !l.Options.Force {
 						return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
 					}
-					err = finalizer.RunInstall()
+					err = finalizer.RunInstall(s)
 					if err != nil && !l.Options.Force {
 						return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
 					}
@@ -404,23 +390,30 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *S
 			}
 		}
 	}
+
 	return nil
 
 }
 
-func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System, downloadOnly bool) error {
+func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) {
 
 	artifact, err := a.Repository.Client().DownloadArtifact(a.Artifact)
 	if err != nil {
-		return errors.Wrap(err, "Error on download artifact")
+		return nil, errors.Wrap(err, "Error on download artifact")
 	}
 
 	err = artifact.Verify()
 	if err != nil && !l.Options.Force {
-		return errors.Wrap(err, "Artifact integrity check failure")
+		return nil, errors.Wrap(err, "Artifact integrity check failure")
 	}
-	if downloadOnly {
-		return nil
+	return artifact, nil
+}
+
+func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
+
+	artifact, err := l.downloadPackage(a)
+	if err != nil && !l.Options.Force {
+		return errors.Wrap(err, "Failed downloading package")
 	}
 
 	files, err := artifact.FileList()
@@ -438,20 +431,39 @@ func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System, downloadOnly
 	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.Package.GetFingerPrint(), Files: files})
 }
 
-func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System, downloadOnly bool) error {
+func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch) error {
 	defer wg.Done()
 
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
-		err := l.installPackage(p, s, downloadOnly)
+		_, err := l.downloadPackage(p)
 		if err != nil && !l.Options.Force {
 			//TODO: Uninstall, rollback.
 			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
 			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
 		}
-		if err == nil && downloadOnly {
+		if err == nil {
 			Info(":package: ", p.Package.HumanReadableString(), "downloaded")
-		} else if err == nil {
+		} else if err != nil && l.Options.Force {
+			Info(":package: ", p.Package.HumanReadableString(), "downloaded with failures (force download)")
+		}
+	}
+
+	return nil
+}
+
+func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System) error {
+	defer wg.Done()
+
+	for p := range c {
+		// TODO: Keep trace of what was added from the tar, and save it into system
+		err := l.installPackage(p, s)
+		if err != nil && !l.Options.Force {
+			//TODO: Uninstall, rollback.
+			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
+			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
+		}
+		if err == nil {
 			Info(":package: ", p.Package.HumanReadableString(), "installed")
 		} else if err != nil && l.Options.Force {
 			Info(":package: ", p.Package.HumanReadableString(), "installed with failures (force install)")

pkg/installer/installer_test.go

@@ -47,9 +47,9 @@ var _ = Describe("Installer", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
 
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
@@ -62,9 +62,9 @@ var _ = Describe("Installer", func() {
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
+			c.SetConcurrency(2)
 
-			artifact, err := compiler.Compile(false, spec)
+			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -86,12 +86,133 @@ var _ = Describe("Installer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
+			err = repo.Write(tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+
+			fakeroot, err := ioutil.TempDir("", "fakeroot")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(fakeroot) // clean up
+
+			inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
+			repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
+name: "test"
+type: "disk"
+urls:
+  - "`+tmpdir+`"
+`), pkg.NewInMemoryDatabase(false))
+			Expect(err).ToNot(HaveOccurred())
+
+			inst.Repositories(Repositories{repo2})
+			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
+			Expect(repo.GetType()).To(Equal("disk"))
+			systemDB := pkg.NewInMemoryDatabase(false)
+			system := &System{Database: systemDB, Target: fakeroot}
+			err = inst.Install([]pkg.Package{&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}}, system, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).To(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).To(BeTrue())
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			files, err := systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(files).To(Equal([]string{"artifact42", "test5", "test6"}))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(system.Database.GetPackages())).To(Equal(1))
+			p, err := system.Database.GetPackage(system.Database.GetPackages()[0])
+			Expect(err).ToNot(HaveOccurred())
+			Expect(p.GetName()).To(Equal("b"))
+
+			err = inst.Uninstall(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}, system)
+			Expect(err).ToNot(HaveOccurred())
+
+			// Nothing should be there anymore (files, packagedb entry)
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test5"))).ToNot(BeTrue())
+			Expect(helpers.Exists(filepath.Join(fakeroot, "test6"))).ToNot(BeTrue())
+
+			_, err = systemDB.FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+			_, err = systemDB.GetPackageFiles(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).To(HaveOccurred())
+		})
+
+	})
+
+	Context("Writes a repository definition without compression", func() {
+		It("Writes a repo and can install packages from it", func() {
+			//repo:=NewLuetSystemRepository()
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err = generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
+
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(),
+				generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err = ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+
+			spec.SetOutputPath(tmpdir)
+			c.SetConcurrency(2)
+
+			artifact, err := c.Compile(false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			content1, err := helpers.Read(spec.Rel("test5"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := helpers.Read(spec.Rel("test6"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("artifact5\n"))
+			Expect(content2).To(Equal("artifact6\n"))
+
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("b-test-1.0.metadata.yaml"))).To(BeTrue())
+
+			repo, err := GenerateRepository("test", "description", "disk", []string{tmpdir}, 1, tmpdir, "../../tests/fixtures/buildable", pkg.NewInMemoryDatabase(false))
+			treeFile := NewDefaultTreeRepositoryFile()
+			treeFile.SetCompressionType(compiler.None)
+			repo.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(repo.GetName()).To(Equal("test"))
+			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -160,9 +281,9 @@ urls:
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
 
-			compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
+			c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions())
 
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
@@ -175,9 +296,9 @@ urls:
 			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
 
 			spec.SetOutputPath(tmpdir)
-			compiler.SetConcurrency(2)
+			c.SetConcurrency(2)
 
-			artifact, err := compiler.Compile(false, spec)
+			artifact, err := c.Compile(false, spec)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
 			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@@ -199,12 +320,14 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -307,12 +430,14 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 
@@ -420,14 +545,16 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar.gz"))).To(BeTrue())
 			Expect(helpers.Exists(spec.Rel("b-test-1.1.package.tar"))).ToNot(BeTrue())
 
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 			Expect(repo.GetUrls()[0]).To(Equal(tmpdir))
 			Expect(repo.GetType()).To(Equal("disk"))
 

pkg/installer/interface.go

@@ -46,12 +46,15 @@ type Repository interface {
 	AddUrl(string)
 	GetPriority() int
 	GetIndex() compiler.ArtifactIndex
+	SetIndex(i compiler.ArtifactIndex)
 	GetTree() tree.Builder
 	SetTree(tree.Builder)
 	Write(path string, resetRevision bool) error
 	Sync(bool) (Repository, error)
 	GetTreePath() string
 	SetTreePath(string)
+	GetMetaPath() string
+	SetMetaPath(string)
 	GetType() string
 	SetType(string)
 	SetAuthentication(map[string]string)
@@ -62,8 +65,8 @@ type Repository interface {
 	SetLastUpdate(string)
 	Client() Client
 
-	GetTreeChecksums() compiler.Checksums
-	GetTreeCompressionType() compiler.CompressionImplementation
-	SetTreeCompressionType(c compiler.CompressionImplementation)
-	SetTreeChecksums(c compiler.Checksums)
+	GetRepositoryFile(string) (LuetRepositoryFile, error)
+	SetRepositoryFile(string, LuetRepositoryFile)
+
+	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
 }

pkg/installer/repository.go

@@ -40,32 +40,43 @@ import (
 )
 
 const (
+	REPOSITORY_METAFILE = "repository.meta.yaml"
 	REPOSITORY_SPECFILE = "repository.yaml"
 	TREE_TARBALL        = "tree.tar"
+
+	REPOFILE_TREE_KEY = "tree"
+	REPOFILE_META_KEY = "meta"
 )
 
+type LuetRepositoryFile struct {
+	FileName        string                             `json:"filename"`
+	CompressionType compiler.CompressionImplementation `json:"compressiontype,omitempty"`
+	Checksums       compiler.Checksums                 `json:"checksums,omitempty"`
+}
+
 type LuetSystemRepository struct {
 	*config.LuetRepository
 
-	Index               compiler.ArtifactIndex             `json:"index"`
-	Tree                tree.Builder                       `json:"-"`
-	TreePath            string                             `json:"treepath"`
-	TreeCompressionType compiler.CompressionImplementation `json:"treecompressiontype"`
-	TreeChecksums       compiler.Checksums                 `json:"treechecksums"`
+	Index           compiler.ArtifactIndex        `json:"index"`
+	Tree            tree.Builder                  `json:"-"`
+	RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"`
 }
 
 type LuetSystemRepositorySerialized struct {
-	Name                string                             `json:"name"`
-	Description         string                             `json:"description,omitempty"`
-	Urls                []string                           `json:"urls"`
-	Priority            int                                `json:"priority"`
-	Index               []*compiler.PackageArtifact        `json:"index"`
-	Type                string                             `json:"type"`
-	Revision            int                                `json:"revision,omitempty"`
-	LastUpdate          string                             `json:"last_update,omitempty"`
-	TreePath            string                             `json:"treepath"`
-	TreeCompressionType compiler.CompressionImplementation `json:"treecompressiontype"`
-	TreeChecksums       compiler.Checksums                 `json:"treechecksums"`
+	Name            string                        `json:"name"`
+	Description     string                        `json:"description,omitempty"`
+	Urls            []string                      `json:"urls"`
+	Priority        int                           `json:"priority"`
+	Type            string                        `json:"type"`
+	Revision        int                           `json:"revision,omitempty"`
+	LastUpdate      string                        `json:"last_update,omitempty"`
+	TreePath        string                        `json:"treepath"`
+	MetaPath        string                        `json:"metapath"`
+	RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"`
+}
+
+type LuetSystemRepositoryMetadata struct {
+	Index []*compiler.PackageArtifact `json:"index,omitempty"`
 }
 
 type LuetSearchModeType string
@@ -81,6 +92,90 @@ type LuetSearchOpts struct {
 	Mode    LuetSearchModeType
 }
 
+func NewLuetSystemRepositoryMetadata(file string, removeFile bool) (*LuetSystemRepositoryMetadata, error) {
+	ans := &LuetSystemRepositoryMetadata{}
+	err := ans.ReadFile(file, removeFile)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}
+
+func (m *LuetSystemRepositoryMetadata) WriteFile(path string) error {
+	data, err := yaml.Marshal(m)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(path, data, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *LuetSystemRepositoryMetadata) ReadFile(file string, removeFile bool) error {
+	if file == "" {
+		return errors.New("Invalid path for repository metadata")
+	}
+
+	dat, err := ioutil.ReadFile(file)
+	if err != nil {
+		return err
+	}
+	if removeFile {
+		defer os.Remove(file)
+	}
+
+	err = yaml.Unmarshal(dat, m)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *LuetSystemRepositoryMetadata) ToArtificatIndex() (ans compiler.ArtifactIndex) {
+	for _, a := range m.Index {
+		ans = append(ans, a)
+	}
+	return
+}
+
+func NewDefaultTreeRepositoryFile() LuetRepositoryFile {
+	return LuetRepositoryFile{
+		FileName:        TREE_TARBALL,
+		CompressionType: compiler.GZip,
+	}
+}
+
+func NewDefaultMetaRepositoryFile() LuetRepositoryFile {
+	return LuetRepositoryFile{
+		FileName:        REPOSITORY_METAFILE + ".tar",
+		CompressionType: compiler.None,
+	}
+}
+
+func (f *LuetRepositoryFile) SetFileName(n string) {
+	f.FileName = n
+}
+func (f *LuetRepositoryFile) GetFileName() string {
+	return f.FileName
+}
+func (f *LuetRepositoryFile) SetCompressionType(c compiler.CompressionImplementation) {
+	f.CompressionType = c
+}
+func (f *LuetRepositoryFile) GetCompressionType() compiler.CompressionImplementation {
+	return f.CompressionType
+}
+func (f *LuetRepositoryFile) SetChecksums(c compiler.Checksums) {
+	f.Checksums = c
+}
+func (f *LuetRepositoryFile) GetChecksums() compiler.Checksums {
+	return f.Checksums
+}
+
 func GenerateRepository(name, descr, t string, urls []string, priority int, src, treeDir string, db pkg.PackageDatabase) (Repository, error) {
 
 	art, err := buildPackageIndex(src)
@@ -100,15 +195,17 @@ func GenerateRepository(name, descr, t string, urls []string, priority int, src,
 
 func NewSystemRepository(repo config.LuetRepository) Repository {
 	return &LuetSystemRepository{
-		LuetRepository: &repo,
+		LuetRepository:  &repo,
+		RepositoryFiles: map[string]LuetRepositoryFile{},
 	}
 }
 
 func NewLuetSystemRepository(repo *config.LuetRepository, art []compiler.Artifact, builder tree.Builder) Repository {
 	return &LuetSystemRepository{
-		LuetRepository: repo,
-		Index:          art,
-		Tree:           builder,
+		LuetRepository:  repo,
+		Index:           art,
+		Tree:            builder,
+		RepositoryFiles: map[string]LuetRepositoryFile{},
 	}
 }
 
@@ -118,6 +215,7 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 	if err != nil {
 		return nil, err
 	}
+
 	r := &LuetSystemRepository{
 		LuetRepository: config.NewLuetRepository(
 			p.Name,
@@ -128,9 +226,7 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 			true,
 			false,
 		),
-		TreeCompressionType: p.TreeCompressionType,
-		TreeChecksums:       p.TreeChecksums,
-		TreePath:            p.TreePath,
+		RepositoryFiles: p.RepositoryFiles,
 	}
 	if p.Revision > 0 {
 		r.Revision = p.Revision
@@ -138,11 +234,6 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 	if p.LastUpdate != "" {
 		r.LastUpdate = p.LastUpdate
 	}
-	i := compiler.ArtifactIndex{}
-	for _, ii := range p.Index {
-		i = append(i, ii)
-	}
-	r.Index = i
 	r.Tree = tree.NewInstallerRecipe(db)
 
 	return r, err
@@ -190,22 +281,6 @@ func (r *LuetSystemRepository) GetAuthentication() map[string]string {
 	return r.LuetRepository.Authentication
 }
 
-func (r *LuetSystemRepository) GetTreeCompressionType() compiler.CompressionImplementation {
-	return r.TreeCompressionType
-}
-
-func (r *LuetSystemRepository) GetTreeChecksums() compiler.Checksums {
-	return r.TreeChecksums
-}
-
-func (r *LuetSystemRepository) SetTreeCompressionType(c compiler.CompressionImplementation) {
-	r.TreeCompressionType = c
-}
-
-func (r *LuetSystemRepository) SetTreeChecksums(c compiler.Checksums) {
-	r.TreeChecksums = c
-}
-
 func (r *LuetSystemRepository) GetType() string {
 	return r.LuetRepository.Type
 }
@@ -230,12 +305,21 @@ func (r *LuetSystemRepository) GetTreePath() string {
 func (r *LuetSystemRepository) SetTreePath(p string) {
 	r.TreePath = p
 }
+func (r *LuetSystemRepository) GetMetaPath() string {
+	return r.MetaPath
+}
+func (r *LuetSystemRepository) SetMetaPath(p string) {
+	r.MetaPath = p
+}
 func (r *LuetSystemRepository) SetTree(b tree.Builder) {
 	r.Tree = b
 }
 func (r *LuetSystemRepository) GetIndex() compiler.ArtifactIndex {
 	return r.Index
 }
+func (r *LuetSystemRepository) SetIndex(i compiler.ArtifactIndex) {
+	r.Index = i
+}
 func (r *LuetSystemRepository) GetTree() tree.Builder {
 	return r.Tree
 }
@@ -251,10 +335,19 @@ func (r *LuetSystemRepository) SetLastUpdate(u string) {
 func (r *LuetSystemRepository) IncrementRevision() {
 	r.LuetRepository.Revision++
 }
-
 func (r *LuetSystemRepository) SetAuthentication(auth map[string]string) {
 	r.LuetRepository.Authentication = auth
 }
+func (r *LuetSystemRepository) GetRepositoryFile(name string) (LuetRepositoryFile, error) {
+	ans, ok := r.RepositoryFiles[name]
+	if ok {
+		return ans, nil
+	}
+	return ans, errors.New("Repository file " + name + " not found!")
+}
+func (r *LuetSystemRepository) SetRepositoryFile(name string, f LuetRepositoryFile) {
+	r.RepositoryFiles[name] = f
+}
 
 func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repository, error) {
 	dat, err := ioutil.ReadFile(file)
@@ -271,6 +364,16 @@ func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repos
 		return nil, errors.Wrap(err, "Error reading repository from file "+file)
 	}
 
+	// Check if mandatory key are present
+	_, err = repo.GetRepositoryFile(REPOFILE_TREE_KEY)
+	if err != nil {
+		return nil, errors.New("Invalid repository without the " + REPOFILE_TREE_KEY + " key file.")
+	}
+	_, err = repo.GetRepositoryFile(REPOFILE_META_KEY)
+	if err != nil {
+		return nil, errors.New("Invalid repository without the " + REPOFILE_META_KEY + " key file.")
+	}
+
 	return repo, err
 }
 
@@ -279,7 +382,6 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 	if err != nil {
 		return err
 	}
-	r.Index = r.Index.CleanPath()
 	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
 
 	repospec := filepath.Join(dst, REPOSITORY_SPECFILE)
@@ -302,6 +404,7 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 		r.Name, r.Revision, r.LastUpdate,
 	))
 
+	// Create tree and repository file
 	archive, err := ioutil.TempDir(os.TempDir(), "archive")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for archive")
@@ -311,26 +414,68 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 	if err != nil {
 		return errors.Wrap(err, "Error met while saving the tree")
 	}
-	tpath := r.GetTreePath()
-	if tpath == "" {
-		tpath = TREE_TARBALL
+
+	treeFile, err := r.GetRepositoryFile(REPOFILE_TREE_KEY)
+	if err != nil {
+		treeFile = NewDefaultTreeRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
 	}
 
-	a := compiler.NewPackageArtifact(filepath.Join(dst, tpath))
-	a.SetCompressionType(r.TreeCompressionType)
+	a := compiler.NewPackageArtifact(filepath.Join(dst, treeFile.GetFileName()))
+	a.SetCompressionType(treeFile.GetCompressionType())
 	err = a.Compress(archive, 1)
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating package archive")
 	}
 
-	r.TreePath = path.Base(a.GetPath())
+	// Update the tree name with the name created by compression selected.
+	treeFile.SetFileName(path.Base(a.GetPath()))
 	err = a.Hash()
 	if err != nil {
 		return errors.Wrap(err, "Failed generating checksums for tree")
 	}
-	r.TreeChecksums = a.GetChecksums()
+	treeFile.SetChecksums(a.GetChecksums())
+	r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
 
-	data, err := yaml.Marshal(r)
+	// Create Metadata struct and serialized repository
+	meta, serialized := r.Serialize()
+
+	// Create metadata file and repository file
+	metaTmpDir, err := ioutil.TempDir(os.TempDir(), "metadata")
+	defer os.RemoveAll(metaTmpDir) // clean up
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+
+	metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY)
+	if err != nil {
+		metaFile = NewDefaultMetaRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	}
+
+	repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE)
+	// Create repository.meta.yaml file
+	err = meta.WriteFile(repoMetaSpec)
+	if err != nil {
+		return err
+	}
+
+	a = compiler.NewPackageArtifact(filepath.Join(dst, metaFile.GetFileName()))
+	a.SetCompressionType(metaFile.GetCompressionType())
+	err = a.Compress(metaTmpDir, 1)
+	if err != nil {
+		return errors.Wrap(err, "Error met while archiving repository metadata")
+	}
+
+	metaFile.SetFileName(path.Base(a.GetPath()))
+	r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for metadata")
+	}
+	metaFile.SetChecksums(a.GetChecksums())
+
+	data, err := yaml.Marshal(serialized)
 	if err != nil {
 		return err
 	}
@@ -358,7 +503,7 @@ func (r *LuetSystemRepository) Client() Client {
 }
 func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	var repoUpdated bool = false
-	var treefs string
+	var treefs, metafs string
 
 	Debug("Sync of the repository", r.Name, "in progress...")
 	c := r.Client()
@@ -396,37 +541,72 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		} else {
 			treefs = r.GetTreePath()
 		}
+		if r.GetMetaPath() == "" {
+			metafs = filepath.Join(repobasedir, "metafs")
+		} else {
+			metafs = r.GetMetaPath()
+		}
 
 	} else {
 		treefs, err = ioutil.TempDir(os.TempDir(), "treefs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 		}
+		// If we always remove them, later on, no other structure can access
+		// to the tree for e.g. to retrieve finalizers
+		//defer os.RemoveAll(treefs)
+
+		metafs, err = ioutil.TempDir(os.TempDir(), "metafs")
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met whilte creating tempdir for metafs")
+		}
+		//defer os.RemoveAll(metafs)
+
 	}
 
+	// POST: treeFile and metaFile are present. I check this inside
+	// ReadSpecFile and NewLuetSystemRepositoryFromYaml
+	treeFile, _ := repo.GetRepositoryFile(REPOFILE_TREE_KEY)
+	metaFile, _ := repo.GetRepositoryFile(REPOFILE_META_KEY)
+
 	if !repoUpdated {
-		tpath := repo.GetTreePath()
-		if tpath == "" {
-			tpath = TREE_TARBALL
-		}
-		a := compiler.NewPackageArtifact(tpath)
 
-		artifact, err := c.DownloadArtifact(a)
+		// Get Tree
+		a := compiler.NewPackageArtifact(treeFile.GetFileName())
+		artifactTree, err := c.DownloadArtifact(a)
 		if err != nil {
-			return nil, errors.Wrap(err, "While downloading "+tpath)
+			return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName())
 		}
-		defer os.Remove(artifact.GetPath())
+		defer os.Remove(artifactTree.GetPath())
 
-		artifact.SetChecksums(repo.GetTreeChecksums())
-		artifact.SetCompressionType(repo.GetTreeCompressionType())
+		artifactTree.SetChecksums(treeFile.GetChecksums())
+		artifactTree.SetCompressionType(treeFile.GetCompressionType())
 
-		err = artifact.Verify()
+		err = artifactTree.Verify()
 		if err != nil {
 			return nil, errors.Wrap(err, "Tree integrity check failure")
 		}
 
 		Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
 
+		// Get Repository Metadata
+		a = compiler.NewPackageArtifact(metaFile.GetFileName())
+		artifactMeta, err := c.DownloadArtifact(a)
+		if err != nil {
+			return nil, errors.Wrap(err, "While downloading "+metaFile.GetFileName())
+		}
+		defer os.Remove(artifactMeta.GetPath())
+
+		artifactMeta.SetChecksums(metaFile.GetChecksums())
+		artifactMeta.SetCompressionType(metaFile.GetCompressionType())
+
+		err = artifactMeta.Verify()
+		if err != nil {
+			return nil, errors.Wrap(err, "Metadata integrity check failure")
+		}
+
+		Debug("Metadata tarball for the repository " + r.GetName() + " downloaded correctly.")
+
 		if r.Cached {
 			// Copy updated repository.yaml file to repo dir now that the tree is synced.
 			err = helpers.CopyFile(file, filepath.Join(repobasedir, REPOSITORY_SPECFILE))
@@ -435,14 +615,24 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 			}
 			// Remove previous tree
 			os.RemoveAll(treefs)
+			// Remove previous meta dir
+			os.RemoveAll(metafs)
 		}
 		Debug("Decompress tree of the repository " + r.Name + "...")
 
-		err = artifact.Unpack(treefs, true)
+		err = artifactTree.Unpack(treefs, true)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while unpacking tree")
 		}
 
+		// FIXME: It seems that tar with only one file doesn't create destination
+		//       directory. I create directory directly for now.
+		os.MkdirAll(metafs, os.ModePerm)
+		err = artifactMeta.Unpack(metafs, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while unpacking metadata")
+		}
+
 		tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64)
 
 		InfoC(
@@ -455,6 +645,14 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		Info("Repository", r.GetName(), "is already up to date.")
 	}
 
+	meta, err := NewLuetSystemRepositoryMetadata(
+		filepath.Join(metafs, REPOSITORY_METAFILE), false,
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "While processing "+REPOSITORY_METAFILE)
+	}
+	repo.SetIndex(meta.ToArtificatIndex())
+
 	reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
 	err = reciper.Load(treefs)
 	if err != nil {
@@ -469,6 +667,34 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	return repo, nil
 }
 
+func (r *LuetSystemRepository) Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized) {
+
+	serialized := LuetSystemRepositorySerialized{
+		Name:            r.Name,
+		Description:     r.Description,
+		Urls:            r.Urls,
+		Priority:        r.Priority,
+		Type:            r.Type,
+		Revision:        r.Revision,
+		LastUpdate:      r.LastUpdate,
+		RepositoryFiles: r.RepositoryFiles,
+	}
+
+	// Check if is needed set the index or simply use
+	// value returned by CleanPath
+	r.Index = r.Index.CleanPath()
+
+	meta := &LuetSystemRepositoryMetadata{
+		Index: []*compiler.PackageArtifact{},
+	}
+	for _, a := range r.Index {
+		art := a.(*compiler.PackageArtifact)
+		meta.Index = append(meta.Index, art)
+	}
+
+	return meta, serialized
+}
+
 func (r Repositories) Len() int      { return len(r) }
 func (r Repositories) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
 func (r Repositories) Less(i, j int) bool {

pkg/installer/repository_test.go

@@ -86,12 +86,14 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
 			err = repo.Write(tmpdir, false)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
-			Expect(helpers.Exists(spec.Rel(TREE_TARBALL))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).To(BeTrue())
 		})
 	})
 	Context("Matching packages", func() {

pkg/package/package.go

@@ -141,16 +141,16 @@ func (t *DefaultPackage) JSON() ([]byte, error) {
 
 // DefaultPackage represent a standard package definition
 type DefaultPackage struct {
-	ID               int      `storm:"id,increment" json:"id"` // primary key with auto increment
-	Name             string   `json:"name"`                    // Affects YAML field names too.
-	Version          string   `json:"version"`                 // Affects YAML field names too.
-	Category         string   `json:"category"`                // Affects YAML field names too.
-	UseFlags         []string `json:"use_flags"`               // Affects YAML field names too.
-	State            State
-	PackageRequires  []*DefaultPackage `json:"requires"`  // Affects YAML field names too.
-	PackageConflicts []*DefaultPackage `json:"conflicts"` // Affects YAML field names too.
-	IsSet            bool              `json:"set"`       // Affects YAML field names too.
-	Provides         []*DefaultPackage `json:"provides"`  // Affects YAML field names too.
+	ID               int               `storm:"id,increment" json:"id"` // primary key with auto increment
+	Name             string            `json:"name"`                    // Affects YAML field names too.
+	Version          string            `json:"version"`                 // Affects YAML field names too.
+	Category         string            `json:"category"`                // Affects YAML field names too.
+	UseFlags         []string          `json:"use_flags,omitempty"`     // Affects YAML field names too.
+	State            State             `json:"state,omitempty"`
+	PackageRequires  []*DefaultPackage `json:"requires"`           // Affects YAML field names too.
+	PackageConflicts []*DefaultPackage `json:"conflicts"`          // Affects YAML field names too.
+	IsSet            bool              `json:"set,omitempty"`      // Affects YAML field names too.
+	Provides         []*DefaultPackage `json:"provides,omitempty"` // Affects YAML field names too.
 
 	// TODO: Annotations?
 

scripts/ginkgo.coverage.sh

@@ -23,7 +23,7 @@ coveragetxt="coverage.txt"
 
 
 generate_cover_data() {
-  ginkgo -failFast -cover -r .
+  ginkgo -flakeAttempts=3 -race -failFast -cover -r .
   echo "" > ${coveragetxt}
   find . -type f -name "*.coverprofile" | while read -r file;  do cat "$file" >> ${coveragetxt} && mv "$file" "${coverdir}"; done
   echo "mode: $covermode" >"$profile"

tests/fixtures/finalizers/alpine/build.yaml

@@ -0,0 +1,2 @@
+image: "alpine"
+unpack: true

tests/fixtures/finalizers/alpine/definition.yaml

@@ -0,0 +1,3 @@
+category: "seed"
+name: "alpine"
+version: "1.0"

tests/fixtures/finalizers/alpine/finalize.yaml

@@ -0,0 +1,2 @@
+install:
+- touch /tmp/foo

tests/integration/01_simple_gzip.sh

@@ -28,7 +28,7 @@ testRepo() {
     --descr "Test Repo" \
     --urls $tmpdir/testrootfs \
     --tree-compression gzip \
-    --tree-path foo.tar \
+    --tree-filename foo.tar \
     --type disk > /dev/null
 
     createst=$?

tests/integration/01_simple_meta_gzip.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c-1.0 > /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --tree-compression gzip \
+    --tree-filename foo.tar \
+    --meta-filename repository.meta.tar \
+    --meta-compression gzip \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+    assertTrue 'create named tree in gzip' "[ -e '$tmpdir/testbuild/foo.tar.gz' ]"
+    assertTrue 'create tree in gzip-only' "[ ! -e '$tmpdir/testbuild/foo.tar' ]"
+    assertTrue 'create named meta in gzip' "[ -e '$tmpdir/testbuild/repository.meta.tar.gz' ]"
+    assertTrue 'create meta in gzip-only' "[ ! -e '$tmpdir/testbuild/repository.meta.tar' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/c-1.0
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testReInstall() {
+    output=$(luet install --config $tmpdir/luet.yaml  test/c-1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertContains 'contains warning' "$output" 'Filtering out'
+}
+
+testUnInstall() {
+    luet uninstall --config $tmpdir/luet.yaml test/c-1.0
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+testInstallAgain() {
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    output=$(luet install --config $tmpdir/luet.yaml test/c-1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'Filtering out'
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/05_upgrade.sh

@@ -108,13 +108,15 @@ testInstall() {
 }
 
 testUpgrade() {
-    luet --config $tmpdir/luet.yaml upgrade
+    upgrade=$(luet --config $tmpdir/luet.yaml upgrade)
     installst=$?
     assertEquals 'install test successfully' "$installst" "0"
     assertTrue 'package uninstalled B' "[ ! -e '$tmpdir/testrootfs/test5' ]"
     assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/newc' ]"
     assertTrue 'package uninstalled A' "[ ! -e '$tmpdir/testrootfs/testaa' ]"
     assertTrue 'package installed new A' "[ -e '$tmpdir/testrootfs/testlatest' ]"
+    assertNotContains 'does not contain test/c-1.0' "$upgrade" 'test/c-1.0'
+    assertNotContains 'does not attempt to download test/c-1.0' "$upgrade" 'test/c-1.0 downloaded'
 }
 
 # Load shUnit2.

tests/integration/06_search.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/b-1.0 
+    buildst=$?
+    assertTrue 'create package B 1.0' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/b-1.1
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package B 1.1' "[ -e '$tmpdir/testbuild/b-test-1.1.package.tar.gz' ]"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/a-1.0 
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package A 1.0' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/a-1.1
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    assertTrue 'create package A 1.1' "[ -e '$tmpdir/testbuild/a-test-1.1.package.tar.gz' ]"
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/a-1.2
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    assertTrue 'create package A 1.2' "[ -e '$tmpdir/testbuild/a-test-1.2.package.tar.gz' ]"
+
+
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" --destination $tmpdir/testbuild --compression gzip test/c-1.0
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package C 1.0' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/upgrade_integration" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/b-1.0
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/test5' ]"
+
+    luet install --config $tmpdir/luet.yaml test/a-1.0
+    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    luet install --config $tmpdir/luet.yaml test/a-1.1
+    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/testaa' ]"
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package keeps old A' "[ -e '$tmpdir/testrootfs/testaa' ]"
+    assertTrue 'package new A was not installed' "[ ! -e '$tmpdir/testrootfs/testlatest' ]"
+
+    luet install --config $tmpdir/luet.yaml test/c-1.0
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed C' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testSearch() {
+    installed=$(luet --config $tmpdir/luet.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+
+    assertContains 'contains test/b-1.0' "$installed" 'test b 1.0'
+    assertContains 'contains test/a-1.0' "$installed" 'test a 1.0'
+    assertContains 'contains test/c-1.0' "$installed" 'test c 1.0'
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/07_finalizer.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/finalizers" --destination $tmpdir/testbuild --compression gzip --all > /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/alpine-seed-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/finalizers" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml seed/alpine
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/bin/busybox' ]"
+    assertTrue 'finalizer runs' "[ -e '$tmpdir/testrootfs/tmp/foo' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+