Tag 0.8.5

cmd/tree/validate: Integrate validation of buildtime deps

.github/workflows/release.yml

@@ -0,0 +1,23 @@
+on: push
+name: Build and release on push
+jobs:
+  release:
+    name: Test and Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Tests
+        run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage
+      - name: Build
+        run: sudo -E env "PATH=$PATH" make multiarch-build && sudo chmod -R 777 release/
+      - name: Release
+        uses: fnkr/github-action-ghr@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GHR_PATH: release/
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+
+on: pull_request
+name: Build and Test
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make deps multiarch-build test-integration test-coverage

.travis.yml

@@ -6,14 +6,14 @@ go:
 env:
   - "GO15VENDOREXPERIMENT=1"
 before_install:
-  - make deps
+  - sudo -E env "PATH=$PATH" apt-get install -y libcap2-bin
-  - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && mkdir -p $HOME/bin && export PATH=$PATH:$HOME/bin && mv container-diff-linux-amd64 $HOME/bin/container-diff
+  - sudo -E env "PATH=$PATH" make deps
 script:
-  - make multiarch-build test-integration test-coverage
+  - sudo -E env "PATH=$PATH" make multiarch-build test-integration test-coverage
-after_success:
+#after_success:
-  - |
+#  - |
-      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+#      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
-        git config --global user.name "Deployer" && git config --global user.email foo@bar.com
+#        sudo -E env "PATH=$PATH" git config --global user.name "Deployer" && git config --global user.email foo@bar.com
-        go get github.com/tcnksm/ghr
+#        sudo -E env "PATH=$PATH" go get github.com/tcnksm/ghr
-        ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+#        sudo -E env "PATH=$PATH" ghr -u mudler -r luet --replace $TRAVIS_TAG release/
-      fi
+#      fi

cmd/build.go

@@ -79,6 +79,7 @@ var buildCmd = &cobra.Command{
 		onlydeps := viper.GetBool("onlydeps")
 		keepExportedImages := viper.GetBool("keep-exported-images")
 		full, _ := cmd.Flags().GetBool("full")
+		skip, _ := cmd.Flags().GetBool("skip-if-metadata-exists")
 
 		compilerSpecs := compiler.NewLuetCompilationspecs()
 		var compilerBackend compiler.CompilerBackend
@@ -143,6 +144,7 @@ var buildCmd = &cobra.Command{
 		opts.OnlyDeps = onlydeps
 		opts.NoDeps = nodeps
 		opts.KeepImageExport = keepExportedImages
+		opts.SkipIfMetadataExists = skip
 
 		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts)
 		luetCompiler.SetConcurrency(concurrency)
@@ -231,6 +233,7 @@ func init() {
 	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
 	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
 	buildCmd.Flags().Bool("keep-exported-images", false, "Keep exported images used during building")
+	buildCmd.Flags().Bool("skip-if-metadata-exists", false, "Skip package if metadata exists")
 
 	buildCmd.Flags().String("solver-type", "", "Solver strategy")
 	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")

cmd/root.go

@@ -38,7 +38,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.8.2"
+	LuetCLIVersion = "0.8.5"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/search.go

@@ -32,6 +32,7 @@ type PackageResult struct {
 	Category   string `json:"category"`
 	Version    string `json:"version"`
 	Repository string `json:"repository"`
+	Hidden     bool   `json:"hidden"`
 }
 
 type Results struct {
@@ -58,6 +59,9 @@ var searchCmd = &cobra.Command{
 		if len(args) != 1 {
 			Fatal("Wrong number of arguments (expected 1)")
 		}
+
+		hidden, _ := cmd.Flags().GetBool("hidden")
+
 		installed := LuetCfg.Viper.GetBool("installed")
 		stype := LuetCfg.Viper.GetString("solver.type")
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
@@ -114,25 +118,31 @@ var searchCmd = &cobra.Command{
 			}
 			for _, m := range matches {
 				if !revdeps {
-					Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
+					if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
-					results.Packages = append(results.Packages,
+						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
-						PackageResult{
+						results.Packages = append(results.Packages,
-							Name:       m.Package.GetName(),
+							PackageResult{
-							Version:    m.Package.GetVersion(),
+								Name:       m.Package.GetName(),
-							Category:   m.Package.GetCategory(),
+								Version:    m.Package.GetVersion(),
-							Repository: m.Repo.GetName(),
+								Category:   m.Package.GetCategory(),
-						})
+								Repository: m.Repo.GetName(),
+								Hidden:     m.Package.IsHidden(),
+							})
+					}
 				} else {
 					visited := make(map[string]interface{})
 					for _, revdep := range m.Package.ExpandedRevdeps(m.Repo.GetTree().GetDatabase(), visited) {
-						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
-						results.Packages = append(results.Packages,
+							Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
-							PackageResult{
+							results.Packages = append(results.Packages,
-								Name:       revdep.GetName(),
+								PackageResult{
-								Version:    revdep.GetVersion(),
+									Name:       revdep.GetName(),
-								Category:   revdep.GetCategory(),
+									Version:    revdep.GetVersion(),
-								Repository: m.Repo.GetName(),
+									Category:   revdep.GetCategory(),
-							})
+									Repository: m.Repo.GetName(),
+									Hidden:     revdep.IsHidden(),
+								})
+						}
 					}
 				}
 			}
@@ -162,26 +172,32 @@ var searchCmd = &cobra.Command{
 
 			for _, pack := range iMatches {
 				if !revdeps {
-					Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+					if !pack.IsHidden() || pack.IsHidden() && hidden {
-					results.Packages = append(results.Packages,
+						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
-						PackageResult{
+						results.Packages = append(results.Packages,
-							Name:       pack.GetName(),
+							PackageResult{
-							Version:    pack.GetVersion(),
+								Name:       pack.GetName(),
-							Category:   pack.GetCategory(),
+								Version:    pack.GetVersion(),
-							Repository: "system",
+								Category:   pack.GetCategory(),
-						})
+								Repository: "system",
+								Hidden:     pack.IsHidden(),
+							})
+					}
 				} else {
 					visited := make(map[string]interface{})
 
 					for _, revdep := range pack.ExpandedRevdeps(system.Database, visited) {
-						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
-						results.Packages = append(results.Packages,
+							Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
-							PackageResult{
+							results.Packages = append(results.Packages,
-								Name:       revdep.GetName(),
+								PackageResult{
-								Version:    revdep.GetVersion(),
+									Name:       revdep.GetName(),
-								Category:   revdep.GetCategory(),
+									Version:    revdep.GetVersion(),
-								Repository: "system",
+									Category:   revdep.GetCategory(),
-							})
+									Repository: "system",
+									Hidden:     revdep.IsHidden(),
+								})
+						}
 					}
 				}
 			}
@@ -223,5 +239,7 @@ func init() {
 	searchCmd.Flags().Bool("by-label", false, "Search packages through label")
 	searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex")
 	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
+
 	RootCmd.AddCommand(searchCmd)
 }

cmd/tree/bump.go

@@ -18,12 +18,11 @@ package cmd_tree
 
 import (
 	"fmt"
-	//"os"
-	//"sort"
 
 	. "github.com/mudler/luet/pkg/logger"
 	spectooling "github.com/mudler/luet/pkg/spectooling"
 	tree "github.com/mudler/luet/pkg/tree"
+	version "github.com/mudler/luet/pkg/versioner"
 
 	"github.com/spf13/cobra"
 )
@@ -43,17 +42,26 @@ func NewTreeBumpCommand() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			spec, _ := cmd.Flags().GetString("definition-file")
 			toStdout, _ := cmd.Flags().GetBool("to-stdout")
+			pkgVersion, _ := cmd.Flags().GetString("pkg-version")
 			pack, err := tree.ReadDefinitionFile(spec)
 			if err != nil {
 				Fatal(err.Error())
 			}
 
-			// Retrieve version build section with Gentoo parser
+			if pkgVersion != "" {
-			err = pack.BumpBuildVersion()
+				validator := &version.WrappedVersioner{}
-			if err != nil {
+				err := validator.Validate(pkgVersion)
-				Fatal("Error on increment build version: " + err.Error())
+				if err != nil {
+					Fatal("Invalid version string: " + err.Error())
+				}
+				pack.SetVersion(pkgVersion)
+			} else {
+				// Retrieve version build section with Gentoo parser
+				err = pack.BumpBuildVersion()
+				if err != nil {
+					Fatal("Error on increment build version: " + err.Error())
+				}
 			}
-
 			if toStdout {
 				data, err := spectooling.NewDefaultPackageSanitized(&pack).Yaml()
 				if err != nil {
@@ -72,6 +80,7 @@ func NewTreeBumpCommand() *cobra.Command {
 		},
 	}
 
+	ans.Flags().StringP("pkg-version", "p", "", "Set a specific package version")
 	ans.Flags().StringP("definition-file", "f", "", "Path of the definition to bump.")
 	ans.Flags().BoolP("to-stdout", "o", false, "Bump package to output.")
 

cmd/tree/validate.go

@@ -35,248 +35,398 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func validateWorker(i int,
+type ValidateOpts struct {
-	wg *sync.WaitGroup,
+	WithSolver    bool
-	c <-chan pkg.Package,
+	OnlyRuntime   bool
-	reciper tree.Builder,
+	OnlyBuildtime bool
-	withSolver bool,
+	RegExcludes   []*regexp.Regexp
-	regExcludes, regMatches []*regexp.Regexp,
+	RegMatches    []*regexp.Regexp
-	excludes, matches []string,
+	Excludes      []string
-	errs chan error) {
+	Matches       []string
 
-	defer wg.Done()
+	// Runtime validate stuff
+	RuntimeCacheDeps *pkg.InMemoryDatabase
+	RuntimeReciper   *tree.InstallerRecipe
+
+	// Buildtime validate stuff
+	BuildtimeCacheDeps *pkg.InMemoryDatabase
+	BuildtimeReciper   *tree.CompilerRecipe
+
+	Mutex      sync.Mutex
+	BrokenPkgs int
+	BrokenDeps int
+
+	Errors []error
+}
+
+func (o *ValidateOpts) IncrBrokenPkgs() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenPkgs++
+}
+
+func (o *ValidateOpts) IncrBrokenDeps() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenDeps++
+}
+
+func (o *ValidateOpts) AddError(err error) {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.Errors = append(o.Errors, err)
+}
+
+func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, reciper tree.Builder, cacheDeps *pkg.InMemoryDatabase) error {
+	var errstr string
+	var ans error
 
 	var depSolver solver.PackageSolver
-	var cacheDeps *pkg.InMemoryDatabase
-	brokenPkgs := 0
-	brokenDeps := 0
-	var errstr string
 
-	emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+	if opts.WithSolver {
-	if withSolver {
+		emptyInstallationDb := pkg.NewInMemoryDatabase(false)
 		depSolver = solver.NewSolver(pkg.NewInMemoryDatabase(false),
 			reciper.GetDatabase(),
 			emptyInstallationDb)
-
-		// Use Singleton in memory cache for speedup dependencies
-		// analysis
-		cacheDeps = pkg.NewInMemoryDatabase(true).(*pkg.InMemoryDatabase)
 	}
 
-	for p := range c {
+	found, err := reciper.GetDatabase().FindPackages(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  ">=0",
+		},
+	)
 
-		found, err := reciper.GetDatabase().FindPackages(
+	if err != nil || len(found) < 1 {
-			&pkg.DefaultPackage{
+		if err != nil {
-				Name:     p.GetName(),
+			errstr = err.Error()
-				Category: p.GetCategory(),
+		} else {
-				Version:  ">=0",
+			errstr = "No packages"
-			},
+		}
+		Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			errstr,
+		))
+
+		opts.IncrBrokenDeps()
+
+		return errors.New(
+			fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+				checkType,
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				errstr,
+			))
+	}
+
+	// Ensure that we use the right package from right recipier for deps
+	pReciper, err := reciper.GetDatabase().FindPackage(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  p.GetVersion(),
+		},
+	)
+	if err != nil {
+		errstr = fmt.Sprintf("[%9s] %s/%s-%s: Error on retrieve package - %s.",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			err.Error(),
 		)
+		Error(errstr)
 
-		if err != nil || len(found) < 1 {
+		return errors.New(errstr)
+	}
+	p = pReciper
+
+	pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
+		p.GetVersion())
+
+	validpkg := true
+
+	if len(opts.Matches) > 0 {
+		matched := false
+		for _, rgx := range opts.RegMatches {
+			if rgx.MatchString(pkgstr) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return nil
+		}
+	}
+
+	if len(opts.Excludes) > 0 {
+		excluded := false
+		for _, rgx := range opts.RegExcludes {
+			if rgx.MatchString(pkgstr) {
+				excluded = true
+				break
+			}
+		}
+
+		if excluded {
+			return nil
+		}
+	}
+
+	Info(fmt.Sprintf("[%9s] Checking package ", checkType)+
+		fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
+		"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
+
+	all := p.GetRequires()
+	all = append(all, p.GetConflicts()...)
+	for idx, r := range all {
+
+		var deps pkg.Packages
+		var err error
+		if r.IsSelector() {
+			deps, err = reciper.GetDatabase().FindPackages(
+				&pkg.DefaultPackage{
+					Name:     r.GetName(),
+					Category: r.GetCategory(),
+					Version:  r.GetVersion(),
+				},
+			)
+		} else {
+			deps = append(deps, r)
+		}
+
+		if err != nil || len(deps) < 1 {
 			if err != nil {
 				errstr = err.Error()
 			} else {
 				errstr = "No packages"
 			}
-			Error(fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
+			Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+				checkType,
 				p.GetCategory(), p.GetName(), p.GetVersion(),
+				r.GetCategory(), r.GetName(), r.GetVersion(),
 				errstr,
 			))
 
-			errs <- errors.New(
+			opts.IncrBrokenDeps()
-				fmt.Sprintf("%s/%s-%s: Broken. No versions could be found by database %s",
-					p.GetCategory(), p.GetName(), p.GetVersion(),
-					errstr,
-				))
 
-			brokenPkgs++
+			ans = errors.New(
-		}
+				fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
-
+					checkType,
-		pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
+					p.GetCategory(), p.GetName(), p.GetVersion(),
-			p.GetVersion())
+					r.GetCategory(), r.GetName(), r.GetVersion(),
-
+					errstr))
-		validpkg := true
+
-
+			validpkg = false
-		if len(matches) > 0 {
+
-			matched := false
+		} else {
-			for _, rgx := range regMatches {
+
-				if rgx.MatchString(pkgstr) {
+			Debug(fmt.Sprintf("[%9s] Find packages for dep", checkType),
-					matched = true
+				fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
-					break
+
-				}
+			if opts.WithSolver {
-			}
+
-
+				Info(fmt.Sprintf("[%9s]  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
-			if !matched {
+					checkType,
-				continue
+					idx+1, len(all),
-			}
-		}
-
-		if len(excludes) > 0 {
-			excluded := false
-			for _, rgx := range regExcludes {
-				if rgx.MatchString(pkgstr) {
-					excluded = true
-					break
-				}
-			}
-
-			if excluded {
-				continue
-			}
-		}
-
-		Info("Checking package "+
-			fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
-			"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
-
-		all := p.GetRequires()
-		all = append(all, p.GetConflicts()...)
-		for idx, r := range all {
-
-			var deps pkg.Packages
-			var err error
-			if r.IsSelector() {
-				deps, err = reciper.GetDatabase().FindPackages(
-					&pkg.DefaultPackage{
-						Name:     r.GetName(),
-						Category: r.GetCategory(),
-						Version:  r.GetVersion(),
-					},
-				)
-			} else {
-				deps = append(deps, r)
-			}
-
-			if err != nil || len(deps) < 1 {
-				if err != nil {
-					errstr = err.Error()
-				} else {
-					errstr = "No packages"
-				}
-				Error(fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
 					p.GetCategory(), p.GetName(), p.GetVersion(),
 					r.GetCategory(), r.GetName(), r.GetVersion(),
-					errstr,
 				))
 
-				errs <- errors.New(
+				// Check if the solver is already been done for the deep
-					fmt.Sprintf("%s/%s-%s: Broken Dep %s/%s-%s - %s",
+				_, err := cacheDeps.Get(r.HashFingerprint(""))
-						p.GetCategory(), p.GetName(), p.GetVersion(),
+				if err == nil {
-						r.GetCategory(), r.GetName(), r.GetVersion(),
+					Debug(fmt.Sprintf("[%9s]  :direct_hit: Cache Hit for dep", checkType),
-						errstr))
+						fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
-
+					continue
-				brokenDeps++
+				}
-
+
-				validpkg = false
+				Spinner(32)
-
+				solution, err := depSolver.Install(pkg.Packages{r})
-			} else {
+				ass := solution.SearchByName(r.GetPackageName())
-
+				if err == nil {
-				Debug("Find packages for dep",
+					_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
-					fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+				}
-
+				SpinnerStop()
-				if withSolver {
+
-
+				if err != nil {
-					Info(fmt.Sprintf("  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
+
-						idx+1, len(all),
+					Error(fmt.Sprintf("[%9s] %s/%s-%s: solver broken for dep %s/%s-%s - %s",
+						checkType,
 						p.GetCategory(), p.GetName(), p.GetVersion(),
 						r.GetCategory(), r.GetName(), r.GetVersion(),
+						err.Error(),
 					))
 
-					// Check if the solver is already been done for the deep
+					ans = errors.New(
-					_, err := cacheDeps.Get(r.HashFingerprint(""))
+						fmt.Sprintf("[%9s] %s/%s-%s: solver broken for Dep %s/%s-%s - %s",
-					if err == nil {
+							checkType,
-						Debug("  :direct_hit: Cache Hit for dep",
-							fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
-						continue
-					}
-
-					Spinner(32)
-					solution, err := depSolver.Install(pkg.Packages{r})
-					ass := solution.SearchByName(r.GetPackageName())
-					if err == nil {
-						_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
-					}
-					SpinnerStop()
-
-					if err != nil {
-
-						Error(fmt.Sprintf("%s/%s-%s: solver broken for dep %s/%s-%s - %s",
 							p.GetCategory(), p.GetName(), p.GetVersion(),
 							r.GetCategory(), r.GetName(), r.GetVersion(),
-							err.Error(),
+							err.Error()))
-						))
-
-						errs <- errors.New(
-							fmt.Sprintf("%s/%s-%s: solver broken for Dep %s/%s-%s - %s",
-								p.GetCategory(), p.GetName(), p.GetVersion(),
-								r.GetCategory(), r.GetName(), r.GetVersion(),
-								err.Error()))
-
-						brokenDeps++
-						validpkg = false
-					}
-
-					// Register the key
-					cacheDeps.Set(r.HashFingerprint(""), "1")
 
+					opts.IncrBrokenDeps()
+					validpkg = false
 				}
+
+				// Register the key
+				cacheDeps.Set(r.HashFingerprint(""), "1")
+
+			}
+		}
+
+	}
+
+	if !validpkg {
+		opts.IncrBrokenPkgs()
+	}
+
+	return ans
+}
+
+func validateWorker(i int,
+	wg *sync.WaitGroup,
+	c <-chan pkg.Package,
+	opts *ValidateOpts) {
+
+	defer wg.Done()
+
+	for p := range c {
+
+		if opts.OnlyBuildtime {
+			// Check buildtime compiler/deps
+			err := validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+		} else if opts.OnlyRuntime {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+		} else {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+			// Check buildtime compiler/deps
+			err = validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
 			}
 
 		}
 
-		if !validpkg {
+	}
-			brokenPkgs++
+}
+
+func initOpts(opts *ValidateOpts, onlyRuntime, onlyBuildtime, withSolver bool, treePaths []string) {
+	var err error
+
+	opts.OnlyBuildtime = onlyBuildtime
+	opts.OnlyRuntime = onlyRuntime
+	opts.WithSolver = withSolver
+	opts.RuntimeReciper = nil
+	opts.BuildtimeReciper = nil
+	opts.BrokenPkgs = 0
+	opts.BrokenDeps = 0
+
+	if onlyBuildtime {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+	} else if onlyRuntime {
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	} else {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	}
+
+	opts.RuntimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+	opts.BuildtimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+
+	for _, treePath := range treePaths {
+		Info(fmt.Sprintf("Loading :deciduous_tree: %s...", treePath))
+		if opts.BuildtimeReciper != nil {
+			err = opts.BuildtimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
+		}
+		if opts.RuntimeReciper != nil {
+			err = opts.RuntimeReciper.Load(treePath)
+			if err != nil {
+				Fatal("Error on load tree ", err)
+			}
 		}
 	}
+
+	opts.RegExcludes, err = helpers.CreateRegexArray(opts.Excludes)
+	if err != nil {
+		Fatal(err.Error())
+	}
+	opts.RegMatches, err = helpers.CreateRegexArray(opts.Matches)
+	if err != nil {
+		Fatal(err.Error())
+	}
+
 }
 
 func NewTreeValidateCommand() *cobra.Command {
 	var excludes []string
 	var matches []string
 	var treePaths []string
+	var opts ValidateOpts
 
 	var ans = &cobra.Command{
 		Use:   "validate [OPTIONS]",
 		Short: "Validate a tree or a list of packages",
 		Args:  cobra.OnlyValidArgs,
 		PreRun: func(cmd *cobra.Command, args []string) {
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
 			if len(treePaths) < 1 {
 				Fatal("Mandatory tree param missing.")
 			}
+			if onlyRuntime && onlyBuildtime {
+				Fatal("Both --only-runtime and --only-buildtime options are not possibile.")
+			}
 		},
 		Run: func(cmd *cobra.Command, args []string) {
+			var reciper tree.Builder
+
 			concurrency := LuetCfg.GetGeneral().Concurrency
 
 			withSolver, _ := cmd.Flags().GetBool("with-solver")
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
 
-			reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
+			opts.Excludes = excludes
-			for _, treePath := range treePaths {
+			opts.Matches = matches
-				err := reciper.Load(treePath)
+			initOpts(&opts, onlyRuntime, onlyBuildtime, withSolver, treePaths)
-				if err != nil {
-					Fatal("Error on load tree ", err)
-				}
-			}
 
-			regExcludes, err := helpers.CreateRegexArray(excludes)
+			// We need at least one valid reciper for get list of the packages.
-			if err != nil {
+			if onlyBuildtime {
-				Fatal(err.Error())
+				reciper = opts.BuildtimeReciper
-			}
+			} else {
-			regMatches, err := helpers.CreateRegexArray(matches)
+				reciper = opts.RuntimeReciper
-			if err != nil {
-				Fatal(err.Error())
 			}
 
 			all := make(chan pkg.Package)
-			errs := make(chan error)
 
 			var wg = new(sync.WaitGroup)
 
 			for i := 0; i < concurrency; i++ {
 				wg.Add(1)
-				go validateWorker(i, wg, all,
+				go validateWorker(i, wg, all, &opts)
-					reciper, withSolver, regExcludes, regMatches, excludes, matches,
-					errs)
 			}
 			for _, p := range reciper.GetDatabase().World() {
 				all <- p
@@ -286,11 +436,10 @@ func NewTreeValidateCommand() *cobra.Command {
 			// Wait separately and once done close the channel
 			go func() {
 				wg.Wait()
-				close(errs)
 			}()
 
 			stringerrs := []string{}
-			for e := range errs {
+			for _, e := range opts.Errors {
 				stringerrs = append(stringerrs, e.Error())
 			}
 			sort.Strings(stringerrs)
@@ -300,9 +449,9 @@ func NewTreeValidateCommand() *cobra.Command {
 
 			// fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
 			if len(stringerrs) != 0 {
+				Error(fmt.Sprintf("Found %d broken packages and %d broken deps.",
+					opts.BrokenPkgs, opts.BrokenDeps))
 				Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
-				//	if brokenPkgs > 0 {
-				//os.Exit(1)
 			} else {
 				Info("All good! :white_check_mark:")
 				os.Exit(0)
@@ -310,6 +459,8 @@ func NewTreeValidateCommand() *cobra.Command {
 		},
 	}
 
+	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
+	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")
 	ans.Flags().BoolP("with-solver", "s", false,
 		"Enable check of requires also with solver.")
 	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{},

contrib/config/luet.yaml

@@ -16,6 +16,12 @@
 #   Enable JSON log format instead of console mode.
 #   json_format: false.
 #
+#   Disable/Enable color
+#   color: true
+#
+#   Enable/Disable emoji
+#   enable_emoji: true
+#
 # ---------------------------------------------
 # General configuration section:
 # ---------------------------------------------

pkg/compiler/artifact.go

@@ -27,6 +27,7 @@ import (
 	"path/filepath"
 	"regexp"
 
+	system "github.com/docker/docker/pkg/system"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
@@ -477,6 +478,27 @@ type CopyJob struct {
 	Artifact string
 }
 
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
@@ -490,10 +512,13 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 		// 	continue
 		// }
 
-		if !helpers.Exists(job.Dst) {
+		_, err := os.Lstat(job.Dst)
+		if err != nil {
+			fmt.Println("Copying ", job.Src)
 			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
+			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
@@ -550,19 +575,33 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 					}
 				}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	} else {
 		// Otherwise just grab all
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 			for _, a := range l.Diffs.Additions {
+				Debug("File ", a.Name, " added")
 				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	}
 
 	close(toCopy)
 	wg.Wait()
+
 	a := NewPackageArtifact(dst)
 	a.SetCompressionType(t)
 	err = a.Compress(archive, concurrency)

pkg/compiler/backend/diff.go

@@ -0,0 +1,199 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/pkg/errors"
+)
+
+// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
+// example of json return: [
+//   {
+//     "Image1": "luet/base",
+//     "Image2": "alpine",
+//     "DiffType": "File",
+//     "Diff": {
+//       "Adds": null,
+//       "Dels": [
+//         {
+//           "Name": "/luetbuild",
+//           "Size": 5830706
+//         },
+//         {
+//           "Name": "/luetbuild/Dockerfile",
+//           "Size": 50
+//         },
+//         {
+//           "Name": "/luetbuild/output1",
+//           "Size": 5830656
+//         }
+//       ],
+//       "Mods": null
+//     }
+//   }
+// ]
+func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]compiler.ArtifactLayer, error) {
+
+	res := compiler.ArtifactLayer{FromImage: srcImage, ToImage: dstImage}
+
+	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(srcRootFS) // clean up
+
+	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(dstRootFS) // clean up
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(srcImage, "/") {
+		srcImageTar, err := ioutil.TempFile(tmpdiffs, "srctar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(srcImageTar.Name()) // clean up
+		srcImageExport := compiler.CompilerBackendOptions{
+			ImageName:   srcImage,
+			Destination: srcImageTar.Name(),
+		}
+		err = b.ExportImage(srcImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting src image "+srcImage)
+		}
+		srcImage = srcImageTar.Name()
+	}
+
+	srcImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  srcImage,
+		Destination: srcRootFS,
+	}
+	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+srcImage)
+	}
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(dstImage, "/") {
+		dstImageTar, err := ioutil.TempFile(tmpdiffs, "dsttar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(dstImageTar.Name()) // clean up
+		dstImageExport := compiler.CompilerBackendOptions{
+			ImageName:   dstImage,
+			Destination: dstImageTar.Name(),
+		}
+		err = b.ExportImage(dstImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting dst image "+dstImage)
+		}
+		dstImage = dstImageTar.Name()
+	}
+
+	dstImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  dstImage,
+		Destination: dstRootFS,
+	}
+	err = b.ExtractRootfs(dstImageExtract, false)
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+dstImage)
+	}
+
+	// Get Additions/Changes. dst -> src
+	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, dstRootFS, "", -1)
+		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
+		if err == nil {
+			var sizeA, sizeB int64
+			sizeA = fileInfo.Size()
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+
+			if sizeA != sizeB {
+				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
+				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
+					Name: filepath.Join("/", realpath),
+					Size: int(sizeB),
+				})
+			} else {
+				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
+			}
+		} else {
+			var sizeB int64
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+				Size: int(sizeB),
+			})
+
+			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+	}
+
+	// Get deletions. src -> dst
+	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, srcRootFS, "", -1)
+		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
+			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
+			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+			})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+	}
+
+	return []compiler.ArtifactLayer{res}, nil
+}

pkg/compiler/backend/diff_test.go

@@ -0,0 +1,73 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend_test
+
+import (
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Docker image diffs", func() {
+	var b compiler.CompilerBackend
+
+	BeforeEach(func() {
+		b = NewSimpleDockerBackend()
+	})
+
+	Context("Generate diffs from docker images", func() {
+		It("Detect no changes", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "alpine:latest",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, "alpine:latest", "alpine:latest")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+
+		It("Detects additions and changed files", func() {
+			err := b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/micro",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			err = b.DownloadImage(compiler.CompilerBackendOptions{
+				ImageName: "quay.io/mocaccino/extra",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(b, "quay.io/mocaccino/micro", "quay.io/mocaccino/extra")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+
+			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+	})
+})

pkg/compiler/backend/simpledocker.go

@@ -16,7 +16,6 @@
 package backend
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -222,64 +221,9 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 	return nil
 }
 
-// 	container-diff diff daemon://luet/base alpine --type=file -j
+// Changes retrieves changes between image layers
-// [
+func (d *SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-//   {
+	diffs, err := GenerateChanges(d, fromImage, toImage)
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("tmpdiffs")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-	var errorBuffer bytes.Buffer
-
-	diffargs := []string{"diff", fromImage, toImage, "-v", "error", "-q", "--type=file", "-j", "-n", "-c", tmpdiffs}
-	cmd := exec.Command("container-diff", diffargs...)
-	cmd.Stderr = &errorBuffer
-	out, err := cmd.Output()
-
-	if string(errorBuffer.Bytes()) != "" {
-		Warning("container-diff errored with: " + string(errorBuffer.Bytes()))
-	}
-
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
-	}
-
-	if config.LuetCfg.GetGeneral().ShowBuildOutput {
-		Info(string(out))
-	}
-
-	var diffs []compiler.ArtifactLayer
-
-	err = json.Unmarshal(out, &diffs)
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
-	}
 
 	if config.LuetCfg.GetGeneral().Debug {
 		summary := compiler.ComputeArtifactLayerSummary(diffs)
@@ -292,5 +236,5 @@ func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLaye
 		}
 	}
 
-	return diffs, nil
+	return diffs, err
 }

pkg/compiler/backend/simpleimg.go

@@ -149,8 +149,8 @@ func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms
 
 // TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
 // Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+func (i *SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	return NewSimpleDockerBackend().Changes(fromImage, toImage)
+	return GenerateChanges(i, fromImage, toImage)
 }
 
 func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {

pkg/compiler/compiler.go

@@ -256,7 +256,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 
 	if !cs.Clean {
 		exists := cs.Backend.ImageExists(buildertaggedImage) && cs.Backend.ImageExists(packageImage)
-		if art, err := LoadArtifactFromYaml(p); err == nil && exists {
+		if art, err := LoadArtifactFromYaml(p); err == nil && (cs.Options.SkipIfMetadataExists || exists) {
 			Debug("Artifact reloaded. Skipping build")
 			return art, err
 		}
@@ -372,7 +372,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	}
 	//	}
 
-	var diffs []ArtifactLayer
 	var artifact Artifact
 	unpack := p.ImageUnpack()
 
@@ -382,14 +381,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		unpack = true
 	}
 
-	if !unpack {
-		// we have to get diffs only if spec is not unpacked
-		diffs, err = cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate changes from layers")
-		}
-	}
-
 	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
@@ -432,6 +423,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 		artifact.SetCompressionType(cs.CompressionType)
+
 		err = artifact.Compress(rootfs, concurrency)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating package archive")
@@ -440,7 +432,10 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		artifact.SetCompileSpec(p)
 	} else {
 		Info(pkgTag, "Generating delta")
-
+		diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not generate changes from layers")
+		}
 		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not generate deltas")
@@ -558,7 +553,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 
 	if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" {
 		Error("Package with no deps and no seed image supplied, bailing out")
-		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() + "with no deps and no seed image supplied, bailing out")
+		return nil, errors.New("Package " + p.GetPackage().GetFingerPrint() +
+			" with no deps and no seed image supplied, bailing out")
 	}
 
 	targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())

pkg/compiler/interface.go

@@ -52,9 +52,10 @@ type CompilerOptions struct {
 	Clean                    bool
 	KeepImageExport          bool
 
-	OnlyDeps      bool
+	OnlyDeps             bool
-	NoDeps        bool
+	NoDeps               bool
-	SolverOptions config.LuetSolverOptions
+	SolverOptions        config.LuetSolverOptions
+	SkipIfMetadataExists bool
 }
 
 func NewDefaultCompilerOptions() *CompilerOptions {

pkg/package/package.go

@@ -42,8 +42,7 @@ type Package interface {
 	Encode(PackageDatabase) (string, error)
 
 	BuildFormula(PackageDatabase, PackageDatabase) ([]bf.Formula, error)
-	IsFlagged(bool) Package
+
-	Flagged() bool
 	GetFingerPrint() string
 	GetPackageName() string
 	Requires([]*DefaultPackage) Package
@@ -65,6 +64,7 @@ type Package interface {
 	GetCategory() string
 
 	GetVersion() string
+	SetVersion(string)
 	RequiresContains(PackageDatabase, Package) (bool, error)
 	Matches(m Package) bool
 	BumpBuildVersion() error
@@ -99,6 +99,7 @@ type Package interface {
 	HasAnnotation(string) bool
 	MatchAnnotation(*regexp.Regexp) bool
 
+	IsHidden() bool
 	IsSelector() bool
 	VersionMatchSelector(string, version.Versioner) (bool, error)
 	SelectorMatchVersion(string, version.Versioner) (bool, error)
@@ -164,8 +165,8 @@ type DefaultPackage struct {
 	State            State             `json:"state,omitempty"`
 	PackageRequires  []*DefaultPackage `json:"requires"`           // Affects YAML field names too.
 	PackageConflicts []*DefaultPackage `json:"conflicts"`          // Affects YAML field names too.
-	IsSet            bool              `json:"set,omitempty"`      // Affects YAML field names too.
 	Provides         []*DefaultPackage `json:"provides,omitempty"` // Affects YAML field names too.
+	Hidden           bool              `json:"hidden,omitempty"`   // Affects YAML field names too.
 
 	// Annotations are used for core features/options
 	Annotations map[string]string `json:"annotations,omitempty"` // Affects YAML field names too
@@ -260,6 +261,10 @@ func (p *DefaultPackage) IsSelector() bool {
 	return strings.ContainsAny(p.GetVersion(), "<>=")
 }
 
+func (p *DefaultPackage) IsHidden() bool {
+	return p.Hidden
+}
+
 func (p *DefaultPackage) HasLabel(label string) bool {
 	return helpers.MapHasKey(&p.Labels, label)
 }
@@ -316,15 +321,6 @@ func (p *DefaultPackage) Yaml() ([]byte, error) {
 	return y, nil
 }
 
-func (p *DefaultPackage) IsFlagged(b bool) Package {
-	p.IsSet = b
-	return p
-}
-
-func (p *DefaultPackage) Flagged() bool {
-	return p.IsSet
-}
-
 func (p *DefaultPackage) GetName() string {
 	return p.Name
 }
@@ -332,6 +328,9 @@ func (p *DefaultPackage) GetName() string {
 func (p *DefaultPackage) GetVersion() string {
 	return p.Version
 }
+func (p *DefaultPackage) SetVersion(v string) {
+	p.Version = v
+}
 func (p *DefaultPackage) GetDescription() string {
 	return p.Description
 }
@@ -763,7 +762,6 @@ func (p *DefaultPackage) Explain() {
 	fmt.Println("Name: ", p.GetName())
 	fmt.Println("Category: ", p.GetCategory())
 	fmt.Println("Version: ", p.GetVersion())
-	fmt.Println("Installed: ", p.IsSet)
 
 	for _, req := range p.GetRequires() {
 		fmt.Println("\t-> ", req)

pkg/package/package_test.go

@@ -314,7 +314,6 @@ var _ = Describe("Package", func() {
 
 			Expect(p.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(p.GetName()).To(Equal(a.GetName()))
-			Expect(p.Flagged()).To(Equal(a.Flagged()))
 			Expect(p.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(p.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(p.GetRequires())).To(Equal(len(a.GetRequires())))
@@ -374,7 +373,6 @@ var _ = Describe("Package", func() {
 			a2 := a.Clone()
 			Expect(a2.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(a2.GetName()).To(Equal(a.GetName()))
-			Expect(a2.Flagged()).To(Equal(a.Flagged()))
 			Expect(a2.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(a2.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(a2.GetRequires())).To(Equal(len(a.GetRequires())))

pkg/solver/solver.go

@@ -532,7 +532,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 	for _, a := range asserts {
 		if a.Value {
 			if !checkconflicts {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 				continue
 			}
 
@@ -543,7 +543,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 
 			// If doesn't conflict with installed we just consider it for removal and look for the next one
 			if !c {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 				continue
 			}
 
@@ -553,7 +553,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 				return nil, err
 			}
 			if !c {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 			}
 
 		}

pkg/solver/solver_test.go

@@ -596,7 +596,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -622,7 +622,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"}, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -646,7 +646,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -670,7 +670,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				Expect(len(solution)).To(Equal(1))
 			})
@@ -693,8 +693,8 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
-				Expect(solution).ToNot(ContainElement(B.IsFlagged(false)))
+				Expect(solution).ToNot(ContainElement(B))
 
 				Expect(len(solution)).To(Equal(1))
 			})
@@ -718,8 +718,8 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
-				Expect(solution).To(ContainElement(C.IsFlagged(false)))
+				Expect(solution).To(ContainElement(C))
 
 				Expect(len(solution)).To(Equal(2))
 			})
@@ -744,9 +744,9 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
-				Expect(solution).To(ContainElement(B.IsFlagged(false)))
+				Expect(solution).To(ContainElement(B))
-				Expect(solution).To(ContainElement(D.IsFlagged(false)))
+				Expect(solution).To(ContainElement(D))
 
 				Expect(len(solution)).To(Equal(3))
 
@@ -773,7 +773,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -800,7 +800,7 @@ var _ = Describe("Solver", func() {
 					&pkg.DefaultPackage{Name: "A", Version: ">1.0"}})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -824,7 +824,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -848,8 +848,8 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
-				Expect(solution).To(ContainElement(B.IsFlagged(false)))
+				Expect(solution).To(ContainElement(B))
 
 				Expect(len(solution)).To(Equal(2))
 			})
@@ -874,9 +874,9 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
-				Expect(solution).To(ContainElement(B.IsFlagged(false)))
+				Expect(solution).To(ContainElement(B))
-				Expect(solution).To(ContainElement(C.IsFlagged(false)))
+				Expect(solution).To(ContainElement(C))
 
 				Expect(len(solution)).To(Equal(3))
 			})
@@ -900,8 +900,8 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
-				Expect(solution).To(ContainElement(C.IsFlagged(false)))
+				Expect(solution).To(ContainElement(C))
 
 				Expect(len(solution)).To(Equal(2))
 			})
@@ -926,9 +926,9 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
-				Expect(solution).To(ContainElement(B.IsFlagged(false)))
+				Expect(solution).To(ContainElement(B))
-				Expect(solution).To(ContainElement(D.IsFlagged(false)))
+				Expect(solution).To(ContainElement(D))
 
 				Expect(len(solution)).To(Equal(3))
 

pkg/spectooling/definition.go

@@ -29,7 +29,6 @@ type DefaultPackageSanitized struct {
 	UseFlags         []string                   `json:"use_flags,omitempty" yaml:"use_flags,omitempty"`
 	PackageRequires  []*DefaultPackageSanitized `json:"requires,omitempty" yaml:"requires,omitempty"`
 	PackageConflicts []*DefaultPackageSanitized `json:"conflicts,omitempty" yaml:"conflicts,omitempty"`
-	IsSet            bool                       `json:"set,omitempty" yaml:"set,omitempty"`
 	Provides         []*DefaultPackageSanitized `json:"provides,omitempty" yaml:"provides,omitempty"`
 
 	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
@@ -40,6 +39,7 @@ type DefaultPackageSanitized struct {
 	Description string   `json:"description,omitempty" yaml:"description,omitempty"`
 	Uri         []string `json:"uri,omitempty" yaml:"uri,omitempty"`
 	License     string   `json:"license,omitempty" yaml:"license,omitempty"`
+	Hidden      bool     `json:"hidden,omitempty" yaml:"hidden,omitempty"`
 
 	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
 }
@@ -50,7 +50,7 @@ func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
 		Version:     p.GetVersion(),
 		Category:    p.GetCategory(),
 		UseFlags:    p.GetUses(),
-		IsSet:       p.Flagged(),
+		Hidden:      p.IsHidden(),
 		Path:        p.GetPath(),
 		Description: p.GetDescription(),
 		Uri:         p.GetURI(),
@@ -68,6 +68,7 @@ func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
 					Name:     r.Name,
 					Version:  r.Version,
 					Category: r.Category,
+					Hidden:   r.IsHidden(),
 				},
 			)
 		}
@@ -82,6 +83,7 @@ func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
 					Name:     c.Name,
 					Version:  c.Version,
 					Category: c.Category,
+					Hidden:   c.IsHidden(),
 				},
 			)
 		}
@@ -96,6 +98,7 @@ func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
 					Name:     prov.Name,
 					Version:  prov.Version,
 					Category: prov.Category,
+					Hidden:   prov.IsHidden(),
 				},
 			)
 		}

tests/fixtures/caps/pkgA/0.1/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+- apk add libcap
+unpack: true
+includes:
+  - /file1
+steps:
+- echo "test" > /file1
+- setcap cap_net_raw+ep /file1

tests/fixtures/caps/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "caps"
+version: "0.1"

tests/fixtures/caps/pkgB/0.1/build.yaml

@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+- apk add libcap
+steps:
+- echo "test" > /file2
+- setcap cap_net_raw+ep /file2

tests/fixtures/caps/pkgB/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "caps2"
+version: "0.1"

tests/fixtures/symlinks/pkgA/0.1/build.yaml

@@ -0,0 +1,5 @@
+image: "alpine"
+prelude:
+- echo "test" > /file2
+steps:
+- ln -s /file2 /file1

tests/fixtures/symlinks/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "pkgAsym"
+version: "0.1"

tests/fixtures/symlinks/pkgB/0.1/build.yaml

@@ -0,0 +1,8 @@
+image: "alpine"
+unpack: true
+includes:
+- /file3
+prelude:
+- echo "test" > /file2
+steps:
+- ln -s /file2 /file3

tests/fixtures/symlinks/pkgB/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "pkgBsym"
+version: "0.1"

tests/integration/09_docker.sh

@@ -60,19 +60,15 @@ testInstall() {
     docker build --rm --no-cache -t luet:test .
     docker rm luet-runtime-test || true
     docker run --name luet-runtime-test \
-       -ti -v /tmp:/tmp \
+       -v /tmp:/tmp \
        -v $tmpdir/luet.yaml:/etc/luet/luet.yaml:ro \
        luet:test install seed/alpine
     installst=$?
     assertEquals 'install test successfully' "0" "$installst"
 
     docker commit luet-runtime-test luet-runtime-test-image
-    test=$(docker run --rm -t --entrypoint /bin/sh luet-runtime-test-image -c 'echo "ftw"')
+    test=$(docker run --rm --entrypoint /bin/sh luet-runtime-test-image -c 'echo "ftw"')
     assertContains 'generated image runs successfully' "$test" "ftw"
-   # docker rm luet-runtime-test || true
-   # docker rmi luet-runtime-test-image || true
-
-   # docker rmi luet:test || true
 }
 
 # Load shUnit2.

tests/integration/15_symlinks.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/symlinks" --destination $tmpdir/testbuild --compression gzip --full --clean=true
+    buildst=$?
+    assertTrue 'create package pkgAsym 0.1' "[ -e '$tmpdir/testbuild/pkgAsym-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/symlinks" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/pkgAsym test/pkgBsym
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    ls -liah $tmpdir/testrootfs/
+    assertTrue 'package did not install file2' "[ ! -e '$tmpdir/testrootfs/file2' ]"
+    assertTrue 'package installed file3 as symlink' "[ -L '$tmpdir/testrootfs/file3' ]"
+    assertTrue 'package installed file1 as symlink' "[ -L '$tmpdir/testrootfs/file1' ]"
+
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/16_caps.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build -d --tree "$ROOT_DIR/tests/fixtures/caps" --same-owner=true --destination $tmpdir/testbuild --compression gzip --full --clean=true
+    buildst=$?
+    assertTrue 'create package caps 0.1' "[ -e '$tmpdir/testbuild/caps-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/caps" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    $ROOT_DIR/tests/integration/bin/luet install --config $tmpdir/luet.yaml test/caps-0.1 test/caps2-0.1
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+   
+    assertTrue 'package installed file1' "[ -e '$tmpdir/testrootfs/file1' ]"
+    assertTrue 'package installed file2' "[ -e '$tmpdir/testrootfs/file2' ]"
+
+    assertContains 'caps' "$(getcap $tmpdir/testrootfs/file1)" "cap_net_raw+ep"
+    assertContains 'caps' "$(getcap $tmpdir/testrootfs/file2)" "cap_net_raw+ep"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+