Tag 0.8.4

Is it more faster in this way as we already have all the needed folders
to the comparison extracted. In this way we don't repeat I/O operation
twice by calling container-diff.

Do not depend on container-diff anymore

.travis.yml

@@ -6,14 +6,14 @@ go:
 env:
   - "GO15VENDOREXPERIMENT=1"
 before_install:
-  - make deps
-  - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && mkdir -p $HOME/bin && export PATH=$PATH:$HOME/bin && mv container-diff-linux-amd64 $HOME/bin/container-diff
+  - sudo -E env "PATH=$PATH" apt-get install -y libcap2-bin
+  - sudo -E env "PATH=$PATH" make deps
 script:
-  - make multiarch-build test-integration test-coverage
+  - sudo -E env "PATH=$PATH" make multiarch-build test-integration test-coverage
 after_success:
   - |
       if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
-        git config --global user.name "Deployer" && git config --global user.email foo@bar.com
-        go get github.com/tcnksm/ghr
-        ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+        sudo -E env "PATH=$PATH" git config --global user.name "Deployer" && git config --global user.email foo@bar.com
+        sudo -E env "PATH=$PATH" go get github.com/tcnksm/ghr
+        sudo -E env "PATH=$PATH" ghr -u mudler -r luet --replace $TRAVIS_TAG release/
       fi

cmd/root.go

@@ -38,7 +38,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.8.3"
+	LuetCLIVersion = "0.8.4"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/search.go

@@ -32,6 +32,7 @@ type PackageResult struct {
 	Category   string `json:"category"`
 	Version    string `json:"version"`
 	Repository string `json:"repository"`
+	Hidden     bool   `json:"hidden"`
 }
 
 type Results struct {
@@ -58,6 +59,9 @@ var searchCmd = &cobra.Command{
 		if len(args) != 1 {
 			Fatal("Wrong number of arguments (expected 1)")
 		}
+
+		hidden, _ := cmd.Flags().GetBool("hidden")
+
 		installed := LuetCfg.Viper.GetBool("installed")
 		stype := LuetCfg.Viper.GetString("solver.type")
 		discount := LuetCfg.Viper.GetFloat64("solver.discount")
@@ -114,25 +118,31 @@ var searchCmd = &cobra.Command{
 			}
 			for _, m := range matches {
 				if !revdeps {
-					Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
-					results.Packages = append(results.Packages,
-						PackageResult{
-							Name:       m.Package.GetName(),
-							Version:    m.Package.GetVersion(),
-							Category:   m.Package.GetCategory(),
-							Repository: m.Repo.GetName(),
-						})
+					if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
+						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", m.Package.HumanReadableString()))
+						results.Packages = append(results.Packages,
+							PackageResult{
+								Name:       m.Package.GetName(),
+								Version:    m.Package.GetVersion(),
+								Category:   m.Package.GetCategory(),
+								Repository: m.Repo.GetName(),
+								Hidden:     m.Package.IsHidden(),
+							})
+					}
 				} else {
 					visited := make(map[string]interface{})
 					for _, revdep := range m.Package.ExpandedRevdeps(m.Repo.GetTree().GetDatabase(), visited) {
-						Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
-						results.Packages = append(results.Packages,
-							PackageResult{
-								Name:       revdep.GetName(),
-								Version:    revdep.GetVersion(),
-								Category:   revdep.GetCategory(),
-								Repository: m.Repo.GetName(),
-							})
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							Info(fmt.Sprintf(":file_folder:%s", m.Repo.GetName()), fmt.Sprintf(":package:%s", revdep.HumanReadableString()))
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: m.Repo.GetName(),
+									Hidden:     revdep.IsHidden(),
+								})
+						}
 					}
 				}
 			}
@@ -162,26 +172,32 @@ var searchCmd = &cobra.Command{
 
 			for _, pack := range iMatches {
 				if !revdeps {
-					Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
-					results.Packages = append(results.Packages,
-						PackageResult{
-							Name:       pack.GetName(),
-							Version:    pack.GetVersion(),
-							Category:   pack.GetCategory(),
-							Repository: "system",
-						})
+					if !pack.IsHidden() || pack.IsHidden() && hidden {
+						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+						results.Packages = append(results.Packages,
+							PackageResult{
+								Name:       pack.GetName(),
+								Version:    pack.GetVersion(),
+								Category:   pack.GetCategory(),
+								Repository: "system",
+								Hidden:     pack.IsHidden(),
+							})
+					}
 				} else {
 					visited := make(map[string]interface{})
 
 					for _, revdep := range pack.ExpandedRevdeps(system.Database, visited) {
-						Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
-						results.Packages = append(results.Packages,
-							PackageResult{
-								Name:       revdep.GetName(),
-								Version:    revdep.GetVersion(),
-								Category:   revdep.GetCategory(),
-								Repository: "system",
-							})
+						if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+							Info(fmt.Sprintf(":package:%s", pack.HumanReadableString()))
+							results.Packages = append(results.Packages,
+								PackageResult{
+									Name:       revdep.GetName(),
+									Version:    revdep.GetVersion(),
+									Category:   revdep.GetCategory(),
+									Repository: "system",
+									Hidden:     revdep.IsHidden(),
+								})
+						}
 					}
 				}
 			}
@@ -223,5 +239,7 @@ func init() {
 	searchCmd.Flags().Bool("by-label", false, "Search packages through label")
 	searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex")
 	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
+
 	RootCmd.AddCommand(searchCmd)
 }

pkg/compiler/artifact.go

@@ -27,6 +27,7 @@ import (
 	"path/filepath"
 	"regexp"
 
+	system "github.com/Luet-lab/moby/pkg/system"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
@@ -477,6 +478,27 @@ type CopyJob struct {
 	Artifact string
 }
 
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
@@ -490,10 +512,13 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 		// 	continue
 		// }
 
-		if !helpers.Exists(job.Dst) {
+		_, err := os.Lstat(job.Dst)
+		if err != nil {
+			fmt.Println("Copying ", job.Src)
 			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
+			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
@@ -550,19 +575,33 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 					}
 				}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	} else {
 		// Otherwise just grab all
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 			for _, a := range l.Diffs.Additions {
+				Debug("File ", a.Name, " added")
 				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	}
 
 	close(toCopy)
 	wg.Wait()
+
 	a := NewPackageArtifact(dst)
 	a.SetCompressionType(t)
 	err = a.Compress(archive, concurrency)

pkg/compiler/backend/diff.go

@@ -0,0 +1,199 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/pkg/errors"
+)
+
+// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
+// example of json return: [
+//   {
+//     "Image1": "luet/base",
+//     "Image2": "alpine",
+//     "DiffType": "File",
+//     "Diff": {
+//       "Adds": null,
+//       "Dels": [
+//         {
+//           "Name": "/luetbuild",
+//           "Size": 5830706
+//         },
+//         {
+//           "Name": "/luetbuild/Dockerfile",
+//           "Size": 50
+//         },
+//         {
+//           "Name": "/luetbuild/output1",
+//           "Size": 5830656
+//         }
+//       ],
+//       "Mods": null
+//     }
+//   }
+// ]
+func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]compiler.ArtifactLayer, error) {
+
+	res := compiler.ArtifactLayer{FromImage: srcImage, ToImage: dstImage}
+
+	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(srcRootFS) // clean up
+
+	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(dstRootFS) // clean up
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(srcImage, "/") {
+		srcImageTar, err := ioutil.TempFile(tmpdiffs, "srctar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(srcImageTar.Name()) // clean up
+		srcImageExport := compiler.CompilerBackendOptions{
+			ImageName:   srcImage,
+			Destination: srcImageTar.Name(),
+		}
+		err = b.ExportImage(srcImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting src image "+srcImage)
+		}
+		srcImage = srcImageTar.Name()
+	}
+
+	srcImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  srcImage,
+		Destination: srcRootFS,
+	}
+	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+srcImage)
+	}
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(dstImage, "/") {
+		dstImageTar, err := ioutil.TempFile(tmpdiffs, "dsttar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(dstImageTar.Name()) // clean up
+		dstImageExport := compiler.CompilerBackendOptions{
+			ImageName:   dstImage,
+			Destination: dstImageTar.Name(),
+		}
+		err = b.ExportImage(dstImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting dst image "+dstImage)
+		}
+		dstImage = dstImageTar.Name()
+	}
+
+	dstImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  dstImage,
+		Destination: dstRootFS,
+	}
+	err = b.ExtractRootfs(dstImageExtract, false)
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+dstImage)
+	}
+
+	// Get Additions/Changes. dst -> src
+	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, dstRootFS, "", -1)
+		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
+		if err == nil {
+			var sizeA, sizeB int64
+			sizeA = fileInfo.Size()
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+
+			if sizeA != sizeB {
+				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
+				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
+					Name: filepath.Join("/", realpath),
+					Size: int(sizeB),
+				})
+			} else {
+				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
+			}
+		} else {
+			var sizeB int64
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+				Size: int(sizeB),
+			})
+
+			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+	}
+
+	// Get deletions. src -> dst
+	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, srcRootFS, "", -1)
+		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
+			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
+			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+			})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+	}
+
+	return []compiler.ArtifactLayer{res}, nil
+}

pkg/compiler/backend/diff_test.go

@@ -0,0 +1,59 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend_test
+
+import (
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Docker backend", func() {
+	var b compiler.CompilerBackend
+
+	BeforeEach(func() {
+		b = NewSimpleDockerBackend()
+	})
+
+	Context("Simple Docker backend satisfies main interface functionalities", func() {
+		It("Builds and generate tars", func() {
+			layers, err := GenerateChanges(b, "alpine:latest", "alpine:latest")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+
+		It("Builds and generate tars", func() {
+			layers, err := GenerateChanges(b, "quay.io/mocaccino/micro", "quay.io/mocaccino/extra")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+
+			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+	})
+})

pkg/compiler/backend/simpledocker.go

@@ -16,7 +16,6 @@
 package backend
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -222,64 +221,9 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 	return nil
 }
 
-// 	container-diff diff daemon://luet/base alpine --type=file -j
-// [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("tmpdiffs")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-	var errorBuffer bytes.Buffer
-
-	diffargs := []string{"diff", fromImage, toImage, "-v", "error", "-q", "--type=file", "-j", "-n", "-c", tmpdiffs}
-	cmd := exec.Command("container-diff", diffargs...)
-	cmd.Stderr = &errorBuffer
-	out, err := cmd.Output()
-
-	if string(errorBuffer.Bytes()) != "" {
-		Warning("container-diff errored with: " + string(errorBuffer.Bytes()))
-	}
-
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
-	}
-
-	if config.LuetCfg.GetGeneral().ShowBuildOutput {
-		Info(string(out))
-	}
-
-	var diffs []compiler.ArtifactLayer
-
-	err = json.Unmarshal(out, &diffs)
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
-	}
+// Changes retrieves changes between image layers
+func (d *SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+	diffs, err := GenerateChanges(d, fromImage, toImage)
 
 	if config.LuetCfg.GetGeneral().Debug {
 		summary := compiler.ComputeArtifactLayerSummary(diffs)
@@ -292,5 +236,5 @@ func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLaye
 		}
 	}
 
-	return diffs, nil
+	return diffs, err
 }

pkg/compiler/backend/simpleimg.go

@@ -149,8 +149,8 @@ func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms
 
 // TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
 // Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	return NewSimpleDockerBackend().Changes(fromImage, toImage)
+func (i *SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+	return GenerateChanges(i, fromImage, toImage)
 }
 
 func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {

pkg/compiler/compiler.go

@@ -372,7 +372,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	}
 	//	}
 
-	var diffs []ArtifactLayer
 	var artifact Artifact
 	unpack := p.ImageUnpack()
 
@@ -382,14 +381,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		unpack = true
 	}
 
-	if !unpack {
-		// we have to get diffs only if spec is not unpacked
-		diffs, err = cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate changes from layers")
-		}
-	}
-
 	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
@@ -432,6 +423,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 		artifact.SetCompressionType(cs.CompressionType)
+
 		err = artifact.Compress(rootfs, concurrency)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating package archive")
@@ -440,7 +432,10 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		artifact.SetCompileSpec(p)
 	} else {
 		Info(pkgTag, "Generating delta")
-
+		diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not generate changes from layers")
+		}
 		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not generate deltas")

pkg/package/package.go

@@ -42,8 +42,7 @@ type Package interface {
 	Encode(PackageDatabase) (string, error)
 
 	BuildFormula(PackageDatabase, PackageDatabase) ([]bf.Formula, error)
-	IsFlagged(bool) Package
-	Flagged() bool
+
 	GetFingerPrint() string
 	GetPackageName() string
 	Requires([]*DefaultPackage) Package
@@ -99,6 +98,7 @@ type Package interface {
 	HasAnnotation(string) bool
 	MatchAnnotation(*regexp.Regexp) bool
 
+	IsHidden() bool
 	IsSelector() bool
 	VersionMatchSelector(string, version.Versioner) (bool, error)
 	SelectorMatchVersion(string, version.Versioner) (bool, error)
@@ -164,8 +164,8 @@ type DefaultPackage struct {
 	State            State             `json:"state,omitempty"`
 	PackageRequires  []*DefaultPackage `json:"requires"`           // Affects YAML field names too.
 	PackageConflicts []*DefaultPackage `json:"conflicts"`          // Affects YAML field names too.
-	IsSet            bool              `json:"set,omitempty"`      // Affects YAML field names too.
 	Provides         []*DefaultPackage `json:"provides,omitempty"` // Affects YAML field names too.
+	Hidden           bool              `json:"hidden,omitempty"`   // Affects YAML field names too.
 
 	// Annotations are used for core features/options
 	Annotations map[string]string `json:"annotations,omitempty"` // Affects YAML field names too
@@ -260,6 +260,10 @@ func (p *DefaultPackage) IsSelector() bool {
 	return strings.ContainsAny(p.GetVersion(), "<>=")
 }
 
+func (p *DefaultPackage) IsHidden() bool {
+	return p.Hidden
+}
+
 func (p *DefaultPackage) HasLabel(label string) bool {
 	return helpers.MapHasKey(&p.Labels, label)
 }
@@ -316,15 +320,6 @@ func (p *DefaultPackage) Yaml() ([]byte, error) {
 	return y, nil
 }
 
-func (p *DefaultPackage) IsFlagged(b bool) Package {
-	p.IsSet = b
-	return p
-}
-
-func (p *DefaultPackage) Flagged() bool {
-	return p.IsSet
-}
-
 func (p *DefaultPackage) GetName() string {
 	return p.Name
 }
@@ -763,7 +758,6 @@ func (p *DefaultPackage) Explain() {
 	fmt.Println("Name: ", p.GetName())
 	fmt.Println("Category: ", p.GetCategory())
 	fmt.Println("Version: ", p.GetVersion())
-	fmt.Println("Installed: ", p.IsSet)
 
 	for _, req := range p.GetRequires() {
 		fmt.Println("\t-> ", req)

pkg/package/package_test.go

@@ -314,7 +314,6 @@ var _ = Describe("Package", func() {
 
 			Expect(p.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(p.GetName()).To(Equal(a.GetName()))
-			Expect(p.Flagged()).To(Equal(a.Flagged()))
 			Expect(p.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(p.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(p.GetRequires())).To(Equal(len(a.GetRequires())))
@@ -374,7 +373,6 @@ var _ = Describe("Package", func() {
 			a2 := a.Clone()
 			Expect(a2.GetVersion()).To(Equal(a.GetVersion()))
 			Expect(a2.GetName()).To(Equal(a.GetName()))
-			Expect(a2.Flagged()).To(Equal(a.Flagged()))
 			Expect(a2.GetFingerPrint()).To(Equal(a.GetFingerPrint()))
 			Expect(len(a2.GetConflicts())).To(Equal(len(a.GetConflicts())))
 			Expect(len(a2.GetRequires())).To(Equal(len(a.GetRequires())))

pkg/solver/solver.go

@@ -532,7 +532,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 	for _, a := range asserts {
 		if a.Value {
 			if !checkconflicts {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 				continue
 			}
 
@@ -543,7 +543,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 
 			// If doesn't conflict with installed we just consider it for removal and look for the next one
 			if !c {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 				continue
 			}
 
@@ -553,7 +553,7 @@ func (s *Solver) Uninstall(c pkg.Package, checkconflicts, full bool) (pkg.Packag
 				return nil, err
 			}
 			if !c {
-				res = append(res, a.Package.IsFlagged(false))
+				res = append(res, a.Package)
 			}
 
 		}

pkg/solver/solver_test.go

@@ -596,7 +596,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -622,7 +622,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(&pkg.DefaultPackage{Name: "A", Version: ">1.0"}, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -646,7 +646,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -670,7 +670,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				Expect(len(solution)).To(Equal(1))
 			})
@@ -693,8 +693,8 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
-				Expect(solution).ToNot(ContainElement(B.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).ToNot(ContainElement(B))
 
 				Expect(len(solution)).To(Equal(1))
 			})
@@ -718,8 +718,8 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
-				Expect(solution).To(ContainElement(C.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(C))
 
 				Expect(len(solution)).To(Equal(2))
 			})
@@ -744,9 +744,9 @@ var _ = Describe("Solver", func() {
 				solution, err := s.Uninstall(A, true, true)
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
-				Expect(solution).To(ContainElement(B.IsFlagged(false)))
-				Expect(solution).To(ContainElement(D.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(D))
 
 				Expect(len(solution)).To(Equal(3))
 
@@ -773,7 +773,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -800,7 +800,7 @@ var _ = Describe("Solver", func() {
 					&pkg.DefaultPackage{Name: "A", Version: ">1.0"}})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -824,7 +824,7 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
 
 				//	Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
 				Expect(len(solution)).To(Equal(1))
@@ -848,8 +848,8 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
-				Expect(solution).To(ContainElement(B.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
 
 				Expect(len(solution)).To(Equal(2))
 			})
@@ -874,9 +874,9 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
-				Expect(solution).To(ContainElement(B.IsFlagged(false)))
-				Expect(solution).To(ContainElement(C.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(C))
 
 				Expect(len(solution)).To(Equal(3))
 			})
@@ -900,8 +900,8 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
-				Expect(solution).To(ContainElement(C.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(C))
 
 				Expect(len(solution)).To(Equal(2))
 			})
@@ -926,9 +926,9 @@ var _ = Describe("Solver", func() {
 				solution, err := s.UninstallUniverse(pkg.Packages{A})
 				Expect(err).ToNot(HaveOccurred())
 
-				Expect(solution).To(ContainElement(A.IsFlagged(false)))
-				Expect(solution).To(ContainElement(B.IsFlagged(false)))
-				Expect(solution).To(ContainElement(D.IsFlagged(false)))
+				Expect(solution).To(ContainElement(A))
+				Expect(solution).To(ContainElement(B))
+				Expect(solution).To(ContainElement(D))
 
 				Expect(len(solution)).To(Equal(3))
 

pkg/spectooling/definition.go

@@ -29,7 +29,6 @@ type DefaultPackageSanitized struct {
 	UseFlags         []string                   `json:"use_flags,omitempty" yaml:"use_flags,omitempty"`
 	PackageRequires  []*DefaultPackageSanitized `json:"requires,omitempty" yaml:"requires,omitempty"`
 	PackageConflicts []*DefaultPackageSanitized `json:"conflicts,omitempty" yaml:"conflicts,omitempty"`
-	IsSet            bool                       `json:"set,omitempty" yaml:"set,omitempty"`
 	Provides         []*DefaultPackageSanitized `json:"provides,omitempty" yaml:"provides,omitempty"`
 
 	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
@@ -40,6 +39,7 @@ type DefaultPackageSanitized struct {
 	Description string   `json:"description,omitempty" yaml:"description,omitempty"`
 	Uri         []string `json:"uri,omitempty" yaml:"uri,omitempty"`
 	License     string   `json:"license,omitempty" yaml:"license,omitempty"`
+	Hidden      bool     `json:"hidden,omitempty" yaml:"hidden,omitempty"`
 
 	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
 }
@@ -50,7 +50,7 @@ func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
 		Version:     p.GetVersion(),
 		Category:    p.GetCategory(),
 		UseFlags:    p.GetUses(),
-		IsSet:       p.Flagged(),
+		Hidden:      p.IsHidden(),
 		Path:        p.GetPath(),
 		Description: p.GetDescription(),
 		Uri:         p.GetURI(),
@@ -68,6 +68,7 @@ func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
 					Name:     r.Name,
 					Version:  r.Version,
 					Category: r.Category,
+					Hidden:   r.IsHidden(),
 				},
 			)
 		}
@@ -82,6 +83,7 @@ func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
 					Name:     c.Name,
 					Version:  c.Version,
 					Category: c.Category,
+					Hidden:   c.IsHidden(),
 				},
 			)
 		}
@@ -96,6 +98,7 @@ func NewDefaultPackageSanitized(p pkg.Package) *DefaultPackageSanitized {
 					Name:     prov.Name,
 					Version:  prov.Version,
 					Category: prov.Category,
+					Hidden:   prov.IsHidden(),
 				},
 			)
 		}

tests/fixtures/caps/pkgA/0.1/build.yaml

@@ -0,0 +1,9 @@
+image: "alpine"
+prelude:
+- apk add libcap
+unpack: true
+includes:
+  - /file1
+steps:
+- echo "test" > /file1
+- setcap cap_net_raw+ep /file1

tests/fixtures/caps/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "caps"
+version: "0.1"

tests/fixtures/caps/pkgB/0.1/build.yaml

@@ -0,0 +1,6 @@
+image: "alpine"
+prelude:
+- apk add libcap
+steps:
+- echo "test" > /file2
+- setcap cap_net_raw+ep /file2

tests/fixtures/caps/pkgB/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "caps2"
+version: "0.1"

tests/fixtures/symlinks/pkgA/0.1/build.yaml

@@ -0,0 +1,5 @@
+image: "alpine"
+prelude:
+- echo "test" > /file2
+steps:
+- ln -s /file2 /file1

tests/fixtures/symlinks/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "pkgAsym"
+version: "0.1"

tests/fixtures/symlinks/pkgB/0.1/build.yaml

@@ -0,0 +1,8 @@
+image: "alpine"
+unpack: true
+includes:
+- /file3
+prelude:
+- echo "test" > /file2
+steps:
+- ln -s /file2 /file3

tests/fixtures/symlinks/pkgB/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "pkgBsym"
+version: "0.1"

tests/integration/15_symlinks.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/symlinks" --destination $tmpdir/testbuild --compression gzip --full --clean=true
+    buildst=$?
+    assertTrue 'create package pkgAsym 0.1' "[ -e '$tmpdir/testbuild/pkgAsym-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/symlinks" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml test/pkgAsym test/pkgBsym
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    ls -liah $tmpdir/testrootfs/
+    assertTrue 'package did not install file2' "[ ! -e '$tmpdir/testrootfs/file2' ]"
+    assertTrue 'package installed file3 as symlink' "[ -L '$tmpdir/testrootfs/file3' ]"
+    assertTrue 'package installed file1 as symlink' "[ -L '$tmpdir/testrootfs/file1' ]"
+
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/16_caps.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build -d --tree "$ROOT_DIR/tests/fixtures/caps" --same-owner=true --destination $tmpdir/testbuild --compression gzip --full --clean=true
+    buildst=$?
+    assertTrue 'create package caps 0.1' "[ -e '$tmpdir/testbuild/caps-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/caps" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    $ROOT_DIR/tests/integration/bin/luet install --config $tmpdir/luet.yaml test/caps-0.1 test/caps2-0.1
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+   
+    assertTrue 'package installed file1' "[ -e '$tmpdir/testrootfs/file1' ]"
+    assertTrue 'package installed file2' "[ -e '$tmpdir/testrootfs/file2' ]"
+
+    assertContains 'caps' "$(getcap $tmpdir/testrootfs/file1)" "cap_net_raw+ep"
+    assertContains 'caps' "$(getcap $tmpdir/testrootfs/file2)" "cap_net_raw+ep"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+