Tag 0.20.7

Add api call which uses go-container registry to create OCI images from
standard tar archives.

Consume new API when generating final images instead of docker building them
and adapts/add tests as necessary.

This change now allows to carry over xattrs to final images.

Fixes #266

cmd/root.go

@@ -30,7 +30,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.20.4"
+	LuetCLIVersion = "0.20.7"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -103,7 +103,7 @@ To build a package, from a tree definition:
 		if len(bus.Manager.Plugins) != 0 {
 			util.DefaultContext.Info(":lollipop:Enabled plugins:")
 			for _, p := range bus.Manager.Plugins {
-				util.DefaultContext.Info("\t:arrow_right:", p.Name)
+				util.DefaultContext.Info(fmt.Sprintf("\t:arrow_right: %s (at %s)", p.Name, p.Executable))
 			}
 		}
 	},

pkg/api/core/bus/events.go

@@ -99,6 +99,12 @@ func (b *Bus) Initialize(ctx *types.Context, plugin ...string) {
 			if r.Errored() {
 				err := fmt.Sprintf("Plugin %s at %s had an error: %s", p.Name, p.Executable, r.Error)
 				ctx.Fatal(err)
+			} else {
+				if r.State != "" {
+					message := fmt.Sprintf(":lollipop: Plugin %s at %s succeded, state reported:", p.Name, p.Executable)
+					ctx.Success(message)
+					ctx.Info(r.State)
+				}
 			}
 		})
 	}

pkg/api/core/image/cache.go

@@ -17,7 +17,6 @@ package image
 
 import (
 	"encoding/json"
-	"fmt"
 	"os"
 	"strings"
 
@@ -156,10 +155,7 @@ func (c *Cache) All(fn func(CacheResult)) {
 	}
 
 	for key := range c.store.Keys(nil) {
-		val, err := c.store.Read(key)
-		if err != nil {
-			panic(fmt.Sprintf("key %s had no value", key))
-		}
+		val, _ := c.store.Read(key)
 		fn(CacheResult{key: key, value: string(val)})
 	}
 }

pkg/api/core/image/create.go

@@ -0,0 +1,94 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"io"
+	"os"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/pkg/errors"
+)
+
+func imageFromTar(imagename, architecture, OS string, r io.Reader) (name.Reference, v1.Image, error) {
+	newRef, err := name.ParseReference(imagename)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	layer, err := tarball.LayerFromReader(r)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	baseImage := empty.Image
+	cfg, err := baseImage.ConfigFile()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfg.Architecture = architecture
+	cfg.OS = OS
+
+	baseImage, err = mutate.ConfigFile(baseImage, cfg)
+	if err != nil {
+		return nil, nil, err
+	}
+	img, err := mutate.Append(baseImage, mutate.Addendum{
+		Layer: layer,
+		History: v1.History{
+			CreatedBy: "luet",
+			Comment:   "Custom image",
+		},
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return newRef, img, nil
+}
+
+// CreateTar a imagetarball from a standard tarball
+func CreateTar(srctar, dstimageTar, imagename, architecture, OS string) error {
+	f, err := os.Open(srctar)
+	if err != nil {
+		return errors.Wrap(err, "Cannot open "+srctar)
+	}
+	defer f.Close()
+
+	return CreateTarReader(f, dstimageTar, imagename, architecture, OS)
+}
+
+// CreateTarReader a imagetarball from a standard tarball
+func CreateTarReader(r io.Reader, dstimageTar, imagename, architecture, OS string) error {
+	dstFile, err := os.Create(dstimageTar)
+	if err != nil {
+		return errors.Wrap(err, "Cannot create "+dstimageTar)
+	}
+	defer dstFile.Close()
+
+	newRef, img, err := imageFromTar(imagename, architecture, OS, r)
+	if err != nil {
+		return err
+	}
+
+	// NOTE: We might also stream that back to the daemon with daemon.Write(tag, img)
+	return tarball.Write(newRef, img, dstFile)
+}

pkg/api/core/image/create_test.go

@@ -0,0 +1,80 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
+	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Create", func() {
+	Context("Creates an OCI image from a standard tar", func() {
+		It("creates an image which is loadable", func() {
+			ctx := types.NewContext()
+
+			dst, err := ctx.Config.System.TempFile("dst")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(dst.Name())
+			srcTar, err := ctx.Config.System.TempFile("srcTar")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(srcTar.Name())
+
+			b := backend.NewSimpleDockerBackend(ctx)
+
+			b.DownloadImage(backend.Options{ImageName: "alpine"})
+			img, err := b.ImageReference("alpine", false)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, dir, err := Extract(ctx, img, false, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			defer os.RemoveAll(dir)
+
+			Expect(file.Touch(filepath.Join(dir, "test"))).ToNot(HaveOccurred())
+			Expect(file.Exists(filepath.Join(dir, "bin"))).To(BeTrue())
+
+			a := artifact.NewPackageArtifact(srcTar.Name())
+			a.Compress(dir, 1)
+
+			// Unfortunately there is no other easy way to test this
+			err = CreateTar(srcTar.Name(), dst.Name(), "testimage", runtime.GOARCH, runtime.GOOS)
+			Expect(err).ToNot(HaveOccurred())
+
+			b.LoadImage(dst.Name())
+
+			Expect(b.ImageExists("testimage")).To(BeTrue())
+
+			img, err = b.ImageReference("testimage", false)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, dir, err = Extract(ctx, img, false, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			defer os.RemoveAll(dir)
+			Expect(file.Exists(filepath.Join(dir, "bin"))).To(BeTrue())
+			Expect(file.Exists(filepath.Join(dir, "test"))).To(BeTrue())
+		})
+	})
+})

pkg/api/core/image/extract.go

@@ -19,6 +19,7 @@ import (
 	"archive/tar"
 	"context"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -32,11 +33,10 @@ import (
 	"github.com/pkg/errors"
 )
 
-// Extract dir:
-// -> First extract delta considering the dir
-// Afterward create artifact pointing to the dir
-
-// ExtractDeltaFiles returns an handler to extract files in a list
+// ExtractDeltaAdditionsFromImages is a filter that takes two images
+// an includes and an excludes list. It computes the delta between the images
+// considering the added files only, and applies a filter on them based on the regexes
+// in the lists.
 func ExtractDeltaAdditionsFiles(
 	ctx *types.Context,
 	srcimg v1.Image,
@@ -127,6 +127,9 @@ func ExtractDeltaAdditionsFiles(
 	}, nil
 }
 
+// ExtractFiles returns a filter that extracts files from the given path (if not empty)
+// It then filters files by an include and exclude list.
+// The list can be regexes
 func ExtractFiles(
 	ctx *types.Context,
 	prefixPath string,
@@ -190,6 +193,9 @@ func ExtractFiles(
 	}
 }
 
+// ExtractReader perform the extracting action over the io.ReadCloser
+// it extracts the files over output. Accepts a filter as an option
+// and additional containerd Options
 func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	defer reader.Close()
 
@@ -203,6 +209,7 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keep
 		PAX, Xattrs map[string]string
 		Uid, Gid    int
 		Name        string
+		FileMode    fs.FileMode
 	}
 
 	permstore, err := ctx.Config.System.TempDir("permstore")
@@ -217,8 +224,9 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keep
 			perms.SetValue(h.Name, permData{
 				PAX: h.PAXRecords,
 				Uid: h.Uid, Gid: h.Gid,
-				Xattrs: h.Xattrs,
-				Name:   h.Name,
+				Xattrs:   h.Xattrs,
+				Name:     h.Name,
+				FileMode: h.FileInfo().Mode(),
 			})
 			//perms = append(perms, })
 		}
@@ -244,6 +252,10 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keep
 				if err := os.Lchown(ff, p.Uid, p.Gid); err != nil {
 					ctx.Warning(err, "failed chowning file")
 				}
+				ctx.Debug("Set", p.Name, p.FileMode)
+				if err := os.Chmod(ff, p.FileMode); err != nil {
+					ctx.Warning(err, "failed chmod file")
+				}
 			}
 			for _, attrs := range []map[string]string{p.Xattrs, p.PAX} {
 				for k, attr := range attrs {
@@ -259,6 +271,7 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keep
 	return c, output, nil
 }
 
+// Extract is just syntax sugar around ExtractReader. It extracts an image into a dir
 func Extract(ctx *types.Context, img v1.Image, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
 	if err != nil {
@@ -267,6 +280,7 @@ func Extract(ctx *types.Context, img v1.Image, keepPerms bool, filter func(h *ta
 	return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, keepPerms, filter, opts...)
 }
 
+// ExtractTo is just syntax sugar around ExtractReader
 func ExtractTo(ctx *types.Context, img v1.Image, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	return ExtractReader(ctx, mutate.Extract(img), output, keepPerms, filter, opts...)
 }

pkg/api/core/types/artifact/artifact.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 
 	"github.com/docker/docker/pkg/pools"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -174,12 +175,6 @@ func (a *PackageArtifact) GetFileName() string {
 	return path.Base(a.Path)
 }
 
-func (a *PackageArtifact) genDockerfile() string {
-	return `
-FROM scratch
-COPY . /`
-}
-
 // CreateArtifactForFile creates a new artifact from the given file
 func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
 	if _, err := os.Stat(s); os.IsNotExist(err) {
@@ -211,52 +206,36 @@ func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageAr
 
 type ImageBuilder interface {
 	BuildImage(backend.Options) error
+	LoadImage(path string) error
 }
 
 // GenerateFinalImage takes an artifact and builds a Docker image with its content
-func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) {
-	builderOpts := backend.Options{}
-	archive, err := ctx.Config.GetSystem().TempDir("archive")
+func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) error {
+	archiveFile, err := os.Open(a.Path)
 	if err != nil {
-		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+		return errors.Wrap(err, "Cannot open "+a.Path)
 	}
-	defer os.RemoveAll(archive) // clean up
+	defer archiveFile.Close()
 
-	uncompressedFiles := filepath.Join(archive, "files")
-	dockerFile := filepath.Join(archive, "Dockerfile")
-
-	if err := os.MkdirAll(uncompressedFiles, os.ModePerm); err != nil {
-		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
-	}
-
-	if err := a.Unpack(ctx, uncompressedFiles, keepPerms); err != nil {
-		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
-	}
-
-	empty, err := fileHelper.DirectoryIsEmpty(uncompressedFiles)
+	decompressed, err := containerdCompression.DecompressStream(archiveFile)
 	if err != nil {
-		return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles)
+		return errors.Wrap(err, "Cannot open "+a.Path)
 	}
 
-	// See https://github.com/moby/moby/issues/38039.
-	// We can't generate FROM scratch empty images. Docker will refuse to export them
-	// workaround: Inject a .virtual empty file
-	if empty {
-		fileHelper.Touch(filepath.Join(uncompressedFiles, ".virtual"))
+	tempimage, err := ctx.Config.GetSystem().TempFile("tempimage")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	}
+	defer os.RemoveAll(tempimage.Name()) // clean up
+
+	if err := image.CreateTarReader(decompressed, tempimage.Name(), imageName, runtime.GOARCH, runtime.GOOS); err != nil {
+		return errors.Wrap(err, "could not create image from tar")
 	}
 
-	data := a.genDockerfile()
-	if err := ioutil.WriteFile(dockerFile, []byte(data), 0644); err != nil {
-		return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path)
+	if err := b.LoadImage(tempimage.Name()); err != nil {
+		return errors.Wrap(err, "while loading image")
 	}
-
-	builderOpts = backend.Options{
-		ImageName:      imageName,
-		SourcePath:     archive,
-		DockerFileName: dockerFile,
-		Context:        uncompressedFiles,
-	}
-	return builderOpts, b.BuildImage(builderOpts)
+	return nil
 }
 
 // Compress is responsible to archive and compress to the artifact Path.

pkg/api/core/types/artifact/artifact_test.go

@@ -146,9 +146,8 @@ RUN echo bar > /test2`))
 			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false)
+			err = a.GenerateFinalImage(ctx, resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(opts.ImageName).To(Equal(resultingImage))
 
 			Expect(b.ImageExists(resultingImage)).To(BeTrue())
 
@@ -196,9 +195,8 @@ RUN echo bar > /test2`))
 			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
 			resultingImage := imageprefix + "foo--1.0"
-			opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false)
+			err = a.GenerateFinalImage(ctx, resultingImage, b, false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(opts.ImageName).To(Equal(resultingImage))
 
 			Expect(b.ImageExists(resultingImage)).To(BeTrue())
 
@@ -217,11 +215,7 @@ RUN echo bar > /test2`))
 			)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(fileHelper.DirectoryIsEmpty(result)).To(BeFalse())
-			content, err := ioutil.ReadFile(filepath.Join(result, ".virtual"))
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(string(content)).To(Equal(""))
+			Expect(fileHelper.DirectoryIsEmpty(result)).To(BeTrue())
 		})
 
 		It("Retrieves uncompressed name", func() {

pkg/api/core/types/context.go

@@ -52,7 +52,6 @@ type Context struct {
 	s           *pterm.SpinnerPrinter
 	spinnerLock *sync.Mutex
 	z           *zap.Logger
-	AreaPrinter *pterm.AreaPrinter
 	ProgressBar *pterm.ProgressbarPrinter
 }
 

pkg/compiler/backend.go

@@ -26,6 +26,7 @@ func NewBackend(ctx *types.Context, s string) (CompilerBackend, error) {
 type CompilerBackend interface {
 	BuildImage(backend.Options) error
 	ExportImage(backend.Options) error
+	LoadImage(string) error
 	RemoveImage(backend.Options) error
 	ImageDefinitionToTar(backend.Options) error
 

pkg/compiler/backend/simpledocker.go

@@ -71,6 +71,17 @@ func (s *SimpleDocker) CopyImage(src, dst string) error {
 	return nil
 }
 
+func (s *SimpleDocker) LoadImage(path string) error {
+	s.ctx.Debug(":whale: Loading image:", path)
+	cmd := exec.Command("docker", "load", "-i", path)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed loading image: "+string(out))
+	}
+	s.ctx.Success(":whale: Loaded image:", path)
+	return nil
+}
+
 func (s *SimpleDocker) DownloadImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePull, opts)

pkg/compiler/backend/simpleimg.go

@@ -35,6 +35,10 @@ func NewSimpleImgBackend(ctx *types.Context) *SimpleImg {
 	return &SimpleImg{ctx: ctx}
 }
 
+func (s *SimpleImg) LoadImage(string) error {
+	return errors.New("Not supported")
+}
+
 // TODO: Missing still: labels, and build args expansion
 func (s *SimpleImg) BuildImage(opts Options) error {
 	name := opts.ImageName

pkg/compiler/compiler.go

@@ -962,14 +962,14 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 
 	joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp)
 	cs.Options.Context.Info(joinTag, ":droplet: generating image from artifact", joinImageName)
-	opts, err := a.GenerateFinalImage(cs.Options.Context, joinImageName, cs.Backend, keepPermissions)
+	err = a.GenerateFinalImage(cs.Options.Context, joinImageName, cs.Backend, keepPermissions)
 	if err != nil {
 		return errors.Wrap(err, "could not create final image")
 	}
 	if cs.Options.Push {
 		cs.Options.Context.Info(joinTag, ":droplet: pushing image from artifact", joinImageName)
-		if err = cs.Backend.Push(opts); err != nil {
-			return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName)
+		if err = cs.Backend.Push(backend.Options{ImageName: joinImageName}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", joinImageName)
 		}
 	}
 	cs.Options.Context.Info(joinTag, ":droplet: Consuming image", joinImageName)

pkg/installer/client/http.go

@@ -118,11 +118,7 @@ func (c *HttpClient) DownloadFile(p string) (string, error) {
 		// Initialize a progressbar only if we have one in the current context
 		var pb *pterm.ProgressbarPrinter
 		if c.context.ProgressBar != nil {
-			pb = pterm.DefaultProgressbar.WithTotal(int(resp.Size()))
-			if c.context.AreaPrinter != nil {
-				pb = pb.WithPrintTogether(c.context.AreaPrinter)
-			}
-			pb, _ = pb.WithTitle(filepath.Base(resp.Request.HTTPRequest.URL.RequestURI())).Start()
+			pb, _ = c.context.ProgressBar.WithTotal(int(resp.Size())).WithTitle(filepath.Base(resp.Request.HTTPRequest.URL.RequestURI())).Start()
 		}
 		// start download loop
 		t := time.NewTicker(500 * time.Millisecond)

pkg/installer/installer.go

@@ -576,19 +576,19 @@ func (l *LuetInstaller) download(syncedRepos Repositories, toDownload map[string
 	// Check if the terminal is big enough to display a progress bar
 	// https://github.com/pterm/pterm/blob/4c725e56bfd9eb38e1c7b9dec187b50b93baa8bd/progressbar_printer.go#L190
 	w, _, err := ctx.GetTerminalSize()
+
+	var pb *pterm.ProgressbarPrinter
 	if ctx.IsTerminal && err == nil && w > 100 {
 		area, _ := pterm.DefaultArea.Start()
-		p, _ := pterm.DefaultProgressbar.WithPrintTogether(area).WithTotal(len(toDownload)).WithTitle("Downloading packages").Start()
-
-		ctx.AreaPrinter = area
-		ctx.ProgressBar = p
+		ctx.ProgressBar = pterm.DefaultProgressbar.WithPrintTogether(area).WithTotal(len(toDownload)).WithTitle("Downloading packages")
+		pb, _ = ctx.ProgressBar.Start()
 		defer area.Stop()
 	}
 
 	// Download
 	for i := 0; i < l.Options.Concurrency; i++ {
 		wg.Add(1)
-		go l.downloadWorker(i, wg, all, ctx)
+		go l.downloadWorker(i, wg, pb, all, ctx)
 	}
 	for _, c := range toDownload {
 		all <- c
@@ -920,7 +920,7 @@ func (l *LuetInstaller) installPackage(m ArtifactMatch, s *System) error {
 	return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: m.Package.GetFingerPrint(), Files: files})
 }
 
-func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, ctx *types.Context) error {
+func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, pb *pterm.ProgressbarPrinter, c <-chan ArtifactMatch, ctx *types.Context) error {
 	defer wg.Done()
 
 	for p := range c {
@@ -932,8 +932,8 @@ func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan Artif
 		} else {
 			l.Options.Context.Success(":package: Package ", p.Package.HumanReadableString(), "downloaded")
 		}
-		if ctx.ProgressBar != nil {
-			ctx.ProgressBar.Increment()
+		if pb != nil {
+			pb.Increment()
 		}
 	}
 

pkg/installer/repository_docker.go

@@ -92,8 +92,8 @@ func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDataba
 		} else {
 			l.context.Info("Generating final image", packageImage,
 				"for package ", a.CompileSpec.GetPackage().HumanReadableString())
-			if opts, err := a.GenerateFinalImage(l.context, packageImage, l.b, true); err != nil {
-				return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+			if err := a.GenerateFinalImage(l.context, packageImage, l.b, true); err != nil {
+				return errors.Wrap(err, "Failed generating metadata tree"+packageImage)
 			}
 		}
 		if l.imagePush {
@@ -125,8 +125,8 @@ func pushImage(ctx *types.Context, b compiler.CompilerBackend, image string, for
 
 func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string) error {
 	d.context.Debug("Generating image", imageTree)
-	if opts, err := a.GenerateFinalImage(d.context, imageTree, d.b, false); err != nil {
-		return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
+	if err := a.GenerateFinalImage(d.context, imageTree, d.b, false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree "+imageTree)
 	}
 	if d.imagePush {
 		if err := pushImage(d.context, d.b, imageTree, true); err != nil {

pkg/installer/repository_test.go

@@ -619,11 +619,8 @@ urls:
 
 			Expect(a.Unpack(ctx, extracted, false)).ToNot(HaveOccurred())
 
-			Expect(fileHelper.DirectoryIsEmpty(extracted)).To(BeFalse())
-			content, err := ioutil.ReadFile(filepath.Join(extracted, ".virtual"))
-			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.DirectoryIsEmpty(extracted)).To(BeTrue())
 
-			Expect(string(content)).To(Equal(""))
 		})
 
 		It("Searches files", func() {

tests/fixtures/caps/pkgC/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "empty"
+version: "0.1"

tests/fixtures/extra_perms/pkgA/0.1/build.yaml

@@ -0,0 +1,18 @@
+image: "alpine"
+unpack: true
+includes:
+  - /foo
+  - /foo/bar
+  - /foo/bar/suid
+  - /foo/bar/sticky
+  - /foo/bar/sgid
+steps:
+- mkdir -p /foo/bar
+- touch /foo/bar/suid
+- touch /foo/bar/sgid
+- touch /foo/bar/sticky
+- chown 100:100 /foo/bar
+- chown 101:101 /foo/bar/suid
+- chmod u+s /foo/bar/suid
+- chmod u-s,g+s /foo/bar/sgid
+- chmod +t /foo/bar/sticky

tests/fixtures/extra_perms/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "extra-perms"
+version: "0.1"

tests/integration/01_simple_docker.sh

@@ -49,8 +49,10 @@ testRepo() {
     --meta-compression zstd \
     --type docker --push-images --force-push)
 
+    echo "$createres"
+
     createst=$?
-    assertEquals 'create repo successfully' "$createst" "0"
+    assertEquals 'create repo successfully' "0" "$createst"
     assertContains 'contains image push' "$createres" 'Pushed image: quay.io/mocaccinoos/integration-test:z-test-1.0-2'
 }
 

tests/integration/35_caps_docker.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+
+    mkdir $tmpdir/testbuild
+    luet build -d --tree "$ROOT_DIR/tests/fixtures/caps" --same-owner=true --destination $tmpdir/testbuild --compression gzip --full
+    buildst=$?
+    assertTrue 'create package caps 0.1' "[ -e '$tmpdir/testbuild/caps-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/caps" \
+    --output $TEST_DOCKER_IMAGE \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --push-images --force-push \
+    --type docker
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+}
+
+testConfig() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "$TEST_DOCKER_IMAGE"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+
+    $ROOT_DIR/tests/integration/bin/luet install -y --config $tmpdir/luet.yaml test/caps@0.1 test/caps2@0.1 test/empty
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+   
+    assertTrue 'package installed file1' "[ -e '$tmpdir/testrootfs/file1' ]"
+    assertTrue 'package installed file2' "[ -e '$tmpdir/testrootfs/file2' ]"
+
+    assertContains 'caps' "$(getcap $tmpdir/testrootfs/file1)" "cap_net_raw+ep"
+    assertContains 'caps' "$(getcap $tmpdir/testrootfs/file2)" "cap_net_raw+ep"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/36_extra_perm_bits.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testbuild
+    luet build -d --tree "$ROOT_DIR/tests/fixtures/extra_perms" --same-owner=true --destination $tmpdir/testbuild --compression gzip --full
+    buildst=$?
+    assertTrue 'create package perms 0.1' "[ -e '$tmpdir/testbuild/extra-perms-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/extra_perms" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    $ROOT_DIR/tests/integration/bin/luet install -y --config $tmpdir/luet.yaml test/extra-perms
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+   
+    tree $tmpdir/testrootfs/foo/bar
+    assertTrue 'package installed bar' "[ -d '$tmpdir/testrootfs/foo/bar' ]"
+
+    assertContains 'perms2' "$(stat -c %u:%g $tmpdir/testrootfs/foo/bar)" "100:100"
+    assertContains 'suid' "$(stat -c %a $tmpdir/testrootfs/foo/bar/suid)" "4644"
+    assertContains 'sgid' "$(stat -c %a $tmpdir/testrootfs/foo/bar/sgid)" "2644"
+    assertContains 'sticky' "$(stat -c %a $tmpdir/testrootfs/foo/bar/sticky)" "1644"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+