tests: pull image before running

pkg/api/core/image/delta_test.go

@@ -16,7 +16,6 @@
 package image_test
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -31,9 +30,9 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("DeltaInfo", func() {
+var _ = Describe("Delta", func() {
 	Context("Generates deltas of images", func() {
-		It("Correctly detect packages", func() {
+		It("computes delta", func() {
 			ref, err := name.ParseReference("alpine")
 			Expect(err).ToNot(HaveOccurred())
 
@@ -78,7 +77,7 @@ var _ = Describe("DeltaInfo", func() {
 
 				Expect(len(diff.Additions) > 0).To(BeTrue())
 				Expect(len(diff.Changes) > 0).To(BeTrue())
-				Expect(len(diff.Deletions) > 0).To(BeTrue())
+				Expect(len(diff.Deletions) == 0).To(BeTrue())
 			})
 
 			It("Extract all deltas", func() {
@@ -90,14 +89,13 @@ var _ = Describe("DeltaInfo", func() {
 				Expect(err).ToNot(HaveOccurred())
 				defer os.RemoveAll(tmpdir) // clean up
 
-				fmt.Println(file.ListDir(tmpdir))
 				Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeTrue())
 				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
 				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
 				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeTrue())
 			})
 
-			It("Extract deltas and excludes stuff", func() {
+			It("Extract deltas and excludes /usr/local/go", func() {
 				tmpdir, err := Extract(
 					ctx,
 					img2,
@@ -107,7 +105,7 @@ var _ = Describe("DeltaInfo", func() {
 				defer os.RemoveAll(tmpdir) // clean up
 				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeFalse())
 			})
-			It("Extract deltas and excludes stuff", func() {
+			It("Extract deltas and excludes /usr/local/go/bin, but includes /usr/local/go", func() {
 				tmpdir, err := Extract(
 					ctx,
 					img2,
@@ -119,7 +117,7 @@ var _ = Describe("DeltaInfo", func() {
 				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeFalse())
 			})
 
-			It("Extract deltas and excludes stuff", func() {
+			It("Extract deltas and includes /usr/local/go", func() {
 				tmpdir, err := Extract(
 					ctx,
 					img2,

pkg/api/core/image/extract.go

@@ -18,11 +18,14 @@ package image
 import (
 	"archive/tar"
 	"context"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	containerdarchive "github.com/containerd/containerd/archive"
+	"github.com/docker/docker/pkg/system"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/mudler/luet/pkg/api/core/types"
@@ -100,42 +103,6 @@ func ExtractDeltaFiles(
 	}
 }
 
-func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) {
-	src := mutate.Extract(img)
-	defer src.Close()
-
-	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
-	if err != nil {
-		return "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-
-	perms := map[string][]int{}
-	f := func(h *tar.Header) (bool, error) {
-		perms[h.Name] = []int{h.Gid, h.Uid}
-		return filter(h)
-	}
-
-	if filter != nil {
-		opts = append(opts, containerdarchive.WithFilter(f))
-	}
-
-	_, err = containerdarchive.Apply(context.Background(), tmpdiffs, src, opts...)
-	if err != nil {
-		return "", err
-	}
-
-	for f, p := range perms {
-		ff := filepath.Join(tmpdiffs, f)
-		if _, err := os.Lstat(ff); err == nil {
-			if err := os.Chown(ff, p[0], p[1]); err != nil {
-				ctx.Warning(err, "failed chowning file")
-			}
-		}
-	}
-
-	return tmpdiffs, nil
-}
-
 func ExtractFiles(
 	ctx *types.Context,
 	prefixPath string,
@@ -164,7 +131,7 @@ func ExtractFiles(
 			for _, i := range includeRegexp {
 				if i.MatchString(filepath.Join(prefixPath, fileName)) {
 					if prefixPath != "" {
-						return strings.HasPrefix(h.Name, prefixPath), nil
+						return strings.HasPrefix(fileName, prefixPath), nil
 					}
 					ctx.Debug("Adding name", fileName)
 
@@ -198,3 +165,64 @@ func ExtractFiles(
 		}
 	}
 }
+
+func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) {
+	defer reader.Close()
+
+	perms := map[string][]int{}
+	xattrs := map[string]map[string]string{}
+	paxrecords := map[string]map[string]string{}
+
+	f := func(h *tar.Header) (bool, error) {
+		perms[h.Name] = []int{h.Gid, h.Uid}
+		xattrs[h.Name] = h.Xattrs
+		paxrecords[h.Name] = h.PAXRecords
+
+		return filter(h)
+	}
+
+	if filter != nil {
+		opts = append(opts, containerdarchive.WithFilter(f))
+	}
+
+	_, err := containerdarchive.Apply(context.Background(), output, reader, opts...)
+	if err != nil {
+		return "", err
+	}
+
+	for f, p := range perms {
+		ff := filepath.Join(output, f)
+		if _, err := os.Lstat(ff); err == nil {
+			if err := os.Lchown(ff, p[1], p[0]); err != nil {
+				ctx.Warning(err, "failed chowning file")
+			}
+		}
+	}
+
+	for _, m := range []map[string]map[string]string{xattrs, paxrecords} {
+		for key, attrs := range m {
+			ff := filepath.Join(output, key)
+			for k, attr := range attrs {
+				if err := system.Lsetxattr(ff, k, []byte(attr), 0); err != nil {
+					if errors.Is(err, syscall.ENOTSUP) {
+						ctx.Debug("ignored xattr %s in archive", key)
+					}
+				}
+			}
+		}
+	}
+
+	return output, nil
+}
+
+func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) {
+	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
+	if err != nil {
+		return "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, filter, opts...)
+}
+
+func ExtractTo(ctx *types.Context, img v1.Image, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) {
+	return ExtractReader(ctx, mutate.Extract(img), output, filter, opts...)
+}

pkg/api/core/image/extract_test.go

@@ -31,6 +31,7 @@ import (
 )
 
 var _ = Describe("Extract", func() {
+
 	Context("extract files from images", func() {
 		Context("ExtractFiles", func() {
 			ctx := types.NewContext()
@@ -70,7 +71,7 @@ var _ = Describe("Extract", func() {
 				tmpdir, err := Extract(
 					ctx,
 					img,
-					ExtractFiles(ctx, "usr", []string{}, []string{}),
+					ExtractFiles(ctx, "/usr", []string{}, []string{}),
 				)
 				Expect(err).ToNot(HaveOccurred())
 				defer os.RemoveAll(tmpdir) // clean up
@@ -83,7 +84,7 @@ var _ = Describe("Extract", func() {
 				tmpdir, err := Extract(
 					ctx,
 					img,
-					ExtractFiles(ctx, "usr", []string{"bin"}, []string{"sbin"}),
+					ExtractFiles(ctx, "/usr", []string{"bin"}, []string{"sbin"}),
 				)
 				Expect(err).ToNot(HaveOccurred())
 				defer os.RemoveAll(tmpdir) // clean up
@@ -97,7 +98,7 @@ var _ = Describe("Extract", func() {
 				tmpdir, err := Extract(
 					ctx,
 					img,
-					ExtractFiles(ctx, "", []string{"usr", "usr/bin"}, []string{"^bin"}),
+					ExtractFiles(ctx, "", []string{"/usr|/usr/bin"}, []string{"^/bin"}),
 				)
 				Expect(err).ToNot(HaveOccurred())
 				defer os.RemoveAll(tmpdir) // clean up

pkg/api/core/image/mutator_suite_test.go

@@ -18,11 +18,17 @@ package image_test
 import (
 	"testing"
 
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestMutator(t *testing.T) {
 	RegisterFailHandler(Fail)
+	b := backend.NewSimpleDockerBackend(types.NewContext())
+	b.DownloadImage(backend.Options{ImageName: "alpine"})
+	b.DownloadImage(backend.Options{ImageName: "golang:alpine"})
+
 	RunSpecs(t, "Mutator API Suite")
 }

pkg/compiler/compiler.go

@@ -238,7 +238,7 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
 		img,
 		image.ExtractFiles(
 			cs.Options.Context,
-			strings.TrimLeft(p.GetPackageDir(), "/"),
+			p.GetPackageDir(),
 			p.GetIncludes(),
 			p.GetExcludes(),
 		),

tests/integration/09_docker.sh

@@ -12,7 +12,7 @@ oneTimeTearDown() {
 
 testBuild() {
     mkdir $tmpdir/testbuild
-    luet build --tree "$ROOT_DIR/tests/fixtures/docker" --destination $tmpdir/testbuild --compression gzip --all > /dev/null
+    luet build --tree "$ROOT_DIR/tests/fixtures/docker" --destination $tmpdir/testbuild --compression gzip --all
     buildst=$?
     assertEquals 'builds successfully' "$buildst" "0"
     assertTrue 'create package' "[ -e '$tmpdir/testbuild/alpine-seed-1.0.package.tar.gz' ]"