Extract with new image API

cmd/cleanup.go

@@ -47,9 +47,7 @@ var cleanupCmd = &cobra.Command{
 
 			for _, file := range files {
 
-				if util.DefaultContext.Config.GetGeneral().Debug {
-					util.DefaultContext.Info("Removing ", file.Name())
-				}
+				util.DefaultContext.Debug("Removing ", file.Name())
 
 				err := os.RemoveAll(
 					filepath.Join(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))

pkg/api/core/image/extract.go

@@ -18,6 +18,9 @@ package image
 import (
 	"archive/tar"
 	"context"
+	"os"
+	"path/filepath"
+	"strings"
 
 	containerdarchive "github.com/containerd/containerd/archive"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -50,7 +53,7 @@ func ExtractDeltaFiles(
 							return false, nil
 						}
 					}
-					ctx.Info("Adding name", h.Name)
+					ctx.Debug("Adding name", h.Name)
 
 					return true, nil
 				}
@@ -60,7 +63,7 @@ func ExtractDeltaFiles(
 			for _, a := range d.Additions {
 				for _, i := range includeRegexp {
 					if i.MatchString(a.Name) && h.Name == a.Name {
-						ctx.Info("Adding name", h.Name)
+						ctx.Debug("Adding name", h.Name)
 
 						return true, nil
 					}
@@ -76,7 +79,7 @@ func ExtractDeltaFiles(
 								return false, nil
 							}
 						}
-						ctx.Info("Adding name", h.Name)
+						ctx.Debug("Adding name", h.Name)
 
 						return true, nil
 					}
@@ -86,7 +89,7 @@ func ExtractDeltaFiles(
 		default:
 			for _, a := range d.Additions {
 				if h.Name == a.Name {
-					ctx.Info("Adding name", h.Name)
+					ctx.Debug("Adding name", h.Name)
 
 					return true, nil
 				}
@@ -106,8 +109,14 @@ func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool,
 		return "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
 
+	perms := map[string][]int{}
+	f := func(h *tar.Header) (bool, error) {
+		perms[h.Name] = []int{h.Gid, h.Uid}
+		return filter(h)
+	}
+
 	if filter != nil {
-		opts = append(opts, containerdarchive.WithFilter(filter))
+		opts = append(opts, containerdarchive.WithFilter(f))
 	}
 
 	_, err = containerdarchive.Apply(context.Background(), tmpdiffs, src, opts...)
@@ -115,5 +124,76 @@ func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool,
 		return "", err
 	}
 
+	for f, p := range perms {
+		ff := filepath.Join(tmpdiffs, f)
+		if _, err := os.Lstat(ff); err == nil {
+			if err := os.Chown(ff, p[0], p[1]); err != nil {
+				ctx.Warning(err, "failed chowning file")
+			}
+		}
+	}
+
 	return tmpdiffs, nil
 }
+
+func ExtractFiles(
+	ctx *types.Context,
+	prefixPath string,
+	includes []string, excludes []string,
+) func(h *tar.Header) (bool, error) {
+	includeRegexp := compileRegexes(includes)
+	excludeRegexp := compileRegexes(excludes)
+
+	return func(h *tar.Header) (bool, error) {
+
+		switch {
+		case len(includes) == 0 && len(excludes) != 0:
+			for _, i := range excludeRegexp {
+				if i.MatchString(filepath.Join(prefixPath, h.Name)) {
+					return false, nil
+				}
+			}
+			if prefixPath != "" {
+				return strings.HasPrefix(h.Name, prefixPath), nil
+			}
+			ctx.Debug("Adding name", h.Name)
+			return true, nil
+
+		case len(includes) > 0 && len(excludes) == 0:
+			for _, i := range includeRegexp {
+				if i.MatchString(filepath.Join(prefixPath, h.Name)) {
+					if prefixPath != "" {
+						return strings.HasPrefix(h.Name, prefixPath), nil
+					}
+					ctx.Debug("Adding name", h.Name)
+
+					return true, nil
+				}
+			}
+			return false, nil
+		case len(includes) != 0 && len(excludes) != 0:
+			for _, i := range includeRegexp {
+				if i.MatchString(filepath.Join(prefixPath, h.Name)) {
+					for _, e := range excludeRegexp {
+						if e.MatchString(filepath.Join(prefixPath, h.Name)) {
+							return false, nil
+						}
+					}
+					if prefixPath != "" {
+						return strings.HasPrefix(h.Name, prefixPath), nil
+					}
+					ctx.Debug("Adding name", h.Name)
+
+					return true, nil
+				}
+			}
+			return false, nil
+		default:
+			if prefixPath != "" {
+				return strings.HasPrefix(h.Name, prefixPath), nil
+			}
+
+			return true, nil
+		}
+	}
+}

pkg/api/core/image/extract_test.go

@@ -0,0 +1,110 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	daemon "github.com/google/go-containerregistry/pkg/v1/daemon"
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Extract", func() {
+	Context("extract files from images", func() {
+		Context("ExtractFiles", func() {
+			ctx := types.NewContext()
+			var tmpfile *os.File
+			var ref name.Reference
+			var img v1.Image
+			var err error
+
+			BeforeEach(func() {
+				ctx = types.NewContext()
+
+				tmpfile, err = ioutil.TempFile("", "extract")
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpfile.Name()) // clean up
+
+				ref, err = name.ParseReference("alpine")
+				Expect(err).ToNot(HaveOccurred())
+
+				img, err = daemon.Image(ref)
+				Expect(err).ToNot(HaveOccurred())
+			})
+
+			It("Extract all files", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img,
+					ExtractFiles(ctx, "", []string{}, []string{}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "bin"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeTrue())
+			})
+
+			It("Extract specific dir", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img,
+					ExtractFiles(ctx, "usr", []string{}, []string{}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "sbin"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "bin"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
+			})
+
+			It("Extract a dir with includes/excludes", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img,
+					ExtractFiles(ctx, "usr", []string{"bin"}, []string{"sbin"}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "bin"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "sbin"))).To(BeFalse())
+			})
+
+			It("Extract with includes/excludes", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img,
+					ExtractFiles(ctx, "", []string{"usr", "usr/bin"}, []string{"^bin"}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "bin"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
+			})
+		})
+	})
+})

pkg/compiler/compiler.go

@@ -227,36 +227,37 @@ func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, includ
 }
 
 func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
-
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	img, err := cs.Backend.ImageReference(runnerOpts.ImageName)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not create tempdir")
+		return nil, err
+	}
+
+	// TODO: Trim includes/excludes from "/" ?
+	rootfs, err := image.Extract(
+		cs.Options.Context,
+		img,
+		image.ExtractFiles(
+			cs.Options.Context,
+			strings.TrimLeft(p.GetPackageDir(), "/"),
+			p.GetIncludes(),
+			p.GetExcludes(),
+		),
+	)
+	if err != nil {
+		return nil, err
 	}
 	defer os.RemoveAll(rootfs) // clean up
 
-	err = cs.Backend.ExtractRootfs(backend.Options{
-		ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions)
-	if err != nil {
-		return nil, errors.Wrap(err, "Could not extract rootfs")
+	toUnpack := rootfs
+
+	if p.PackageDir != "" {
+		toUnpack = filepath.Join(toUnpack, p.PackageDir)
 	}
 
-	if p.GetPackageDir() != "" {
-		cs.Options.Context.Info(":tophat: Packing from output dir", p.GetPackageDir())
-		rootfs = filepath.Join(rootfs, p.GetPackageDir())
-	}
-
-	if len(p.GetIncludes()) > 0 {
-		// strip from includes
-		cs.stripFromRootfs(p.GetIncludes(), rootfs, true)
-	}
-	if len(p.GetExcludes()) > 0 {
-		// strip from excludes
-		cs.stripFromRootfs(p.GetExcludes(), rootfs, false)
-	}
 	a := artifact.NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 	a.CompressionType = cs.Options.CompressionType
 
-	if err := a.Compress(rootfs, concurrency); err != nil {
+	if err := a.Compress(toUnpack, concurrency); err != nil {
 		return nil, errors.Wrap(err, "Error met while creating package archive")
 	}