Add crane-based methods for extraction

- create a new api package to encapsulate image manipulation
- use new api method to calculate delta

Fixes #258
Fixes #204
Fixes #90

pkg/api/core/image/delta.go

@@ -0,0 +1,111 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"archive/tar"
+	"io"
+	"regexp"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+)
+
+func compileRegexes(regexes []string) []*regexp.Regexp {
+	var result []*regexp.Regexp
+	for _, i := range regexes {
+		r, e := regexp.Compile(i)
+		if e != nil {
+			continue
+		}
+		result = append(result, r)
+	}
+	return result
+}
+
+type ImageDiffNode struct {
+	Name string `json:"Name"`
+	Size int    `json:"Size"`
+}
+type ImageDiff struct {
+	Additions []ImageDiffNode `json:"Adds"`
+	Deletions []ImageDiffNode `json:"Dels"`
+	Changes   []ImageDiffNode `json:"Mods"`
+}
+
+func Delta(srcimg, dstimg v1.Image) (res ImageDiff, err error) {
+	srcReader := mutate.Extract(srcimg)
+	defer srcReader.Close()
+
+	dstReader := mutate.Extract(dstimg)
+	defer dstReader.Close()
+
+	filesSrc, filesDst := map[string]int64{}, map[string]int64{}
+
+	srcTar := tar.NewReader(srcReader)
+	dstTar := tar.NewReader(dstReader)
+
+	for {
+		var hdr *tar.Header
+		hdr, err = srcTar.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return
+		}
+		filesSrc[hdr.Name] = hdr.Size
+	}
+
+	for {
+		var hdr *tar.Header
+		hdr, err = dstTar.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return
+		}
+		filesDst[hdr.Name] = hdr.Size
+	}
+	err = nil
+
+	for f, size := range filesDst {
+		if size2, exist := filesSrc[f]; exist && size2 != size {
+			res.Changes = append(res.Changes, ImageDiffNode{
+				Name: f,
+				Size: int(size),
+			})
+		} else if !exist {
+			res.Additions = append(res.Additions, ImageDiffNode{
+				Name: f,
+				Size: int(size),
+			})
+		}
+	}
+	for f, size := range filesSrc {
+		if _, exist := filesDst[f]; !exist {
+			res.Deletions = append(res.Deletions, ImageDiffNode{
+				Name: f,
+				Size: int(size),
+			})
+		}
+	}
+
+	return
+}

pkg/api/core/image/delta_test.go

@@ -0,0 +1,136 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	daemon "github.com/google/go-containerregistry/pkg/v1/daemon"
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("DeltaInfo", func() {
+	Context("Generates deltas of images", func() {
+		It("Correctly detect packages", func() {
+			ref, err := name.ParseReference("alpine")
+			Expect(err).ToNot(HaveOccurred())
+
+			img, err := daemon.Image(ref)
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := Delta(img, img)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers.Changes)).To(Equal(0))
+			Expect(len(layers.Additions)).To(Equal(0))
+			Expect(len(layers.Deletions)).To(Equal(0))
+		})
+
+		Context("ExtractDeltaFiles", func() {
+			ctx := types.NewContext()
+			var tmpfile *os.File
+			var ref, ref2 name.Reference
+			var img, img2 v1.Image
+			var diff ImageDiff
+			var err error
+
+			BeforeEach(func() {
+				ctx = types.NewContext()
+
+				tmpfile, err = ioutil.TempFile("", "delta")
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpfile.Name()) // clean up
+
+				ref, err = name.ParseReference("alpine")
+				Expect(err).ToNot(HaveOccurred())
+
+				ref2, err = name.ParseReference("golang:alpine")
+				Expect(err).ToNot(HaveOccurred())
+
+				img, err = daemon.Image(ref)
+				Expect(err).ToNot(HaveOccurred())
+				img2, err = daemon.Image(ref2)
+				Expect(err).ToNot(HaveOccurred())
+
+				diff, err = Delta(img, img2)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(len(diff.Additions) > 0).To(BeTrue())
+				Expect(len(diff.Changes) > 0).To(BeTrue())
+				Expect(len(diff.Deletions) > 0).To(BeTrue())
+			})
+
+			It("Extract all deltas", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img2,
+					ExtractDeltaFiles(ctx, diff, []string{}, []string{}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				fmt.Println(file.ListDir(tmpdir))
+				Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeTrue())
+			})
+
+			It("Extract deltas and excludes stuff", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img2,
+					ExtractDeltaFiles(ctx, diff, []string{}, []string{"usr/local/go"}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeFalse())
+			})
+			It("Extract deltas and excludes stuff", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img2,
+					ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{"usr/local/go/bin"}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeFalse())
+			})
+
+			It("Extract deltas and excludes stuff", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img2,
+					ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeFalse())
+			})
+		})
+	})
+})

pkg/api/core/image/extract.go

@@ -0,0 +1,119 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"archive/tar"
+	"context"
+
+	containerdarchive "github.com/containerd/containerd/archive"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/pkg/errors"
+)
+
+// Extract dir:
+// -> First extract delta considering the dir
+// Afterward create artifact pointing to the dir
+
+// ExtractDeltaFiles returns an handler to extract files in a list
+func ExtractDeltaFiles(
+	ctx *types.Context,
+	d ImageDiff,
+	includes []string, excludes []string,
+) func(h *tar.Header) (bool, error) {
+
+	includeRegexp := compileRegexes(includes)
+	excludeRegexp := compileRegexes(excludes)
+
+	return func(h *tar.Header) (bool, error) {
+		switch {
+		case len(includes) == 0 && len(excludes) != 0:
+			for _, a := range d.Additions {
+				if h.Name == a.Name {
+					for _, i := range excludeRegexp {
+						if i.MatchString(a.Name) && h.Name == a.Name {
+							return false, nil
+						}
+					}
+					ctx.Info("Adding name", h.Name)
+
+					return true, nil
+				}
+			}
+			return false, nil
+		case len(includes) > 0 && len(excludes) == 0:
+			for _, a := range d.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) && h.Name == a.Name {
+						ctx.Info("Adding name", h.Name)
+
+						return true, nil
+					}
+				}
+			}
+			return false, nil
+		case len(includes) != 0 && len(excludes) != 0:
+			for _, a := range d.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) && h.Name == a.Name {
+						for _, e := range excludeRegexp {
+							if e.MatchString(a.Name) {
+								return false, nil
+							}
+						}
+						ctx.Info("Adding name", h.Name)
+
+						return true, nil
+					}
+				}
+			}
+			return false, nil
+		default:
+			for _, a := range d.Additions {
+				if h.Name == a.Name {
+					ctx.Info("Adding name", h.Name)
+
+					return true, nil
+				}
+			}
+			return false, nil
+		}
+
+	}
+}
+
+func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) {
+	src := mutate.Extract(img)
+	defer src.Close()
+
+	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
+	if err != nil {
+		return "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+
+	if filter != nil {
+		opts = append(opts, containerdarchive.WithFilter(filter))
+	}
+
+	_, err = containerdarchive.Apply(context.Background(), tmpdiffs, src, opts...)
+	if err != nil {
+		return "", err
+	}
+
+	return tmpdiffs, nil
+}

pkg/api/core/image/mutator_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestMutator(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Mutator API Suite")
+}