Add crane-based methods for extraction

- create a new api package to encapsulate image manipulation
- use new api method to calculate delta

Fixes #258
Fixes #204
Fixes #90

pkg/api/core/image/delta.go

@@ -0,0 +1,111 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"archive/tar"
+	"io"
+	"regexp"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+)
+
+func compileRegexes(regexes []string) []*regexp.Regexp {
+	var result []*regexp.Regexp
+	for _, i := range regexes {
+		r, e := regexp.Compile(i)
+		if e != nil {
+			continue
+		}
+		result = append(result, r)
+	}
+	return result
+}
+
+type ImageDiffNode struct {
+	Name string `json:"Name"`
+	Size int    `json:"Size"`
+}
+type ImageDiff struct {
+	Additions []ImageDiffNode `json:"Adds"`
+	Deletions []ImageDiffNode `json:"Dels"`
+	Changes   []ImageDiffNode `json:"Mods"`
+}
+
+func Delta(srcimg, dstimg v1.Image) (res ImageDiff, err error) {
+	srcReader := mutate.Extract(srcimg)
+	defer srcReader.Close()
+
+	dstReader := mutate.Extract(dstimg)
+	defer dstReader.Close()
+
+	filesSrc, filesDst := map[string]int64{}, map[string]int64{}
+
+	srcTar := tar.NewReader(srcReader)
+	dstTar := tar.NewReader(dstReader)
+
+	for {
+		var hdr *tar.Header
+		hdr, err = srcTar.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return
+		}
+		filesSrc[hdr.Name] = hdr.Size
+	}
+
+	for {
+		var hdr *tar.Header
+		hdr, err = dstTar.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return
+		}
+		filesDst[hdr.Name] = hdr.Size
+	}
+	err = nil
+
+	for f, size := range filesDst {
+		if size2, exist := filesSrc[f]; exist && size2 != size {
+			res.Changes = append(res.Changes, ImageDiffNode{
+				Name: f,
+				Size: int(size),
+			})
+		} else if !exist {
+			res.Additions = append(res.Additions, ImageDiffNode{
+				Name: f,
+				Size: int(size),
+			})
+		}
+	}
+	for f, size := range filesSrc {
+		if _, exist := filesDst[f]; !exist {
+			res.Deletions = append(res.Deletions, ImageDiffNode{
+				Name: f,
+				Size: int(size),
+			})
+		}
+	}
+
+	return
+}

pkg/api/core/image/delta_test.go

@@ -0,0 +1,136 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	daemon "github.com/google/go-containerregistry/pkg/v1/daemon"
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("DeltaInfo", func() {
+	Context("Generates deltas of images", func() {
+		It("Correctly detect packages", func() {
+			ref, err := name.ParseReference("alpine")
+			Expect(err).ToNot(HaveOccurred())
+
+			img, err := daemon.Image(ref)
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := Delta(img, img)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers.Changes)).To(Equal(0))
+			Expect(len(layers.Additions)).To(Equal(0))
+			Expect(len(layers.Deletions)).To(Equal(0))
+		})
+
+		Context("ExtractDeltaFiles", func() {
+			ctx := types.NewContext()
+			var tmpfile *os.File
+			var ref, ref2 name.Reference
+			var img, img2 v1.Image
+			var diff ImageDiff
+			var err error
+
+			BeforeEach(func() {
+				ctx = types.NewContext()
+
+				tmpfile, err = ioutil.TempFile("", "delta")
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpfile.Name()) // clean up
+
+				ref, err = name.ParseReference("alpine")
+				Expect(err).ToNot(HaveOccurred())
+
+				ref2, err = name.ParseReference("golang:alpine")
+				Expect(err).ToNot(HaveOccurred())
+
+				img, err = daemon.Image(ref)
+				Expect(err).ToNot(HaveOccurred())
+				img2, err = daemon.Image(ref2)
+				Expect(err).ToNot(HaveOccurred())
+
+				diff, err = Delta(img, img2)
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(len(diff.Additions) > 0).To(BeTrue())
+				Expect(len(diff.Changes) > 0).To(BeTrue())
+				Expect(len(diff.Deletions) > 0).To(BeTrue())
+			})
+
+			It("Extract all deltas", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img2,
+					ExtractDeltaFiles(ctx, diff, []string{}, []string{}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				fmt.Println(file.ListDir(tmpdir))
+				Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeTrue())
+			})
+
+			It("Extract deltas and excludes stuff", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img2,
+					ExtractDeltaFiles(ctx, diff, []string{}, []string{"usr/local/go"}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeFalse())
+			})
+			It("Extract deltas and excludes stuff", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img2,
+					ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{"usr/local/go/bin"}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeFalse())
+			})
+
+			It("Extract deltas and excludes stuff", func() {
+				tmpdir, err := Extract(
+					ctx,
+					img2,
+					ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{}),
+				)
+				Expect(err).ToNot(HaveOccurred())
+				defer os.RemoveAll(tmpdir) // clean up
+
+				Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
+				Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeFalse())
+			})
+		})
+	})
+})

pkg/api/core/image/extract.go

@@ -0,0 +1,119 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"archive/tar"
+	"context"
+
+	containerdarchive "github.com/containerd/containerd/archive"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/pkg/errors"
+)
+
+// Extract dir:
+// -> First extract delta considering the dir
+// Afterward create artifact pointing to the dir
+
+// ExtractDeltaFiles returns an handler to extract files in a list
+func ExtractDeltaFiles(
+	ctx *types.Context,
+	d ImageDiff,
+	includes []string, excludes []string,
+) func(h *tar.Header) (bool, error) {
+
+	includeRegexp := compileRegexes(includes)
+	excludeRegexp := compileRegexes(excludes)
+
+	return func(h *tar.Header) (bool, error) {
+		switch {
+		case len(includes) == 0 && len(excludes) != 0:
+			for _, a := range d.Additions {
+				if h.Name == a.Name {
+					for _, i := range excludeRegexp {
+						if i.MatchString(a.Name) && h.Name == a.Name {
+							return false, nil
+						}
+					}
+					ctx.Info("Adding name", h.Name)
+
+					return true, nil
+				}
+			}
+			return false, nil
+		case len(includes) > 0 && len(excludes) == 0:
+			for _, a := range d.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) && h.Name == a.Name {
+						ctx.Info("Adding name", h.Name)
+
+						return true, nil
+					}
+				}
+			}
+			return false, nil
+		case len(includes) != 0 && len(excludes) != 0:
+			for _, a := range d.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) && h.Name == a.Name {
+						for _, e := range excludeRegexp {
+							if e.MatchString(a.Name) {
+								return false, nil
+							}
+						}
+						ctx.Info("Adding name", h.Name)
+
+						return true, nil
+					}
+				}
+			}
+			return false, nil
+		default:
+			for _, a := range d.Additions {
+				if h.Name == a.Name {
+					ctx.Info("Adding name", h.Name)
+
+					return true, nil
+				}
+			}
+			return false, nil
+		}
+
+	}
+}
+
+func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) {
+	src := mutate.Extract(img)
+	defer src.Close()
+
+	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
+	if err != nil {
+		return "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+
+	if filter != nil {
+		opts = append(opts, containerdarchive.WithFilter(filter))
+	}
+
+	_, err = containerdarchive.Apply(context.Background(), tmpdiffs, src, opts...)
+	if err != nil {
+		return "", err
+	}
+
+	return tmpdiffs, nil
+}

pkg/api/core/image/mutator_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestMutator(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Mutator API Suite")
+}

pkg/api/core/types/artifact/artifact.go

@@ -27,16 +27,16 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
 
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	zstd "github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
 	"strings"
-	"sync"
 
 	config "github.com/mudler/luet/pkg/api/core/config"
+	"github.com/mudler/luet/pkg/api/core/image"
 	types "github.com/mudler/luet/pkg/api/core/types"
 	bus "github.com/mudler/luet/pkg/bus"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
@@ -67,6 +67,22 @@ type PackageArtifact struct {
 	Runtime           *pkg.DefaultPackage               `json:"runtime,omitempty"`
 }
 
+func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
+	tmpdiffs, err := image.Extract(ctx, img, filter)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	a := NewPackageArtifact(output)
+	a.CompressionType = t
+	err = a.Compress(tmpdiffs, 1)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while creating package archive")
+	}
+	return a, nil
+}
+
 func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
 	copy := *p
 	return &copy
@@ -626,181 +642,3 @@ func (a *PackageArtifact) FileList() ([]string, error) {
 	}
 	return files, nil
 }
-
-type CopyJob struct {
-	Src, Dst string
-	Artifact string
-}
-
-func worker(ctx *types.Context, i int, wg *sync.WaitGroup, s <-chan CopyJob) {
-	defer wg.Done()
-
-	for job := range s {
-		_, err := os.Lstat(job.Dst)
-		if err != nil {
-			ctx.Debug("Copying ", job.Src)
-			if err := fileHelper.DeepCopyFile(job.Src, job.Dst); err != nil {
-				ctx.Warning("Error copying", job, err)
-			}
-		}
-	}
-}
-
-func compileRegexes(regexes []string) []*regexp.Regexp {
-	var result []*regexp.Regexp
-	for _, i := range regexes {
-		r, e := regexp.Compile(i)
-		if e != nil {
-			continue
-		}
-		result = append(result, r)
-	}
-	return result
-}
-
-type ArtifactNode struct {
-	Name string `json:"Name"`
-	Size int    `json:"Size"`
-}
-type ArtifactDiffs struct {
-	Additions []ArtifactNode `json:"Adds"`
-	Deletions []ArtifactNode `json:"Dels"`
-	Changes   []ArtifactNode `json:"Mods"`
-}
-
-type ArtifactLayer struct {
-	FromImage string        `json:"Image1"`
-	ToImage   string        `json:"Image2"`
-	Diffs     ArtifactDiffs `json:"Diff"`
-}
-
-// ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(ctx *types.Context, src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) {
-
-	archive, err := ctx.Config.GetSystem().TempDir("archive")
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for archive")
-	}
-	defer os.RemoveAll(archive) // clean up
-
-	if strings.HasSuffix(src, ".tar") {
-		rootfs, err := ctx.Config.GetSystem().TempDir("rootfs")
-		if err != nil {
-			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-		}
-		defer os.RemoveAll(rootfs) // clean up
-		err = helpers.Untar(src, rootfs, keepPerms)
-		if err != nil {
-			return nil, errors.Wrap(err, "Error met while unpacking rootfs")
-		}
-		src = rootfs
-	}
-
-	toCopy := make(chan CopyJob)
-
-	var wg = new(sync.WaitGroup)
-	for i := 0; i < concurrency; i++ {
-		wg.Add(1)
-		go worker(ctx, i, wg, toCopy)
-	}
-
-	// Handle includes in spec. If specified they filter what gets in the package
-
-	if len(includes) > 0 && len(excludes) == 0 {
-		includeRegexp := compileRegexes(includes)
-		for _, l := range layers {
-			// Consider d.Additions (and d.Changes? - warn at least) only
-		ADDS:
-			for _, a := range l.Diffs.Additions {
-				for _, i := range includeRegexp {
-					if i.MatchString(a.Name) {
-						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
-						continue ADDS
-					}
-				}
-			}
-			for _, a := range l.Diffs.Changes {
-				ctx.Debug("File ", a.Name, " changed")
-			}
-			for _, a := range l.Diffs.Deletions {
-				ctx.Debug("File ", a.Name, " deleted")
-			}
-		}
-
-	} else if len(includes) == 0 && len(excludes) != 0 {
-		excludeRegexp := compileRegexes(excludes)
-		for _, l := range layers {
-			// Consider d.Additions (and d.Changes? - warn at least) only
-		ADD:
-			for _, a := range l.Diffs.Additions {
-				for _, i := range excludeRegexp {
-					if i.MatchString(a.Name) {
-						continue ADD
-					}
-				}
-				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
-			}
-			for _, a := range l.Diffs.Changes {
-				ctx.Debug("File ", a.Name, " changed")
-			}
-			for _, a := range l.Diffs.Deletions {
-				ctx.Debug("File ", a.Name, " deleted")
-			}
-		}
-
-	} else if len(includes) != 0 && len(excludes) != 0 {
-		includeRegexp := compileRegexes(includes)
-		excludeRegexp := compileRegexes(excludes)
-
-		for _, l := range layers {
-			// Consider d.Additions (and d.Changes? - warn at least) only
-		EXCLUDES:
-			for _, a := range l.Diffs.Additions {
-				for _, i := range includeRegexp {
-					if i.MatchString(a.Name) {
-						for _, e := range excludeRegexp {
-							if e.MatchString(a.Name) {
-								continue EXCLUDES
-							}
-						}
-						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
-						continue EXCLUDES
-					}
-				}
-			}
-			for _, a := range l.Diffs.Changes {
-				ctx.Debug("File ", a.Name, " changed")
-			}
-			for _, a := range l.Diffs.Deletions {
-				ctx.Debug("File ", a.Name, " deleted")
-			}
-		}
-
-	} else {
-		// Otherwise just grab all
-		for _, l := range layers {
-			// Consider d.Additions (and d.Changes? - warn at least) only
-			for _, a := range l.Diffs.Additions {
-				ctx.Debug("File ", a.Name, " added")
-				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
-			}
-			for _, a := range l.Diffs.Changes {
-				ctx.Debug("File ", a.Name, " changed")
-			}
-			for _, a := range l.Diffs.Deletions {
-				ctx.Debug("File ", a.Name, " deleted")
-			}
-		}
-	}
-
-	close(toCopy)
-	wg.Wait()
-
-	a := NewPackageArtifact(dst)
-	a.CompressionType = t
-	err = a.Compress(archive, concurrency)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating package archive")
-	}
-	return a, nil
-}

pkg/api/core/types/artifact/artifact_test.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/mudler/luet/pkg/api/core/types"
 	. "github.com/mudler/luet/pkg/api/core/types/artifact"
-	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler/backend"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
 	compression "github.com/mudler/luet/pkg/compiler/types/compression"
@@ -30,7 +29,6 @@ import (
 	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 
 	. "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
@@ -117,53 +115,7 @@ RUN echo bar > /test2`))
 			}
 			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
-			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
-			diffs, err := compiler.GenerateChanges(ctx, b, opts, opts2)
-			Expect(err).ToNot(HaveOccurred())
 
-			artifacts := []ArtifactNode{{
-				Name: "/luetbuild/LuetDockerfile",
-				Size: 175,
-			}}
-			if os.Getenv("DOCKER_BUILDKIT") == "1" {
-				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
-			}
-			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
-			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
-
-			Expect(diffs).To(Equal(
-				[]ArtifactLayer{{
-					FromImage: "luet/base",
-					ToImage:   "test",
-					Diffs: ArtifactDiffs{
-						Additions: artifacts,
-					},
-				}}))
-			err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false)
-			Expect(err).ToNot(HaveOccurred())
-
-			a, err := ExtractArtifactFromDelta(ctx, rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(fileHelper.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
-			err = helpers.Untar(a.Path, unpacked, false)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(fileHelper.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
-			Expect(fileHelper.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
-			content1, err := fileHelper.Read(filepath.Join(unpacked, "test"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(content1).To(Equal("foo\n"))
-			content2, err := fileHelper.Read(filepath.Join(unpacked, "test2"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(content2).To(Equal("bar\n"))
-
-			err = a.Hash()
-			Expect(err).ToNot(HaveOccurred())
-			err = a.Verify()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(fileHelper.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
-
-			err = a.Verify()
-			Expect(err).To(HaveOccurred())
 		})
 
 		It("Generates packages images", func() {

pkg/compiler/backend.go

@@ -1,14 +1,8 @@
 package compiler
 
 import (
-	"fmt"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-
 	"github.com/mudler/luet/pkg/api/core/types"
-	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/pkg/errors"
@@ -42,204 +36,6 @@ type CompilerBackend interface {
 	Push(opts backend.Options) error
 	ImageAvailable(string) bool
 
+	ImageReference(img1 string) (v1.Image, error)
 	ImageExists(string) bool
 }
-
-// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
-// example of json return: [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-func GenerateChanges(ctx *types.Context, b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) {
-
-	res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
-
-	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-
-	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(srcRootFS) // clean up
-
-	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(dstRootFS) // clean up
-
-	srcImageExtract := backend.Options{
-		ImageName:   fromImage.ImageName,
-		Destination: srcRootFS,
-	}
-	ctx.Debug("Extracting source image", fromImage.ImageName)
-	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
-	}
-
-	dstImageExtract := backend.Options{
-		ImageName:   toImage.ImageName,
-		Destination: dstRootFS,
-	}
-	ctx.Debug("Extracting destination image", toImage.ImageName)
-	err = b.ExtractRootfs(dstImageExtract, false)
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
-	}
-
-	// Get Additions/Changes. dst -> src
-	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
-		if info.IsDir() {
-			return nil
-		}
-
-		realpath := strings.Replace(path, dstRootFS, "", -1)
-		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
-		if err == nil {
-			var sizeA, sizeB int64
-			sizeA = fileInfo.Size()
-
-			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
-				sizeB = s.Size()
-			}
-
-			if sizeA != sizeB {
-				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
-				res.Diffs.Changes = append(res.Diffs.Changes, artifact.ArtifactNode{
-					Name: filepath.Join("/", realpath),
-					Size: int(sizeB),
-				})
-			} else {
-				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
-			}
-		} else {
-			var sizeB int64
-
-			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
-				sizeB = s.Size()
-			}
-			res.Diffs.Additions = append(res.Diffs.Additions, artifact.ArtifactNode{
-				Name: filepath.Join("/", realpath),
-				Size: int(sizeB),
-			})
-
-			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
-		}
-
-		return nil
-	})
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
-	}
-
-	// Get deletions. src -> dst
-	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
-		if info.IsDir() {
-			return nil
-		}
-
-		realpath := strings.Replace(path, srcRootFS, "", -1)
-		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
-			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
-			res.Diffs.Deletions = append(res.Diffs.Deletions, artifact.ArtifactNode{
-				Name: filepath.Join("/", realpath),
-			})
-		}
-
-		return nil
-	})
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
-	}
-
-	diffs := []artifact.ArtifactLayer{res}
-
-	if ctx.Config.GetGeneral().Debug {
-		summary := ComputeArtifactLayerSummary(diffs)
-		for _, l := range summary.Layers {
-			ctx.Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
-				l.FromImage, l.ToImage,
-				l.AddFiles, l.AddSizes,
-				l.DelFiles, l.DelSizes,
-				l.ChangeFiles, l.ChangeSizes))
-		}
-	}
-
-	return diffs, nil
-}
-
-type ArtifactLayerSummary struct {
-	FromImage   string `json:"image1"`
-	ToImage     string `json:"image2"`
-	AddFiles    int    `json:"add_files"`
-	AddSizes    int64  `json:"add_sizes"`
-	DelFiles    int    `json:"del_files"`
-	DelSizes    int64  `json:"del_sizes"`
-	ChangeFiles int    `json:"change_files"`
-	ChangeSizes int64  `json:"change_sizes"`
-}
-
-type ArtifactLayersSummary struct {
-	Layers []ArtifactLayerSummary `json:"summary"`
-}
-
-func ComputeArtifactLayerSummary(diffs []artifact.ArtifactLayer) ArtifactLayersSummary {
-
-	ans := ArtifactLayersSummary{
-		Layers: make([]ArtifactLayerSummary, 0),
-	}
-
-	for _, layer := range diffs {
-		sum := ArtifactLayerSummary{
-			FromImage:   layer.FromImage,
-			ToImage:     layer.ToImage,
-			AddFiles:    0,
-			AddSizes:    0,
-			DelFiles:    0,
-			DelSizes:    0,
-			ChangeFiles: 0,
-			ChangeSizes: 0,
-		}
-		for _, a := range layer.Diffs.Additions {
-			sum.AddFiles++
-			sum.AddSizes += int64(a.Size)
-		}
-		for _, d := range layer.Diffs.Deletions {
-			sum.DelFiles++
-			sum.DelSizes += int64(d.Size)
-		}
-		for _, c := range layer.Diffs.Changes {
-			sum.ChangeFiles++
-			sum.ChangeSizes += int64(c.Size)
-		}
-		ans.Layers = append(ans.Layers, sum)
-	}
-
-	return ans
-}

pkg/compiler/backend/simpledocker.go

@@ -23,10 +23,13 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/daemon"
 	"github.com/mudler/luet/pkg/api/core/types"
 	bus "github.com/mudler/luet/pkg/bus"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	capi "github.com/mudler/docker-companion/api"
 
 	"github.com/mudler/luet/pkg/helpers"
@@ -144,6 +147,19 @@ func (s *SimpleDocker) Push(opts Options) error {
 	return nil
 }
 
+func (s *SimpleDocker) ImageReference(a string) (v1.Image, error) {
+	ref, err := name.ParseReference(a)
+	if err != nil {
+		return nil, err
+	}
+
+	img, err := daemon.Image(ref)
+	if err != nil {
+		return nil, err
+	}
+	return img, nil
+}
+
 func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")

pkg/compiler/backend/simpledocker_test.go

@@ -17,8 +17,6 @@ package backend_test
 
 import (
 	"github.com/mudler/luet/pkg/api/core/types"
-	"github.com/mudler/luet/pkg/api/core/types/artifact"
-	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
@@ -107,35 +105,6 @@ RUN echo bar > /test2`))
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
-			artifacts := []artifact.ArtifactNode{{
-				Name: "/luetbuild/LuetDockerfile",
-				Size: 175,
-			}}
-			if os.Getenv("DOCKER_BUILDKIT") == "1" {
-				artifacts = append(artifacts, artifact.ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
-			}
-			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4})
-			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4})
-
-			Expect(compiler.GenerateChanges(ctx, b, opts, opts2)).To(Equal(
-				[]artifact.ArtifactLayer{{
-					FromImage: "luet/base",
-					ToImage:   "test",
-					Diffs: artifact.ArtifactDiffs{
-						Additions: artifacts,
-					},
-				}}))
-
-			opts2 = backend.Options{
-				ImageName:      "test",
-				SourcePath:     tmpdir,
-				DockerFileName: "LuetDockerfile",
-				Destination:    filepath.Join(tmpdir, "output3.tar"),
-			}
-
-			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
-			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
-			Expect(b.ImageExists(opts2.ImageName)).To(BeFalse())
 		})
 
 		It("Detects available images", func() {

pkg/compiler/backend/simpleimg.go

@@ -20,6 +20,8 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/google/go-containerregistry/pkg/crane"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/mudler/luet/pkg/api/core/types"
 	bus "github.com/mudler/luet/pkg/bus"
 
@@ -70,6 +72,29 @@ func (s *SimpleImg) RemoveImage(opts Options) error {
 	return nil
 }
 
+func (s *SimpleImg) ImageReference(a string) (v1.Image, error) {
+
+	f, err := s.ctx.Config.GetSystem().TempFile("snapshot")
+	if err != nil {
+		return nil, err
+	}
+	buildarg := []string{"save", a, "-o", f.Name()}
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
+
+	out, err := exec.Command("img", buildarg...).CombinedOutput()
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed saving image: "+string(out))
+	}
+
+	img, err := crane.Load(f.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	return img, nil
+}
+
 func (s *SimpleImg) DownloadImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePull, opts)

pkg/compiler/backend_test.go

@@ -1,79 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler_test
-
-import (
-	"github.com/mudler/luet/pkg/api/core/types"
-	. "github.com/mudler/luet/pkg/compiler"
-	. "github.com/mudler/luet/pkg/compiler/backend"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Docker image diffs", func() {
-	var b CompilerBackend
-	ctx := types.NewContext()
-	BeforeEach(func() {
-		b = NewSimpleDockerBackend(ctx)
-	})
-
-	Context("Generate diffs from docker images", func() {
-		It("Detect no changes", func() {
-			opts := Options{
-				ImageName: "alpine:latest",
-			}
-			err := b.DownloadImage(opts)
-			Expect(err).ToNot(HaveOccurred())
-
-			layers, err := GenerateChanges(ctx, b, opts, opts)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(layers)).To(Equal(1))
-			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
-			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
-			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
-		})
-
-		It("Detects additions and changed files", func() {
-			err := b.DownloadImage(Options{
-				ImageName: "quay.io/mocaccino/micro",
-			})
-			Expect(err).ToNot(HaveOccurred())
-			err = b.DownloadImage(Options{
-				ImageName: "quay.io/mocaccino/extra",
-			})
-			Expect(err).ToNot(HaveOccurred())
-
-			layers, err := GenerateChanges(ctx, b, Options{
-				ImageName: "quay.io/mocaccino/micro",
-			}, Options{
-				ImageName: "quay.io/mocaccino/extra",
-			})
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(layers)).To(Equal(1))
-
-			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
-			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
-			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
-
-			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
-			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
-			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
-
-			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
-		})
-	})
-})

pkg/compiler/compiler.go

@@ -29,6 +29,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	bus "github.com/mudler/luet/pkg/bus"
@@ -280,23 +281,36 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co
 	}
 
 	cs.Options.Context.Info(pkgTag, ":hammer: Generating delta")
-	diffs, err := GenerateChanges(cs.Options.Context, cs.Backend, builderOpts, runnerOpts)
+
+	ref, err := cs.Backend.ImageReference(builderOpts.ImageName)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not generate changes from layers")
+		return nil, err
 	}
 
-	cs.Options.Context.Debug("Extracting image to grab files from delta")
+	ref2, err := cs.Backend.ImageReference(runnerOpts.ImageName)
-	if err := cs.Backend.ExtractRootfs(backend.Options{
-		ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions); err != nil {
-		return nil, errors.Wrap(err, "Could not extract rootfs")
-	}
-	artifact, err := artifact.ExtractArtifactFromDelta(cs.Options.Context, rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.Options.CompressionType)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not generate deltas")
+		return nil, err
 	}
 
-	artifact.CompileSpec = p
+	diff, err := image.Delta(ref, ref2)
-	return artifact, nil
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: includes/excludes might need to get "/" stripped from prefix
+	a, err := artifact.ImageToArtifact(
+		cs.Options.Context,
+		ref2,
+		cs.Options.CompressionType,
+		p.Rel(fmt.Sprintf("%s%s", p.GetPackage().GetFingerPrint(), ".package.tar")),
+		image.ExtractDeltaFiles(cs.Options.Context, diff, p.GetIncludes(), p.GetExcludes()),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	a.CompileSpec = p
+	return a, nil
 }
 
 func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,

tests/integration/01_rootfs.sh

@@ -12,7 +12,7 @@ oneTimeTearDown() {
 
 testBuild() {
     mkdir $tmpdir/testbuild
-    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c > /dev/null
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c
     buildst=$?
     assertEquals 'builds successfully' "$buildst" "0"
     assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"