Add crane-based methods for extraction

- create a new api package to encapsulate image manipulation
- use new api method to calculate delta

Fixes #258
Fixes #204
Fixes #90

pkg/compiler/backend.go

@@ -1,14 +1,8 @@
 package compiler
 
 import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/mudler/luet/pkg/api/core/types"
-	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 
 	"github.com/mudler/luet/pkg/compiler/backend"
 	"github.com/pkg/errors"
@@ -42,204 +36,6 @@ type CompilerBackend interface {
 	Push(opts backend.Options) error
 	ImageAvailable(string) bool
 
+	ImageReference(img1 string) (v1.Image, error)
 	ImageExists(string) bool
 }
-
-// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
-// example of json return: [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-func GenerateChanges(ctx *types.Context, b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) {
-
-	res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
-
-	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-
-	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(srcRootFS) // clean up
-
-	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(dstRootFS) // clean up
-
-	srcImageExtract := backend.Options{
-		ImageName:   fromImage.ImageName,
-		Destination: srcRootFS,
-	}
-	ctx.Debug("Extracting source image", fromImage.ImageName)
-	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
-	}
-
-	dstImageExtract := backend.Options{
-		ImageName:   toImage.ImageName,
-		Destination: dstRootFS,
-	}
-	ctx.Debug("Extracting destination image", toImage.ImageName)
-	err = b.ExtractRootfs(dstImageExtract, false)
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
-	}
-
-	// Get Additions/Changes. dst -> src
-	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
-		if info.IsDir() {
-			return nil
-		}
-
-		realpath := strings.Replace(path, dstRootFS, "", -1)
-		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
-		if err == nil {
-			var sizeA, sizeB int64
-			sizeA = fileInfo.Size()
-
-			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
-				sizeB = s.Size()
-			}
-
-			if sizeA != sizeB {
-				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
-				res.Diffs.Changes = append(res.Diffs.Changes, artifact.ArtifactNode{
-					Name: filepath.Join("/", realpath),
-					Size: int(sizeB),
-				})
-			} else {
-				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
-			}
-		} else {
-			var sizeB int64
-
-			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
-				sizeB = s.Size()
-			}
-			res.Diffs.Additions = append(res.Diffs.Additions, artifact.ArtifactNode{
-				Name: filepath.Join("/", realpath),
-				Size: int(sizeB),
-			})
-
-			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
-		}
-
-		return nil
-	})
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
-	}
-
-	// Get deletions. src -> dst
-	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
-		if info.IsDir() {
-			return nil
-		}
-
-		realpath := strings.Replace(path, srcRootFS, "", -1)
-		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
-			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
-			res.Diffs.Deletions = append(res.Diffs.Deletions, artifact.ArtifactNode{
-				Name: filepath.Join("/", realpath),
-			})
-		}
-
-		return nil
-	})
-	if err != nil {
-		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
-	}
-
-	diffs := []artifact.ArtifactLayer{res}
-
-	if ctx.Config.GetGeneral().Debug {
-		summary := ComputeArtifactLayerSummary(diffs)
-		for _, l := range summary.Layers {
-			ctx.Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
-				l.FromImage, l.ToImage,
-				l.AddFiles, l.AddSizes,
-				l.DelFiles, l.DelSizes,
-				l.ChangeFiles, l.ChangeSizes))
-		}
-	}
-
-	return diffs, nil
-}
-
-type ArtifactLayerSummary struct {
-	FromImage   string `json:"image1"`
-	ToImage     string `json:"image2"`
-	AddFiles    int    `json:"add_files"`
-	AddSizes    int64  `json:"add_sizes"`
-	DelFiles    int    `json:"del_files"`
-	DelSizes    int64  `json:"del_sizes"`
-	ChangeFiles int    `json:"change_files"`
-	ChangeSizes int64  `json:"change_sizes"`
-}
-
-type ArtifactLayersSummary struct {
-	Layers []ArtifactLayerSummary `json:"summary"`
-}
-
-func ComputeArtifactLayerSummary(diffs []artifact.ArtifactLayer) ArtifactLayersSummary {
-
-	ans := ArtifactLayersSummary{
-		Layers: make([]ArtifactLayerSummary, 0),
-	}
-
-	for _, layer := range diffs {
-		sum := ArtifactLayerSummary{
-			FromImage:   layer.FromImage,
-			ToImage:     layer.ToImage,
-			AddFiles:    0,
-			AddSizes:    0,
-			DelFiles:    0,
-			DelSizes:    0,
-			ChangeFiles: 0,
-			ChangeSizes: 0,
-		}
-		for _, a := range layer.Diffs.Additions {
-			sum.AddFiles++
-			sum.AddSizes += int64(a.Size)
-		}
-		for _, d := range layer.Diffs.Deletions {
-			sum.DelFiles++
-			sum.DelSizes += int64(d.Size)
-		}
-		for _, c := range layer.Diffs.Changes {
-			sum.ChangeFiles++
-			sum.ChangeSizes += int64(c.Size)
-		}
-		ans.Layers = append(ans.Layers, sum)
-	}
-
-	return ans
-}

pkg/compiler/backend/simpledocker.go

@@ -23,10 +23,13 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/daemon"
 	"github.com/mudler/luet/pkg/api/core/types"
 	bus "github.com/mudler/luet/pkg/bus"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	capi "github.com/mudler/docker-companion/api"
 
 	"github.com/mudler/luet/pkg/helpers"
@@ -144,6 +147,19 @@ func (s *SimpleDocker) Push(opts Options) error {
 	return nil
 }
 
+func (s *SimpleDocker) ImageReference(a string) (v1.Image, error) {
+	ref, err := name.ParseReference(a)
+	if err != nil {
+		return nil, err
+	}
+
+	img, err := daemon.Image(ref)
+	if err != nil {
+		return nil, err
+	}
+	return img, nil
+}
+
 func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")

pkg/compiler/backend/simpledocker_test.go

@@ -17,8 +17,6 @@ package backend_test
 
 import (
 	"github.com/mudler/luet/pkg/api/core/types"
-	"github.com/mudler/luet/pkg/api/core/types/artifact"
-	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
@@ -107,35 +105,6 @@ RUN echo bar > /test2`))
 			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
 			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
-			artifacts := []artifact.ArtifactNode{{
-				Name: "/luetbuild/LuetDockerfile",
-				Size: 175,
-			}}
-			if os.Getenv("DOCKER_BUILDKIT") == "1" {
-				artifacts = append(artifacts, artifact.ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
-			}
-			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4})
-			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4})
-
-			Expect(compiler.GenerateChanges(ctx, b, opts, opts2)).To(Equal(
-				[]artifact.ArtifactLayer{{
-					FromImage: "luet/base",
-					ToImage:   "test",
-					Diffs: artifact.ArtifactDiffs{
-						Additions: artifacts,
-					},
-				}}))
-
-			opts2 = backend.Options{
-				ImageName:      "test",
-				SourcePath:     tmpdir,
-				DockerFileName: "LuetDockerfile",
-				Destination:    filepath.Join(tmpdir, "output3.tar"),
-			}
-
-			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
-			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
-			Expect(b.ImageExists(opts2.ImageName)).To(BeFalse())
 		})
 
 		It("Detects available images", func() {

pkg/compiler/backend/simpleimg.go

@@ -20,6 +20,8 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/google/go-containerregistry/pkg/crane"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/mudler/luet/pkg/api/core/types"
 	bus "github.com/mudler/luet/pkg/bus"
 
@@ -70,6 +72,29 @@ func (s *SimpleImg) RemoveImage(opts Options) error {
 	return nil
 }
 
+func (s *SimpleImg) ImageReference(a string) (v1.Image, error) {
+
+	f, err := s.ctx.Config.GetSystem().TempFile("snapshot")
+	if err != nil {
+		return nil, err
+	}
+	buildarg := []string{"save", a, "-o", f.Name()}
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
+
+	out, err := exec.Command("img", buildarg...).CombinedOutput()
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed saving image: "+string(out))
+	}
+
+	img, err := crane.Load(f.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	return img, nil
+}
+
 func (s *SimpleImg) DownloadImage(opts Options) error {
 	name := opts.ImageName
 	bus.Manager.Publish(bus.EventImagePrePull, opts)

pkg/compiler/backend_test.go

@@ -1,79 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler_test
-
-import (
-	"github.com/mudler/luet/pkg/api/core/types"
-	. "github.com/mudler/luet/pkg/compiler"
-	. "github.com/mudler/luet/pkg/compiler/backend"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Docker image diffs", func() {
-	var b CompilerBackend
-	ctx := types.NewContext()
-	BeforeEach(func() {
-		b = NewSimpleDockerBackend(ctx)
-	})
-
-	Context("Generate diffs from docker images", func() {
-		It("Detect no changes", func() {
-			opts := Options{
-				ImageName: "alpine:latest",
-			}
-			err := b.DownloadImage(opts)
-			Expect(err).ToNot(HaveOccurred())
-
-			layers, err := GenerateChanges(ctx, b, opts, opts)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(layers)).To(Equal(1))
-			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
-			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
-			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
-		})
-
-		It("Detects additions and changed files", func() {
-			err := b.DownloadImage(Options{
-				ImageName: "quay.io/mocaccino/micro",
-			})
-			Expect(err).ToNot(HaveOccurred())
-			err = b.DownloadImage(Options{
-				ImageName: "quay.io/mocaccino/extra",
-			})
-			Expect(err).ToNot(HaveOccurred())
-
-			layers, err := GenerateChanges(ctx, b, Options{
-				ImageName: "quay.io/mocaccino/micro",
-			}, Options{
-				ImageName: "quay.io/mocaccino/extra",
-			})
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(layers)).To(Equal(1))
-
-			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
-			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
-			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
-
-			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
-			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
-			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
-
-			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
-		})
-	})
-})

pkg/compiler/compiler.go

@@ -29,6 +29,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
 	bus "github.com/mudler/luet/pkg/bus"
@@ -280,23 +281,36 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co
 	}
 
 	cs.Options.Context.Info(pkgTag, ":hammer: Generating delta")
-	diffs, err := GenerateChanges(cs.Options.Context, cs.Backend, builderOpts, runnerOpts)
+
+	ref, err := cs.Backend.ImageReference(builderOpts.ImageName)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not generate changes from layers")
+		return nil, err
 	}
 
-	cs.Options.Context.Debug("Extracting image to grab files from delta")
-	if err := cs.Backend.ExtractRootfs(backend.Options{
-		ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions); err != nil {
-		return nil, errors.Wrap(err, "Could not extract rootfs")
-	}
-	artifact, err := artifact.ExtractArtifactFromDelta(cs.Options.Context, rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.Options.CompressionType)
+	ref2, err := cs.Backend.ImageReference(runnerOpts.ImageName)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not generate deltas")
+		return nil, err
 	}
 
-	artifact.CompileSpec = p
-	return artifact, nil
+	diff, err := image.Delta(ref, ref2)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: includes/excludes might need to get "/" stripped from prefix
+	a, err := artifact.ImageToArtifact(
+		cs.Options.Context,
+		ref2,
+		cs.Options.CompressionType,
+		p.Rel(fmt.Sprintf("%s%s", p.GetPackage().GetFingerPrint(), ".package.tar")),
+		image.ExtractDeltaFiles(cs.Options.Context, diff, p.GetIncludes(), p.GetExcludes()),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	a.CompileSpec = p
+	return a, nil
 }
 
 func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,